Commit 6aaf2c71 authored by Michal 'vorner' Vaner's avatar Michal 'vorner' Vaner

Merge branch 'migration-base' into updater-ng

parents 1a6825af 8162ec18
......@@ -10,6 +10,7 @@
*.so
/src/opkg-trans/opkg-trans
/src/pkgupdate/pkgupdate
/src/migrator/pkgmigrate
# Stuff dump core all over the place.
core
vgcore.*
......
SRC_SUBDIRS := lib opkg-trans pkgupdate
SRC_SUBDIRS := lib opkg-trans pkgupdate migrator
include $(patsubst %,$(S)/src/%/Makefile.dir,$(SRC_SUBDIRS))
LIBRARIES += src/lib/libupdater
OPKG_AUTOLOAD := $(abspath $(wildcard $(S)/src/lib/autoload/*.lua))
$(O)/.gen/src/lib/lautoload.embedlist: $(S)/src/lib/gen_embed.sh $(S)/src/lib/embed_types.h $(OPKG_AUTOLOAD)
$(O)/.gen/src/%.embedlist: $(S)/src/lib/gen_embed.sh $(S)/src/lib/embed_types.h
$(M) GEN $@
$(Q)mkdir -p $(dir $@)
$(Q)$< .lua autoload $(S)/src/lib/embed_types.h $(OPKG_AUTOLOAD) >$@
$(Q)$< "$(SUFFIX)" $(basename $(notdir $@)) $(S)/src/lib/embed_types.h $(abspath $(filter-out $(S)/src/lib/gen_embed.sh, $(filter-out $(S)/src/lib/embed_types.h, $^))) >$@
$(O)/.gen/src/lib/lautoload.embedlist: $(wildcard $(S)/src/lib/autoload/*.lua)
$(O)/.gen/src/lib/lautoload.embedlist: SUFFIX := .lua
libupdater_MODULES := \
arguments \
......
......@@ -68,6 +68,8 @@ static const char *opt_help[COT_LAST] = {
"--ask-approval=<report-file> Require user's approval to proceed (abort if --approve with appropriate ID is not present, plan of action is put into the report-file if approval is needed)\n",
[COT_APPROVE] =
"--approve=<id> Approve actions with given ID (multiple allowed, from a corresponding report-file).\n",
[COT_OUTPUT] =
"--output=<file> Put the output to given file.\n",
[COT_TASK_LOG] =
"--task-log=<file> Append list of executed tasks into a log file.\n"
};
......@@ -78,6 +80,7 @@ enum option_val {
OPT_REEXEC_VAL,
OPT_ASK_APPROVAL_VAL,
OPT_APPROVE_VAL,
OPT_OUTPUT,
OPT_TASK_LOG_VAL,
OPT_LAST
};
......@@ -93,6 +96,7 @@ static const struct option opt_long[] = {
{ .name = "state-log", .has_arg = no_argument, .val = OPT_STATE_LOG_VAL },
{ .name = "ask-approval", .has_arg = required_argument, .val = OPT_ASK_APPROVAL_VAL },
{ .name = "approve", .has_arg = required_argument, .val = OPT_APPROVE_VAL },
{ .name = "output", .has_arg = required_argument, .val = OPT_OUTPUT },
{ .name = "task-log", .has_arg = required_argument, .val = OPT_TASK_LOG_VAL },
{ .name = NULL }
};
......@@ -111,6 +115,7 @@ static const struct simple_opt {
[OPT_STATE_LOG_VAL] = { COT_STATE_LOG, false, true },
[OPT_ASK_APPROVAL_VAL] = { COT_ASK_APPROVAL, true, true },
[OPT_APPROVE_VAL] = { COT_APPROVE, true, true },
[OPT_OUTPUT] = { COT_OUTPUT, true, true },
[OPT_TASK_LOG_VAL] = { COT_TASK_LOG, true, true }
};
......@@ -214,6 +219,7 @@ struct cmd_op *cmd_args_parse(int argc, char *argv[], const enum cmd_op_type acc
case COT_STDERR_LEVEL:
case COT_SYSLOG_NAME:
case COT_ASK_APPROVAL:
case COT_OUTPUT:
case COT_APPROVE:
case COT_TASK_LOG: {
struct cmd_op tmp = result[i];
......
......@@ -56,6 +56,8 @@ enum cmd_op_type {
COT_STDERR_LEVEL,
// Name of the syslog
COT_SYSLOG_NAME,
// Put the output into a file
COT_OUTPUT,
// Log tasks into a file
COT_TASK_LOG,
// Argument isn't option.
......
......@@ -36,11 +36,11 @@ local transaction = require "transaction"
module "updater"
-- luacheck: globals prepare cleanup
-- luacheck: globals prepare cleanup required_pkgs
local cleanup_actions = {}
function prepare(entrypoint)
function required_pkgs(entrypoint)
-- Get the top-level script
local tlc = sandbox.new('Full')
local ep_uri = uri(tlc, entrypoint)
......@@ -56,7 +56,11 @@ function prepare(entrypoint)
state_dump("examine")
-- Go through all the requirements and decide what we need
postprocess.run()
local required = planner.required_pkgs(postprocess.available_packages, requests.content_requests)
return planner.required_pkgs(postprocess.available_packages, requests.content_requests)
end
function prepare(entrypoint)
local required = required_pkgs(entrypoint)
local run_state = backend.run_state()
backend.flags_load()
local tasks = planner.filter_required(run_state.status, required)
......
--[[
Copyright 2016, CZ.NIC z.s.p.o. (http://www.nic.cz/)
This file is part of the turris updater.
Updater is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Updater is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Updater. If not, see <http://www.gnu.org/licenses/>.
]]--
-- luacheck: globals extra_pkgs pkgs_format
local pairs = pairs
local table = table
local utils = require "utils"
local backend = require "backend"
local updater = require "updater"
module "migrator"
function extra_pkgs(entry_point)
local requested = updater.required_pkgs(entry_point)
local requested_set = {}
for _, task in pairs(requested) do
if task.action == "require" or task.action == "reinstall" then
requested_set[task.name] = true
end
end
local installed = backend.status_parse()
local result = {}
for name, pkg in pairs(installed) do
if utils.multi_index(pkg, "Status", 3) == "installed" then
if not requested_set[name] then
result[name] = true
end
end
end
return result
end
function pkgs_format(pkgs, prefix, suffix)
local arr = utils.set2arr(pkgs)
table.sort(arr)
return table.concat(utils.map(arr, function (idx, name)
return idx, prefix .. name .. suffix .. '\n'
end))
end
return _M
......@@ -47,7 +47,7 @@
static const char *crash_file = "/tmp/updater_crash.log";
// From the embed file, lua things that are auto-loaded
extern struct file_index_element autoload[];
extern struct file_index_element lautoload[];
struct interpreter {
lua_State *state;
......@@ -178,6 +178,10 @@ static void extract_registry_value(lua_State *L, char *name) {
lua_getfield(L, LUA_REGISTRYINDEX, REGISTRY_NAME);
// Get the value
lua_getfield(L, -1, name);
// Delete the value the from registry table
lua_pushnil(L);
// The table is now at -3, because we added the result and the nil on top of stack
lua_setfield(L, -3, name);
// Remove the registry table
lua_remove(L, -2);
......@@ -839,7 +843,7 @@ const char *interpreter_include(struct interpreter *interpreter, const char *cod
}
const char *interpreter_autoload(struct interpreter *interpreter) {
for (struct file_index_element *el = autoload; el->name; el ++) {
for (struct file_index_element *el = lautoload; el->name; el ++) {
const char *underscore = rindex(el->name, '_');
// Use the part after the last underscore as the name
const char *name = underscore ? underscore + 1 : el->name;
......@@ -929,6 +933,16 @@ const char *interpreter_call(struct interpreter *interpreter, const char *functi
break;
}
CASE(double, 'f', number);
case 'r': {
const char *s = va_arg(args, const char *);
// Get the registry table
lua_getfield(L, LUA_REGISTRYINDEX, REGISTRY_NAME);
// Get the value
lua_getfield(L, -1, s);
// Remove the registry table. This way we added only the extracted value.
lua_remove(L, -2);
break;
}
default:
DIE("Unknown type specifier '%c' passed", *param_spec);
#undef CASE
......@@ -998,6 +1012,11 @@ int interpreter_collect_results(struct interpreter *interpreter, const char *spe
} else
return pos;
break;
case 'r': {
char **name = va_arg(args, char **);
*name = register_value(L, pos + 1);
break;
}
default:
DIE("Invalid type specifier '%c' passed", *spec);
}
......@@ -1007,6 +1026,12 @@ int interpreter_collect_results(struct interpreter *interpreter, const char *spe
return -1;
}
void interpreter_registry_release(struct interpreter *interpreter, char *name) {
extract_registry_value(interpreter->state, name);
// Get rid of the value on top of the stack as well
lua_pop(interpreter->state, 1);
}
void interpreter_destroy(struct interpreter *interpreter) {
ASSERT(interpreter->state);
lua_close(interpreter->state);
......
......@@ -86,6 +86,9 @@ const char *interpreter_autoload(struct interpreter *interpreter) __attribute__(
* - s: string (null-terminated)
* - S: binary string (with extra parameter ‒ size_t ‒ length)
* - f: double
* - r: A value in registry. When collecting, it is stored in the registry and a string is
* stored in corresponding variable. When calling, pass the name back and it will be
* extracted. Free it with interpreter_registry_release after it is no longer needed.
*
* We use the „usual“ C types (eg. int, not lua_Integer). These functions may not be
* used in case of more complex data types.
......@@ -100,6 +103,9 @@ const char *interpreter_autoload(struct interpreter *interpreter) __attribute__(
*/
const char *interpreter_call(struct interpreter *interpreter, const char *function, size_t *result_count, const char *param_spec, ...);
int interpreter_collect_results(struct interpreter *interpreter, const char *spec, ...);
// Release the value stored as 'r' type from interpreter_collect_results. Free the name parameter.
void interpreter_registry_release(struct interpreter *interpreter, char *name);
/*
* Destroy an interpreter and return its memory.
*/
......
RESTRICT := src/migrator
RELATIVE := ../../
include $(RELATIVE)/Makefile
BINARIES += src/migrator/pkgmigrate
# Borrow the file from there. But for now, we embed just this one.
$(O)/.gen/src/migrator/uriinternal.embedlist: $(S)/src/pkgupdate/internal/entry.lua
pkgmigrate_MODULES := \
main \
uriinternal.embed
pkgmigrate_LOCAL_LIBS := updater
/*
* Copyright 2016, CZ.NIC z.s.p.o. (http://www.nic.cz/)
*
* This file is part of the turris updater.
*
* Updater is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* Updater is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with Updater. If not, see <http://www.gnu.org/licenses/>.
*/
#include "../lib/util.h"
#include "../lib/arguments.h"
#include "../lib/events.h"
#include "../lib/interpreter.h"
#include <stdio.h>
#include <string.h>
#include <errno.h>
extern struct file_index_element uriinternal[];
static const enum cmd_op_type cmd_op_allowed[] = {
COT_BATCH, COT_NO_OP, COT_ROOT_DIR, COT_SYSLOG_LEVEL, COT_STDERR_LEVEL, COT_SYSLOG_NAME, COT_OUTPUT, COT_LAST
};
void print_help() {
fputs("Usage: pkgmigrate [OPTION]...\n", stderr);
cmd_args_help(cmd_op_allowed);
}
int main(int argc, char *argv[]) {
// Set up logging machinery
log_stderr_level(LL_INFO);
log_syslog_level(LL_INFO);
// Parse the arguments
struct cmd_op *ops = cmd_args_parse(argc, argv, cmd_op_allowed);
struct cmd_op *op = ops;
const char *top_level_config = "internal:entry_lua";
const char *root_dir = NULL;
const char *output = "/etc/updater/auto.lua";
bool batch = false, early_exit = false;
for (; op->type != COT_EXIT && op->type != COT_CRASH; op ++)
switch (op->type) {
case COT_HELP: {
print_help();
early_exit = true;
break;
}
case COT_ERR_MSG: {
fputs(op->parameter, stderr);
break;
}
case COT_NO_OP:
top_level_config = op->parameter;
break;
case COT_BATCH:
batch = true;
break;
case COT_ROOT_DIR:
root_dir = op->parameter;
break;
case COT_SYSLOG_LEVEL: {
enum log_level level = log_level_get(op->parameter);
ASSERT_MSG(level != LL_UNKNOWN, "Unknown log level %s", op->parameter);
log_syslog_level(level);
break;
}
case COT_SYSLOG_NAME:
log_syslog_name(op->parameter);
break;
case COT_STDERR_LEVEL: {
enum log_level level = log_level_get(op->parameter);
ASSERT_MSG(level != LL_UNKNOWN, "Unknown log level %s", op->parameter);
log_stderr_level(level);
break;
}
case COT_OUTPUT:
output = op->parameter;
break;
default:
DIE("Unknown COT");
}
enum cmd_op_type exit_type = op->type;
free(ops);
// The interpreter and other environment
struct events *events = events_new();
struct interpreter *interpreter = interpreter_create(events, uriinternal);
const char *error = interpreter_autoload(interpreter);
ASSERT_MSG(!error, "%s", error);
if (root_dir) {
error = interpreter_call(interpreter, "backend.root_dir_set", NULL, "s", root_dir);
ASSERT_MSG(!error, "%s", error);
}
if (early_exit)
goto CLEANUP;
// Let it compute the packages we are interested in
size_t result_count;
error = interpreter_call(interpreter, "migrator.extra_pkgs", &result_count, "s", top_level_config);
ASSERT_MSG(!error, "%s", error);
ASSERT_MSG(result_count == 1, "Wrong number of results of migrator.extra_pkgs: %zu", result_count);
// As the result is a table, we want to store it to the registry
char *extra_pkg_table;
ASSERT_MSG(interpreter_collect_results(interpreter, "r", &extra_pkg_table) == -1, "Couldn't store the result table");
if (!batch) {
// We are in the interactive mode. Ask for confirmation
printf("There are the extra packages I'll put into %s:\n", output);
error = interpreter_call(interpreter, "migrator.pkgs_format", &result_count, "rss", extra_pkg_table, " • ", "");
ASSERT_MSG(!error, "%s", error);
const char *pkg_list;
ASSERT_MSG(result_count == 1, "Wrong number of results of migrator.pkgs_format");
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &pkg_list) == -1, "Couldn't extract package list");
// Use fputs, as puts adds extra \n at the end
fputs(pkg_list, stdout);
puts("Press return to continue, CTRL+C to abort");
getchar();
}
// Compute the list of packages to install additionally and store it into the file
error = interpreter_call(interpreter, "migrator.pkgs_format", &result_count, "rss", extra_pkg_table, "Install \"", "\"");
ASSERT_MSG(!error, "%s", error);
const char *install_list;
ASSERT_MSG(result_count == 1, "Wrong number of results of migrator.pkgs_format");
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &install_list) == -1, "Couldn't extract package installation list");
FILE *fout = fopen(output, "w");
ASSERT_MSG(fout, "Couldn't open output file %s: %s\n", output, strerror(errno));
fprintf(fout, "-- Auto-migration performed (do not delete this line, or it may attempt doing so again)\n");
fputs(install_list, fout);
fclose(fout);
interpreter_registry_release(interpreter, extra_pkg_table);
CLEANUP:
interpreter_destroy(interpreter);
events_destroy(events);
if (exit_type == COT_EXIT)
return 0;
else
return 1;
}
BINARIES += src/pkgupdate/pkgupdate
PKGUPDATE_EMBED := $(abspath $(wildcard $(S)/src/pkgupdate/internal/*))
$(O)/.gen/src/pkgupdate/uriinternal.embedlist: $(S)/src/lib/gen_embed.sh $(S)/src/lib/embed_types.h $(PKGUPDATE_EMBED)
$(M) GEN $@
$(Q)mkdir -p $(dir $@)
$(Q)$< "" uriinternal $(S)/src/lib/embed_types.h $(PKGUPDATE_EMBED) >$@
$(O)/.gen/src/pkgupdate/uriinternal.embedlist: $(wildcard $(S)/src/pkgupdate/internal/*)
pkgupdate_MODULES := \
main \
......
......@@ -50,7 +50,7 @@ static const enum cmd_op_type cmd_op_allows[] = {
};
static void print_help() {
fputs("Usage: updater [OPTION]...\n", stderr);
fputs("Usage: pkgupdate [OPTION]...\n", stderr);
cmd_args_help(cmd_op_allows);
}
......
......@@ -30,6 +30,7 @@ LUA_TESTS := \
events \
interpreter \
journal \
migrator \
planner \
postprocess \
requests \
......
......@@ -262,6 +262,35 @@ START_INTERPRETER_TEST(test_mkdtemp) {
}
END_INTERPRETER_TEST
START_INTERPRETER_TEST(call_registry) {
size_t results;
const char *error = interpreter_call(interpreter, "testing.values", &results, "");
ck_assert_msg(!error, "Failed to run the function: %s", error);
ck_assert_uint_eq(2, results);
char *n1, *n2;
// Extract the two values to registry
ck_assert_int_eq(-1, interpreter_collect_results(interpreter, "rr", &n1, &n2));
// Use one of them as an input to other call
error = interpreter_call(interpreter, "testing.subtable.echo", &results, "r", n2);
ck_assert_msg(!error, "Failed to run the function: %s", error);
ck_assert_uint_eq(1, results);
// Check the value matches
const char *s;
ck_assert_int_eq(-1, interpreter_collect_results(interpreter, "s", &s));
ck_assert_str_eq(s, "hello");
// Try with the other one
error = interpreter_call(interpreter, "testing.subtable.echo", &results, "r", n1);
ck_assert_msg(!error, "Failed to run the function: %s", error);
ck_assert_uint_eq(1, results);
int i;
ck_assert_int_eq(-1, interpreter_collect_results(interpreter, "i", &i));
ck_assert_int_eq(42, i);
// Free them
interpreter_registry_release(interpreter, n1);
interpreter_registry_release(interpreter, n2);
}
END_INTERPRETER_TEST
Suite *gen_test_suite(void) {
Suite *result = suite_create("Lua interpreter");
TCase *interpreter = tcase_create("loading");
......@@ -273,6 +302,7 @@ Suite *gen_test_suite(void) {
tcase_add_test(interpreter, call_method);
tcase_add_test(interpreter, call_echo);
tcase_add_test(interpreter, test_mkdtemp);
tcase_add_test(interpreter, call_registry);
suite_add_tcase(result, interpreter);
return result;
}
......@@ -14,5 +14,6 @@ globals = {
-- From dumper
"DataDumper",
-- Modules
"stacktraceplus", "utils", "backend", "transaction", "uri", "requests", "sandbox", "postprocess", "planner", "updater",
"stacktraceplus", "utils", "backend", "transaction", "uri", "requests", "sandbox", "postprocess", "planner", "updater", "migrator"
}
--[[
Copyright 2016, CZ.NIC z.s.p.o. (http://www.nic.cz/)
This file is part of the turris updater.
Updater is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Updater is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Updater. If not, see <http://www.gnu.org/licenses/>.
]]--
require "lunit"
local migrator = require "migrator"
module("migrator-tests", package.seeall, lunit.testcase)
function test_pkgs_format()
local pkgs = {YYY = true, XXX = true}
assert_equal([[
>>XXX<<<
>>YYY<<<
]], migrator.pkgs_format(pkgs, ">>", "<<<"))
assert_equal("", migrator.pkgs_format({}, "", ""))
assert_equal("", migrator.pkgs_format({}, "!!!", "???"))
assert_equal([[
XXX
YYY
]], migrator.pkgs_format(pkgs, "", ""))
end
function test_extra_pkgs()
mock_gen("updater.required_pkgs", function () return {
{
action = 'reinstall',
name = 'pkg1'
},
{
action = 'require',
name = 'pkg2'
},
{
action = 'remove',
name = 'pkg3'
}
} end)
mock_gen("backend.status_parse", function ()
-- A little bit smaller than the real thing, but this should be enough for the test
local status = utils.map({'pkg1', 'pkg2', 'pkg3', 'pkg4', 'pkg5'}, function (_, name)
return name, {
Package = name,
Status = {'install', 'user', 'installed'}
}
end)
-- This one is mentioned but not actually installed, so check it doesn't get confused by that
status.pkg4.Status[3] = 'not-installed'
return status
end)
local result = migrator.extra_pkgs('epoint')
--[[
pkg1 not present, since it is required by the current configs
pkg2 as well
pkg3 is being removed by the current config, so it is present
pkg4 is not installed, so it is not added
pkg5 is installed but not mentioned by the planner, so we want to add it (this situation probably never happens in real life, though).
]]
assert_table_equal({pkg3 = true, pkg5 = true}, result)
assert_table_equal({
{
f = "updater.required_pkgs",
p = {"epoint"}
},
{
f = "backend.status_parse",
p = {}
}
}, mocks_called)
end
function teardown()
mocks_reset()
end
......@@ -11,6 +11,9 @@ UPD_SYS_TESTS := \
steal-confs \
simple-update
MIGRATE_SYS_TESTS := \
auto-migrate-pkgs
.PHONY: test-sys valgrind-sys check
check: test-sys valgrind-sys
......@@ -31,3 +34,4 @@ endef
$(eval $(foreach TEST,$(TRANS_SYS_TESTS),$(call DO_SYS_TEST,opkg-trans,$(TEST))))
$(eval $(foreach TEST,$(UPD_SYS_TESTS),$(call DO_SYS_TEST,pkgupdate,$(TEST))))
$(eval $(foreach TEST,$(MIGRATE_SYS_TESTS),$(call DO_SYS_TEST,pkgmigrate,$(TEST))))
rm "$ROOT_DIR"/config
rm -r "$ROOT_DIR"/repo
Repository "test" "file://%ROOT_DIR%/repo"
Install "test-package"
#!/bin/sh
echo "Test script installed"
Package: extra-pkg
Version: 1
Source: package/test
License: GPL-2.0
Section: none
Architecture: all
Installed-Size: 24000
Description: Test package. Nothing special here.
Package: test-package
Version: 1
Source: package/test
License: GPL-2.0
Section: none
Architecture: all
Installed-Size: 24000
Description: Test package. Nothing special here.
f299060e0383392ebeac64b714eca7e3 /etc/test
94addd51a7efec8c4d9203f5c14d9471 /usr/bin/test.sh
b8f6d3ba5e12aae70a72ac5dff61f039 /usr/share/test/package-test
/usr/bin/test.sh
/usr/share/test/package-test
/etc/test
Package: test-package
Version: 1
Status: install user installed
Architecture: all
Conffiles:
/etc/test 37d2046a395cbfcb2712ff5c96a727b1966876080047c56717009dbbc235f566
Installed-Time: 1471445715
Package: extra-pkg
Version: 1
Status: install user installed
Architecture: all
Installed-Time: 1471445715
-- Auto-migration performed (do not delete this line, or it may attempt doing so again)
Install "extra-pkg"
#!/bin/sh
echo "Test script installed"
Package: extra-pkg
Version: 1
Source: package/test
License: GPL-2.0
Section: none
Architecture: all
Installed-Size: 24000
Description: Test package. Nothing special here.