Verified Commit 0be0315a authored by Karel Koci's avatar Karel Koci 🤘

updater: do not download package unless we know that it is approved

We do not download any package unless we know that they are going to be
installed with this commit. It is done by split of download to separate
function.
parent 5b9b4d2f
......@@ -51,12 +51,11 @@ local LS_CLEANUP = LS_CLEANUP
local update_state = update_state
local sync = sync
local log_event = log_event
local sha256 = sha256
local system_reboot = system_reboot
module "transaction"
-- luacheck: globals perform recover empty perform_queue recover_pretty queue_remove queue_install queue_install_downloaded approval_hash task_report cleanup_actions
-- luacheck: globals perform recover perform_queue recover_pretty queue_remove queue_install queue_install_downloaded cleanup_actions
-- Wrap the call to the maintainer script, and store any possible errors for later use
local function script(errors_collected, name, suffix, ...)
......@@ -410,23 +409,16 @@ local function errors_format(errors)
end
end
function empty()
return not next(queue)
end
--[[
Run transaction of the queued operations.
]]
function perform_queue()
if empty() then
return true
else
-- Ensure we reset the queue by running it. And also that we allow the garbage collector to collect the data in there.
local queue_cp = queue
queue = {}
collectgarbage() -- explicitly try to collect queue
return errors_format(perform(queue_cp))
end
if not next(queue) then return true end
-- Ensure we reset the queue by running it. And also that we allow the garbage collector to collect the data in there.
local queue_cp = queue
queue = {}
collectgarbage() -- explicitly try to collect queue
return errors_format(perform(queue_cp))
end
-- Just like recover, but with the result formatted.
......@@ -455,28 +447,4 @@ function queue_install_downloaded(file, name, version, modifier)
})
end
local function queued_tasks(extensive)
return utils.map(queue, function (i, task)
local d = {task.op, task.version or '-', task.name}
if extensive then
table.insert(d, task.reboot or '-')
end
return i, table.concat(d, ' ') .. "\n"
end)
end
-- Compute the approval hash of the queued operations
function approval_hash()
-- Convert the tasks into formatted lines, sort them and hash it.
local requests = queued_tasks(true)
table.sort(requests)
return sha256(table.concat(requests))
end
-- Provide a human-readable report of the queued tasks
function task_report(prefix, extensive)
prefix = prefix or ''
return table.concat(utils.map(queued_tasks(extensive), function (i, str) return i, prefix .. str end))
end
return _M
......@@ -17,10 +17,13 @@ You should have received a copy of the GNU General Public License
along with Updater. If not, see <http://www.gnu.org/licenses/>.
]]--
local next = next
local error = error
local ipairs = ipairs
local table = table
local WARN = WARN
local INFO = INFO
local DIE = DIE
local md5 = md5
local sha256 = sha256
local reexec = reexec
......@@ -41,14 +44,17 @@ local transaction = require "transaction"
module "updater"
-- luacheck: globals prepare pre_cleanup cleanup required_pkgs disable_replan
-- luacheck: globals tasks prepare no_tasks tasks_to_transaction pre_cleanup cleanup disable_replan approval_hash task_report
-- Prepared tasks
tasks = {}
local allow_replan = true
function disable_replan()
allow_replan = false
end
function required_pkgs(entrypoint)
local function required_pkgs(entrypoint)
-- Get the top-level script
local tlc = sandbox.new('Full')
local ep_uri = uri(tlc, entrypoint)
......@@ -73,16 +79,34 @@ function prepare(entrypoint)
end
local required = required_pkgs(entrypoint)
local run_state = backend.run_state()
local tasks = planner.filter_required(run_state.status, required, allow_replan)
tasks = planner.filter_required(run_state.status, required, allow_replan)
for _, task in ipairs(tasks) do
if task.action == "require" then
-- TODO downgrade and so on?
INFO("Queue install of " .. task.name .. "/" .. task.package.repo.name .. "/" .. task.package.Version)
elseif task.action == "remove" then
INFO("Queue removal of " .. task.name)
else
DIE("Unknown action " .. task.action)
end
end
end
-- Check if we have some tasks
function no_tasks()
return not next(tasks)
end
-- Download all packages and push tasks to transaction
function tasks_to_transaction()
INFO("Downloading packages")
update_state(LS_DOWN)
--[[
Start download of all the packages. They all start (or queue, if there are
too many). We then start taking them one by one, but that doesn't stop it
from being downloaded in any order.
]]
-- Start packages download
for _, task in ipairs(tasks) do
if task.action == "require" and not task.package.data then -- if we already have data, skip downloading
-- Strip sig verification off, packages from repos don't have their own .sig files, but they are checked by hashes in the (already checked) index.
if task.action == "require" then
-- Strip sig verification off, packages from repos don't have their own .sig
-- files, but they are checked by hashes in the (already checked) index.
local veriopts = utils.shallow_copy(task.package.repo)
local veri = veriopts.verification or utils.private(task.package.repo).context.verification or 'both'
if veri == 'both' then
......@@ -101,7 +125,6 @@ function prepare(entrypoint)
if task.action == "require" then
local ok, data = task.real_uri:get()
if not ok then error(data) end
INFO("Queue install of " .. task.name .. "/" .. task.package.repo.name .. "/" .. task.package.Version)
if task.package.MD5Sum then
local sum = md5(data)
if sum ~= task.package.MD5Sum then
......@@ -118,7 +141,6 @@ function prepare(entrypoint)
utils.write_file(fpath, data)
transaction.queue_install_downloaded(fpath, task.name, task.package.Version, task.modifier)
elseif task.action == "remove" then
INFO("Queue removal of " .. task.name)
transaction.queue_remove(task.name)
else
DIE("Unknown action " .. task.action)
......@@ -126,6 +148,35 @@ function prepare(entrypoint)
end
end
local function queued_tasks(extensive)
return utils.map(tasks, function (i, task)
local d = {task.action, utils.multi_index(task, "package", "Version") or '-', task.name}
if d[1] == "require" then
d[1] = "install"
elseif d[1] == "remove" then
d[2] = '-'
end -- Just to be backward compatible require=install and remove does not have version
if extensive then
table.insert(d, utils.multi_index(task, "modifier", "reboot") or '-')
end
return i, table.concat(d, ' ') .. "\n"
end)
end
-- Compute the approval hash of the queued operations
function approval_hash()
-- Convert the tasks into formatted lines, sort them and hash it.
local reqs = queued_tasks(true)
table.sort(reqs)
return sha256(table.concat(reqs))
end
-- Provide a human-readable report of the queued tasks
function task_report(prefix, extensive)
prefix = prefix or ''
return table.concat(utils.map(queued_tasks(extensive), function (i, str) return i, prefix .. str end))
end
-- Only cleanup actions that we want to give chance to program to react on
function pre_cleanup()
local reboot_delayed = false
......
......@@ -69,11 +69,11 @@ static bool approved(struct interpreter *interpreter, const char *approval_file,
// We need to ask for approval. But we may have gotten it already.
// Compute the hash of our plan first
size_t result_count;
const char *err = interpreter_call(interpreter, "transaction.approval_hash", &result_count, "");
const char *err = interpreter_call(interpreter, "updater.approval_hash", &result_count, "");
ASSERT_MSG(!err, "%s", err);
ASSERT_MSG(result_count == 1, "Wrong number of results from transaction.approval_hash: %zu", result_count);
ASSERT_MSG(result_count == 1, "Wrong number of results from updater.approval_hash: %zu", result_count);
const char *hash;
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &hash) == -1, "The result of transaction.approval_hash is not a string");
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &hash) == -1, "The result of updater.approval_hash is not a string");
for (size_t i = 0; i < approval_count; i ++)
if (strcmp(approvals[i], hash) == 0) {
// Yes, this is approved plan of actions. Go ahead.
......@@ -87,13 +87,14 @@ static bool approved(struct interpreter *interpreter, const char *approval_file,
// Note we need to write the hash out before we start manipulating interpreter again
fputs(hash, report_file);
fputc('\n', report_file);
err = interpreter_call(interpreter, "transaction.task_report", &result_count, "sb", "", true);
err = interpreter_call(interpreter, "updater.task_report", &result_count, "sb", "", true);
ASSERT_MSG(!err, "%s", err);
ASSERT_MSG(result_count == 1, "Wrong number of results from transaction.task_report: %zu", result_count);
ASSERT_MSG(result_count == 1, "Wrong number of results from updater.task_report: %zu", result_count);
const char *report;
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &report) == -1, "The result of transaction.task_report is not a string");
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &report) == -1, "The result of updater.task_report is not a string");
fputs(report, report_file);
fclose(report_file);
INFO("Approval request generated");
return false;
}
......@@ -256,12 +257,12 @@ int main(int argc, char *argv[]) {
err_dump(err);
GOTO_CLEANUP;
}
err = interpreter_call(interpreter, "transaction.empty", &result_count, "");
err = interpreter_call(interpreter, "updater.no_tasks", &result_count, "");
ASSERT_MSG(!err, "%s", err);
ASSERT_MSG(result_count == 1, "Wrong number of results of transaction.empty");
bool trans_empty;
ASSERT_MSG(interpreter_collect_results(interpreter, "b", &trans_empty) == -1, "The result of transaction.empty is not bool");
if (trans_empty) {
ASSERT_MSG(result_count == 1, "Wrong number of results of updater.no_tasks");
bool no_tasks;
ASSERT_MSG(interpreter_collect_results(interpreter, "b", &no_tasks) == -1, "The result of updater.no_tasks is not bool");
if (no_tasks) {
approval_clean(approval_file); // There is nothing to do and if we have approvals enabled we should drop approval file
GOTO_CLEANUP;
}
......@@ -275,6 +276,8 @@ int main(int argc, char *argv[]) {
// Approvals are only for non-interactive mode (implied by batch mode).
// Otherwise user approves on terminal in previous code block.
GOTO_CLEANUP;
err = interpreter_call(interpreter, "updater.tasks_to_transaction", NULL, "");
ASSERT_MSG(!err, "%s", err);
if (!replan) {
update_state(LS_PREUPD);
const char *hook_path = aprintf("%s%s", root_dir, hook_preupdate);
......@@ -286,11 +289,11 @@ int main(int argc, char *argv[]) {
if (log) {
const char *timebuf = time_load();
fprintf(log, "%sTRANSACTION START\n", timebuf);
err = interpreter_call(interpreter, "transaction.task_report", &result_count, "s", timebuf);
err = interpreter_call(interpreter, "updater.task_report", &result_count, "s", timebuf);
ASSERT_MSG(!err, "%s", err);
const char *content;
ASSERT_MSG(result_count == 1, "Wrong number of results of transaction.task_report (%zu)", result_count);
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &content) == -1, "The result of transaction.task_report is not string");
ASSERT_MSG(result_count == 1, "Wrong number of results of updater.task_report (%zu)", result_count);
ASSERT_MSG(interpreter_collect_results(interpreter, "s", &content) == -1, "The result of updater.task_report is not string");
fputs(content, log);
fclose(log);
} else
......
......@@ -27,7 +27,8 @@ LUA_TESTS := \
syscnf \
cleanup \
uri \
picosat
picosat \
updater
ifdef COV
LUA_TESTS += coverage
......
......@@ -9,6 +9,7 @@ TRANS_SYS_TESTS := \
UPD_SYS_TESTS := \
help \
plan \
plan-unapproved \
steal-confs \
simple-update \
multiple-repos \
......
......@@ -7,7 +7,6 @@ Updater is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Updater is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
......@@ -555,114 +554,6 @@ function test_recover_late()
assert_table_equal(expected, mocks_called)
end
function test_empty()
assert(transaction.empty())
transaction.queue_remove("pkg")
assert_false(transaction.empty())
end
function test_approval_hash()
-- When nothing is present, the hash is equal to one of an empty string
assert_equal(sha256(''), transaction.approval_hash())
-- Override the transaction.perform with empty function, so we can clean the queue when we like
mocks_install('transaction.perform', function () return {} end)
local function ops_hash(ops)
for _, op in ipairs(ops) do
transaction["queue_" .. op[1]](unpack(op, 2))
end
local hash = transaction.approval_hash()
-- get rid of the queue
transaction.perform_queue()
return hash
end
local function equal(ops1, ops2)
return ops_hash(ops1) == ops_hash(ops2)
end
-- The same lists of operations return the same hash
assert_true(equal(
{
{'install_downloaded', '', 'pkg', 13, {}},
{'remove', 'pkg2'}
},
{
{'install_downloaded', '', 'pkg', 13, {}},
{'remove', 'pkg2'}
}))
-- The order doesn't matter (since we are not sure if the planner is deterministic in that regard)
assert_true(equal(
{
{'install_downloaded', '', 'pkg', 13, {}},
{'remove', 'pkg2'}
},
{
{'remove', 'pkg2'},
{'install_downloaded', '', 'pkg', 13, {}}
}))
-- Package version changes the hash
assert_false(equal(
{
{'install_downloaded', '', 'pkg', 13, {}},
{'remove', 'pkg2'}
},
{
{'install_downloaded', '', 'pkg', 14, {}},
{'remove', 'pkg2'}
}))
-- Package name changes the hash
assert_false(equal(
{
{'install_downloaded', '', 'pkg', 13, {}},
{'remove', 'pkg2'}
},
{
{'install_downloaded', '', 'pkg3', 13, {}},
{'remove', 'pkg2'}
}))
-- Package the operation changes the hash
assert_false(equal(
{
{'install_downloaded', '', 'pkg', 13, {}},
{'remove', 'pkg2'}
},
{
{'remove', 'pkg'},
{'remove', 'pkg2'}
}))
-- Omitting one of the tasks changes the hash
assert_false(equal(
{
{'install_downloaded', '', 'pkg', 13, {}},
{'remove', 'pkg2'}
},
{
{'remove', 'pkg2'}
}))
end
function test_task_report()
assert_equal('', transaction.task_report())
assert_equal('', transaction.task_report('', true))
assert_equal('', transaction.task_report('prefix '))
transaction.queue_install_downloaded('', "pkg1", 13, {reboot = "finished"})
transaction.queue_remove("pkg2")
assert_equal([[
install 13 pkg1
remove - pkg2
]], transaction.task_report())
assert_equal([[
install 13 pkg1 finished
remove - pkg2 -
]], transaction.task_report('', true))
assert_equal([[
prefix install 13 pkg1
prefix remove - pkg2
]], transaction.task_report('prefix '))
assert_equal([[
prefix install 13 pkg1 finished
prefix remove - pkg2 -
]], transaction.task_report('prefix ', true))
end
function teardown()
-- A trick to clean up the queue
mock_gen('transaction.perform', function () return {} end)
......
--[[
Copyright 2019, CZ.NIC z.s.p.o. (http://www.nic.cz/)
This file is part of the turris updater.
Updater is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Updater is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Updater. If not, see <http://www.gnu.org/licenses/>.
]]--
require 'lunit'
local updater = require "updater"
local table = table
syscnf.set_root_dir()
module("updater-tests", package.seeall, lunit.testcase)
function test_task_report()
assert_equal('', updater.task_report())
assert_equal('', updater.task_report('', true))
assert_equal('', updater.task_report('prefix '))
table.insert(updater.tasks, { action = "require", package = {Version="13"}, name="pkg1", modifier = {reboot="finished"} })
table.insert(updater.tasks, { action = "remove", package = {Version="1"}, name="pkg2" })
assert_equal([[
install 13 pkg1
remove - pkg2
]], updater.task_report())
assert_equal([[
install 13 pkg1 finished
remove - pkg2 -
]], updater.task_report('', true))
assert_equal([[
prefix install 13 pkg1
prefix remove - pkg2
]], updater.task_report('prefix '))
assert_equal([[
prefix install 13 pkg1 finished
prefix remove - pkg2 -
]], updater.task_report('prefix ', true))
end
function test_approval_hash()
-- When nothing is present, the hash is equal to one of an empty string
updater.tasks = {}
assert_equal(sha256(''), updater.approval_hash())
local function ops_hash(ops)
updater.tasks = {}
for _, op in ipairs(ops) do
table.insert(updater.tasks, {
action = op[1],
name = op[2],
package = op[3],
modifier = op[4]
})
end
return updater.approval_hash()
end
local function equal(ops1, ops2)
return ops_hash(ops1) == ops_hash(ops2)
end
-- The same lists of operations return the same hash
assert_true(equal(
{
{'require', 'pkg', {Version=13}, {}},
{'remove', 'pkg2'}
},
{
{'require', 'pkg', {Version=13}, {}},
{'remove', 'pkg2'}
}))
-- The order doesn't matter (since we are not sure if the planner is deterministic in that regard)
assert_true(equal(
{
{'require', 'pkg', {Version=13}, {}},
{'remove', 'pkg2'}
},
{
{'remove', 'pkg2'},
{'require', 'pkg', {Version=13}, {}},
}))
-- Package version changes the hash
assert_false(equal(
{
{'require', 'pkg', {Version=13}, {}},
{'remove', 'pkg2'}
},
{
{'require', 'pkg', {Version=14}, {}},
{'remove', 'pkg2'}
}))
-- Package name changes the hash
assert_false(equal(
{
{'require', 'pkg', {Version=13}, {}},
{'remove', 'pkg2'}
},
{
{'require', 'pkg3', {Version=13}, {}},
{'remove', 'pkg2'}
}))
-- Package the operation changes the hash
assert_false(equal(
{
{'require', 'pkg', {Version=13}, {}},
{'remove', 'pkg2'}
},
{
{'remove', 'pkg'},
{'remove', 'pkg2'}
}))
-- Omitting one of the tasks changes the hash
assert_false(equal(
{
{'require', 'pkg', {Version=13}, {}},
{'remove', 'pkg2'}
},
{
{'remove', 'pkg2'}
}))
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment