Verified Commit 2e276e1e authored by Karel Koci's avatar Karel Koci 🤘

Drop flags

Idea behind flags was that scripts can save some notes for future
execution. We are not using this anywhere and probably we wont ever use
it. Also this somewhat goes against idea of stateless configuration.
Overall we clean clean a lot of code if we drop this unused feature.
parent de2f7b8a
......@@ -229,7 +229,7 @@ They start with a capital letter, since they act as constructors.
Script
~~~~~~
script = Script("script-name", "uri", { extra })
script = Script("uri", { extra })
This command runs another script. The name is the local part of the
script name.
......@@ -266,6 +266,12 @@ ca::
crl::
Options to verify the script integrity.
Note that for backward compatibility we also support following format:
script = Script("script-name", "uri", { extra })
Although we are no longer validating script name and is no longer used anywhere.
Repository
~~~~~~~~~~
......@@ -539,16 +545,6 @@ relation as second argument (for example:
`version_match(installed['pkg'].version, '>=1.0.0')`)
It returns true if version matches given version relation, otherwise false.
StoreFlags
~~~~~~~~~~
StoreFlags("flagname", "flagname")
A script may request to store some of its flags right away, no matter
how the updater terminates. It first stores its flags and then calls
the `StoreFlags` command with the names of the flags to store. The
rest of the flags are stored as usual.
Run
~~~
......@@ -688,28 +684,6 @@ The top-level table is instantiated (not generated through
meta-tables), therefore it is possible to get the list of installed
packages.
flags
~~~~~
A table with flags defined by the scripts. Flags are preserved between
the updater runs, so a script may leave notes for its future self.
The table is indexed by the name of the script, either relative or
absolute. Referencing self as an empty string works. The table is
generated on the fly, therefore the script may not find out what other
scripts ever run.
In case of the own flags, the table is fully instantiated, so they can
be written and iterated (using `next` and `pairs`). If it is flags
belonging to other script, the table is read only and doesn't allow
iterating (eg. it is possible to query a flag value, but not possible
to discover which flags exist).
Flags may be only strings.
Any modification to the own flags table is stored on successful
updater termination or when the script explicitly request it.
Export variables to Script
--------------------------
......
......@@ -27,9 +27,6 @@ local require = require
local next = next
local tostring = tostring
local tonumber = tonumber
local loadfile = loadfile
local setmetatable = setmetatable
local setfenv = setfenv
local assert = assert
local unpack = unpack
local io = io
......@@ -49,19 +46,17 @@ local move = move
local ls = ls
local md5 = md5
local sha256 = sha256
local sync = sync
local DBG = DBG
local WARN = WARN
local DataDumper = DataDumper
local utils = require "utils"
local locks = require "locks"
module "backend"
-- Functions and variables used in other files
-- luacheck: globals pkg_temp_dir repo_parse status_dump pkg_unpack pkg_examine collision_check installed_confs steal_configs dir_ensure pkg_merge_files pkg_merge_control pkg_config_info pkg_cleanup_files control_cleanup version_cmp version_match flags_load flags_get script_run flags_get_ro flags_write flags_mark run_state
-- luacheck: globals pkg_temp_dir repo_parse status_dump pkg_unpack pkg_examine collision_check installed_confs steal_configs dir_ensure pkg_merge_files pkg_merge_control pkg_config_info pkg_cleanup_files control_cleanup version_cmp version_match script_run run_state
-- Variables that we want to access from outside (ex. for testing purposes)
-- luacheck: globals status_file info_dir root_dir pkg_temp_dir flags_storage cmd_timeout cmd_kill_timeout stored_flags dir_opkg_collided
-- luacheck: globals status_file info_dir root_dir pkg_temp_dir cmd_timeout cmd_kill_timeout dir_opkg_collided
-- Functions that we want to access from outside (ex. for testing purposes)
-- luacheck: globals root_dir_set block_parse block_split block_dump_ordered pkg_status_dump package_postprocess status_parse get_parent config_modified
......@@ -80,9 +75,6 @@ root_dir = "/"
-- A directory where unpacked packages live
local pkg_temp_dir_suffix = "/usr/share/updater/unpacked"
pkg_temp_dir = pkg_temp_dir_suffix
-- A file with the flags from various scripts
local flags_storage_suffix = "/usr/share/updater/flags"
flags_storage = flags_storage_suffix
-- Directory where we move files and directories that weren't part of any package.
local dir_opkg_collided_suffix = "/var/opkg-collided"
dir_opkg_collided = dir_opkg_collided_suffix
......@@ -101,7 +93,6 @@ function root_dir_set(dir)
status_file = dir .. status_file_suffix
info_dir = dir .. info_dir_suffix
pkg_temp_dir = dir .. pkg_temp_dir_suffix
flags_storage = dir .. flags_storage_suffix
dir_opkg_collided = dir .. dir_opkg_collided_suffix
end
......@@ -1161,148 +1152,6 @@ function version_match(v, r)
end
end
stored_flags = {}
local function flags_ro_proxy(flags)
return setmetatable({}, {
__index = function (_, name)
local result = flags[name]
if result and type(result) == 'string' then
return result
elseif result then
WARN("Type of flag " .. name .. " is " .. type(result) .. ", foreign access prevented")
end
return nil
end,
__newindex = function ()
error(utils.exception("access violation", "Writing of foreign flags not allowed"))
end
})
end
-- Load flags from the file and warn if it isn't possible for any reason.
function flags_load()
local chunk, err = loadfile(flags_storage)
if not chunk then
WARN("Couldn't load flags: " .. err)
return
end
-- Disallow it to call any functions whatsoever
local chunk_sanitized = setfenv(chunk, {})
local ok, loaded = pcall(chunk_sanitized)
if not ok then
WARN("Flag storage corrupt, not loading flags: " .. err)
return
end
-- Store additional info to each script's flags
stored_flags = utils.map(loaded, function (path, values)
return path, {
values = values,
proxy = flags_ro_proxy(values)
}
end)
end
--[[
Get flags (read-write) for a single context, identified by its path. This is expected
to be done after flags_load(). If the flags for this path aren't loaded (eg. previously
unknown script), a new empty table is provided.
It must not be called multiple times with the same path.
]]
function flags_get(path)
if not path then
-- This is during testing, we don't have any path, so no flags to consider
return nil
end
-- Create the flags for the script if it doesn't exist yet
local flags = stored_flags[path]
if not flags then
local f = {}
flags = {
provided = f,
proxy = flags_ro_proxy(f)
}
stored_flags[path] = flags
return f
end
--[[
Return the flags table (the one the proxy points to, so the proxies see
changes). But keep a copy of the original, in case we want to store partial
changes.
]]
assert(not flags.provided)
local result = flags.values
flags.values = utils.clone(result)
flags.provided = result
return result
end
--[[
Get a read-only proxy to access flags of a script on the given path.
In case the path isn't initiated (the script hasn't run and the flags weren't loaded),
nil is returned.
]]
function flags_get_ro(path)
return utils.multi_index(stored_flags, path, "proxy")
end
function flags_write(full)
if full then
for path, data in pairs(stored_flags) do
if data.provided then
-- Make a fresh copy of the flag data, with all the new changes
data.values = utils.clone(data.provided)
-- Wipe out anything that is not string, as these are disallowed for security reasons
for name, flag in pairs(data.values) do
if type(flag) ~= 'string' then
WARN("Omitting flag " .. name .. " of " .. path .. ", as it is " .. type(flag))
data.values[name] = nil
end
end
else
-- Not used during this run, so drop it
stored_flags[path] = nil
end
end
end
local to_store = utils.map(stored_flags, function (name, data)
return name, data.values
end)
local f, err = io.open(flags_storage .. ".tmp", "w")
if not f then
WARN("Couldn't write the flag storage: " .. err)
return
end
f:write(DataDumper(to_store))
f:close()
sync()
local ok, err = os.rename(flags_storage .. ".tmp", flags_storage)
if not ok then
WARN("Couldn't put flag storage in place: " .. err)
return
end
end
--[[
Mark given flags in the script on the given path for storage.
Eg, push the changes to be written.
The names of the flags are passed by that ellipsis.
]]
function flags_mark(path, ...)
local group = stored_flags[path]
-- This should be called only from within a context and every context should have its own flags
assert(group)
if not group.values then
group.values = {}
end
for _, name in ipairs({...}) do
group.values[name] = group.provided[name]
end
end
local run_state_cache = {}
function run_state_cache:init()
......
......@@ -33,13 +33,12 @@ local assert = assert
local table = table
local utils = require "utils"
local uri = require "uri"
local backend = require "backend"
local DBG = DBG
local WARN = WARN
module "requests"
-- luacheck: globals known_packages package_wrap known_repositories known_repositories_all repo_serial repository repository_get content_requests install uninstall script store_flags known_content_packages
-- luacheck: globals known_packages package_wrap known_repositories known_repositories_all repo_serial repository repository_get content_requests install uninstall script known_content_packages
-- Verifications fields are same for script, repository and package. Lets define them here once and then just append.
local allowed_extras_verification = {
......@@ -441,29 +440,33 @@ local script_insert_options = {
ocsp = true
}
-- Remember here all executed scripts (by name)
local script_executed = {}
--[[
Note that we have filler field just for backward compatibility so when we have
just one argument or two arguments where second one is table we move all arguments
to their appropriate variables.
function script(result, context, name, script_uri, extra)
if script_executed[context.full_name .. '/' .. name] then
error(utils.exception("inconsistent", "Script with name " .. name .. " was already executed."))
Originally filler contained name of script.
]]
function script(result, context, filler, script_uri, extra)
if (not extra and not script_uri) or type(script_uri) == "table" then
extra = script_uri
script_uri = filler
end
script_executed[context.full_name .. '/' .. name] = true
DBG("Running script " .. name)
DBG("Running script " .. script_uri)
extra = allowed_extras_check_type(allowed_script_extras, 'script', extra or {})
extra_check_verification("script", extra)
for name, value in pairs(extra) do
if name == "ignore" then
extra_check_table("script", name, value, {"missing", "integrity"})
extra_check_table("script", script_uri, value, {"missing", "integrity"})
end
end
local u = uri(context, script_uri, extra)
local ok, content = u:get()
if not ok then
if utils.arr2set(extra.ignore or {})["missing"] then
WARN("Script " .. name .. " not found, but ignoring its absence as requested")
WARN("Script " .. script_uri .. " not found, but ignoring its absence as requested")
result.tp = "script"
result.name = name
result.name = script_uri
result.ignored = true
return
end
......@@ -509,7 +512,7 @@ function script(result, context, name, script_uri, extra)
end
end
merge.restrict = restrict
local err = sandbox.run_sandboxed(content, name, extra.security, context, merge)
local err = sandbox.run_sandboxed(content, script_uri, extra.security, context, merge)
if err and err.tp == 'error' then
if not err.origin then
err.oririn = script_uri
......@@ -518,14 +521,7 @@ function script(result, context, name, script_uri, extra)
end
-- Return a dummy handle, just as a formality
result.tp = "script"
result.name = name
result.uri = script_uri
end
function store_flags(_, context, ...)
DBG("Storing flags ", ...)
backend.flags_mark(context.full_name, ...)
backend.flags_write(false)
end
return _M
......@@ -298,10 +298,6 @@ local funcs = {
mode = "morpher",
value = requests.script
},
StoreFlags = {
mode = "morpher",
value = requests.store_flags
},
Unexport = {
mode = "morpher",
value = function(_, context, variable)
......@@ -371,9 +367,6 @@ for _, name in pairs({'model', 'board_name', 'turris_version', 'serial', 'archit
value = name
}
end
funcs.Restricted["flags"] = {
mode = "flags"
}
for name, val in pairs(G) do
funcs.Full[name] = {
mode = "inject",
......@@ -419,35 +412,6 @@ function level(l)
end
end
local function flags_new(context)
return setmetatable({}, {
__index = function (_, path)
-- Make sure the hierarchy is there ‒ it was created by run_sandboxed, not standalone
assert(context.root_parent)
local requested
local full_path
if path == "" then
-- It's us
requested = context
elseif path.match('^/') then
-- It's an absolute path
full_path = path
requested = context.root_parent.hierarchy[path]
else
-- It's relative path ‒ construct the absolute one first
full_path = context.full_name + '/' + path
requested = context.root_parent.hierarchy[full_path]
end
if requested == context then
-- It's us. Provide full access to the table (eg. let it be written there as well)
return context.flags
else
return backend.flags_get_ro(full_path)
end
end
})
end
--[[
Create a new context. The context inherits everything
from its parent (if the parent is not nil). The security
......@@ -457,7 +421,7 @@ inherited).
A new environment, corresponding to the security level,
is constructed and stored in the result as „env“.
]]
function new(sec_level, parent, name)
function new(sec_level, parent)
sec_level = level(sec_level)
local result = {}
--[[
......@@ -471,29 +435,6 @@ function new(sec_level, parent, name)
parent = parent or {}
sec_level = sec_level or parent.sec_level
result.sec_level = sec_level
if name then
--[[
Construct the name, full path and a hierarchy table with all the existing
contexts. The hierarchy table is in the top-level script.
However, there are also name-less sandboxes. These are abused by the
uri module and don't actually run any real code and they don't have flags.
So we completely skip this flag manipulation for them. The flags variable
will be broken in such sandboxes, but as nobody would access it, it
doesn't matter.
]]
result.name = name
if parent and parent.full_name then
result.full_name = parent.full_name .. "/" .. name
result.root_parent = parent.root_parent
else
result.full_name = name
result.root_parent = result
result.hierarchy = {}
end
result.root_parent.hierarchy[result.full_name or ""] = result
result.flags = backend.flags_get(result.full_name)
end
-- Propagate exported
result.exported = utils.shallow_copy(parent.exported or {})
-- Construct a new environment
......@@ -513,8 +454,6 @@ function new(sec_level, parent, name)
load_state_vars()
end
result.env[n] = utils.clone(state_vars[v.value])
elseif v.mode == "flags" then
result.env[n] = flags_new(result)
elseif v.mode == "wrap" then
result.env[n] = function(...)
return v.value(result, ...)
......@@ -570,7 +509,7 @@ function run_sandboxed(chunk, name, sec_level, parent, context_merge, context_mo
return utils.exception("compilation", err)
end
end
local context = new(sec_level, parent, name)
local context = new(sec_level, parent)
utils.table_merge(context, context_merge or {})
context_mod = context_mod or function () end
context_mod(context)
......
......@@ -66,7 +66,6 @@ end
function prepare(entrypoint)
local required = required_pkgs(entrypoint)
local run_state = backend.run_state()
backend.flags_load()
local tasks = planner.filter_required(run_state.status, required, allow_replan)
--[[
Start download of all the packages. They all start (or queue, if there are
......@@ -139,7 +138,7 @@ function pre_cleanup()
end
-- Note: This function don't have to return
function cleanup(success, reboot_finished)
function cleanup(reboot_finished)
if transaction.cleanup_actions.reexec and allow_replan then
if reboot_finished then
reexec('--reboot-finished')
......@@ -147,9 +146,6 @@ function cleanup(success, reboot_finished)
reexec()
end
end
if success then
backend.flags_write(true)
end
end
return _M
......@@ -293,7 +293,7 @@ int main(int argc, char *argv[]) {
const char *hook_path = aprintf("%s%s", root_dir, hook_reboot_delayed);
exec_dir(events, hook_path);
}
err = interpreter_call(interpreter, "updater.cleanup", NULL, "bb", trans_ok, reboot_finished);
err = interpreter_call(interpreter, "updater.cleanup", NULL, "bb", reboot_finished);
ASSERT_MSG(!err, "%s", err);
if (task_log) {
FILE *log = fopen(task_log, "a");
......
......@@ -246,7 +246,6 @@ end
local orig_status_file = B.status_file
local orig_info_dir = B.info_dir
local orig_root_dir = B.root_dir
local orig_flags_storage = B.flags_storage
local tmp_dirs = {}
--[[
......@@ -960,131 +959,11 @@ function test_version_match()
assert_true(B.version_match("1.2.3", "1.2.3")) -- without comparator do exact match (just as corner case)
assert_false(B.version_match("1.2.3", "1.3.3"))
end
local function check_stored_flags(full, expected)
local test_root = mkdtemp()
table.insert(tmp_dirs, test_root)
B.flags_storage = test_root .. "/flags"
B.flags_write(full)
assert_table_equal(expected, loadfile(B.flags_storage)())
end
function test_flags()
assert_table_equal({}, B.stored_flags)
B.flags_load()
-- The meta tables are not checked here by assert_table_equal
assert_table_equal({
["/path"] = {
values = {
a = "hello",
b = "hi"
},
proxy = {}
}
}, B.stored_flags)
local flags = B.flags_get("/path")
assert_table_equal({
a = "hello",
b = "hi"
}, flags)
assert_table_equal({
["/path"] = {
values = {
a = "hello",
b = "hi"
},
provided = {
a = "hello",
b = "hi"
},
proxy = {}
}
}, B.stored_flags)
flags.x = "Greetings"
assert_table_equal({
["/path"] = {
values = {
a = "hello",
b = "hi"
},
provided = {
a = "hello",
b = "hi",
x = "Greetings"
},
proxy = {}
}
}, B.stored_flags)
local ro = B.flags_get_ro("/path")
assert_equal("hello", ro.a)
assert_nil(ro.c)
assert_equal("Greetings", ro.x)
assert_nil(B.flags_get_ro("/another"))
assert_exception(function () ro.c = "xyz" end, "access violation")
assert_exception(function () ro.d = "xyz" end, "access violation")
local new = B.flags_get("/another")
new.x = "y"
assert_equal("y", B.flags_get_ro("/another").x)
check_stored_flags(true, {
["/path"] = {
a = "hello",
b = "hi",
x = "Greetings"
},
["/another"] = {
x = "y"
}
})
end
function test_flags_mark()
B.flags_load()
local old = B.flags_get("/path")
old.x = "123"
old.y = "2345"
old.a = "5678"
old.b = nil
local new = B.flags_get("/another")
new.a = "123"
B.flags_mark("/path", "x", "a")
check_stored_flags(false, {
["/path"] = {
a = "5678",
b = "hi",
x = "123"
-- y is /not/ present, because we haven't marked it.
}
-- Also, /new isn't here
})
B.flags_mark("/another", "a")
B.flags_mark("/path", "b")
check_stored_flags(false, {
["/path"] = {
a = "5678",
x = "123"
},
["/another"] = {
a = "123"
}
})
check_stored_flags(true, {
["/path"] = {
a = "5678",
x = "123",
y = "2345"
},
["/another"] = {
a = "123"
}
})
end
function setup()
local sdir = os.getenv("S") or "."
-- Use a shortened version of a real status file for tests
B.status_file = sdir .. "/tests/data/opkg/status"
B.info_dir = sdir .. "/tests/data/opkg/info/"
B.flags_storage = sdir .. "/tests/data/flags"
end
function teardown()
......@@ -1092,8 +971,6 @@ function teardown()
B.status_file = orig_status_file
B.info_dir = orig_info_dir
B.root_dir= orig_root_dir
B.flags_storage = orig_flags_storage
utils.cleanup_dirs(tmp_dirs)
tmp_dirs = {}
B.stored_flags = {}
end
......@@ -34,7 +34,6 @@ local sandbox_fun_i = 0
local function run_sandbox_fun(func_code, level)
local chunk = "result = " .. func_code
local env
backend.stored_flags = {}
local result = sandbox.run_sandboxed(chunk, "Function chunk" .. tostring(sandbox_fun_i), level or "Restricted", nil, nil, function (context)
env = context.env
end)
......@@ -146,7 +145,7 @@ function test_script()
mocks_reset()
-- The URI contains 'Install "pkg"'
local result = sandbox.run_sandboxed([[
Script "test-script" "data:base64,SW5zdGFsbCAicGtnIgo=" { security = 'Restricted' }
Script("data:base64,SW5zdGFsbCAicGtnIgo=", { security = 'Restricted' })
]], "test_script_chunk", "Restricted")
assert_equal("context", result.tp, result.msg)
assert_table_equal({
......@@ -161,19 +160,31 @@ function test_script()
}, requests.content_requests)
end
function test_script_duplicate()
-- Test legacy syntax of script
function test_script_legacy()
-- We actually don't want any mocks here, let uri work as expected
mocks_reset()
local err = sandbox.run_sandboxed([[
Script "test-script" "data:base64,SW5zdGFsbCAicGtnIgo=" { security = 'Restricted' }
Script "test-script" "data:base64,SW5zdGFsbCAicGtnIgo=" { security = 'Restricted' }
]], "test_script_duplicate_chunk", "Restricted")
assert_table_equal(utils.exception("inconsistent", "Script with name test-script was already executed."), err)
-- The URI contains 'Install "pkg"'
local result = sandbox.run_sandboxed([[
Script("name", "data:base64,SW5zdGFsbCAicGtnIgo=", { security = 'Restricted' })
]], "test_script_chunk", "Restricted")
assert_equal("context", result.tp, result.msg)
assert_table_equal({
{
tp = 'install',
package = {
tp = 'package',
name = 'pkg'
},
priority = 50
}
}, requests.content_requests)
end
function test_script_missing()
mocks_reset()
local result = sandbox.run_sandboxed([[
Script "test-script" "file:///does/not/exist" { ignore = {"missing"}, security = "local" }
Script("file:///does/not/exist", { ignore = {"missing"}, security = "local" })
]], "test_script_missing_chunk", "Local")
-- It doesn't produce an error, even when the script doesn't exist
assert_equal("context", result.tp, result.msg)
......@@ -183,7 +194,7 @@ end
function test_script_raise_level()
mocks_reset()
local err = sandbox.run_sandboxed([[
Script "test-script" "data:," { security = 'Full' }
Script("data:,", { security = 'Full' })
]], "test_script_raise_level_chunk", "Restricted")
assert_table_equal(utils.exception("access violation", "Attempt to raise security level from Restricted to Full"), err)
end
......@@ -195,7 +206,7 @@ function test_script_level_transition()
for i, from in ipairs(levels) do
for j, to in ipairs(levels) do
local result = sandbox.run_sandboxed([[
Script "test-script" "data:," { security = ']] .. to .. [[' }
Script("data:,", { security = ']] .. to .. [[' })
]], "test_script_level_transition_chunk " .. from .. "/" .. to, from)
if i > j then
assert_table_equal(utils.exception("access violation", "Attempt to raise security level from " .. from .. " to " .. to), result)
......@@ -211,11 +222,10 @@ function test_script_pass_validation()
local bad_i = 0
local function bad(opts, msg, exctype)
local err = sandbox.run_sandboxed([[
Script "test-script" "data:," { security = 'Restricted']] .. opts .. [[ }
Script("data:,", { security = 'Restricted']] .. opts .. [[ })
]], "test_script_pass_validation_chunk1" .. tostring(bad_i), "Restricted")
bad_i = bad_i + 1
assert_table_equal(utils.exception(exctype or "bad value", msg), err)
backend.stored_flags = {}
end
-- Bad uri inside something
bad(", verification = 'sig', pubkey = 'invalid://'", "Unknown URI schema invalid")
......@@ -225,7 +235,7 @@ function test_script_pass_validation()
bad(", pubkey = 'file:///dev/null'", "At least Local level required for file URI", "access violation")
-- But we allow it if there's a high enough level
local result = sandbox.run_sandboxed([[
Script "test-script" "data:," { security = 'Restricted', pubkey = 'file:///dev/null' }