Verified Commit 982995a1 authored by Karel Koci's avatar Karel Koci 🤘

Check for collisions between directories and files

parent 410e7055
......@@ -560,6 +560,9 @@ with removed packages.
Note that when upgrading, the old packages needs to be considered removed
(and listed in the remove_pkgs set).
Note that we are only working with files and directories they are in. Directories
containing no files are not checked.
The current_status is what is returned from status_parse(). The remove_pkgs
is set of package names (without versions) to remove. It's not a problem if
the package is not installed. The add_pkgs is a table, keys are names of packages
......@@ -569,31 +572,144 @@ It returns a table, values are name of files where are new collisions, values
are tables where the keys are names of packages and values are either `existing`
or `new`.
The second result is a set of all the files that shall disappear after
The second result is table of file-directory/directory-file collisions, those can be
resolvable by early deletions. Keys are names of packages and values are sets of
all files to be deleted.
The third result is a set of all the files that shall disappear after
performing these operations.
]]
function collision_check(current_status, remove_pkgs, add_pkgs)
-- List of all files in the OS
local files_all = {}
-- Files that might disappear (but we need to check if another package claims them as well)
local remove_candidates = {}
-- Mark the given file as belonging to the package. Return if there's a collision.
--[[
This is tree constructed with tables. There can be two kinds of nodes,
directories and others. Directories contains field "nodes" containing
other nodes. Other non-directory nodes has package they belong to under "pkg"
key, one of string "to-remove", "existing" or "new" under "when" key. And
both have full path under "path" key.
--]]
local files_tree = {}
-- First returned result. Table with collisions. Key is collision path and value is table with packages names as keys and "when" as values.
local collisions = {}
-- Second returned result. We fill this with nodes we want to remove before given package is merged to file system
local early_remove = {}
-- Iterates trough all non-directory nodes from given node.
local function files_tree_iterate(root_node)
local function iterate_internal(nodes)
if #nodes == 0 then
return nil
end
local n = nodes[#nodes]
nodes[#nodes] = nil
if n.nodes then
local indx = 0
utils.arr_append(nodes, utils.map(n.nodes, function (_, val)
indx = indx + 1
return indx, val
end
))
return iterate_internal(nodes)
end
return nodes, n
end
return iterate_internal, { root_node }
end
-- Adds file to files tree and detect collisions
local function file_insert(fname, pkg_name, when)
local collision = true
-- The file hasn't existed yet, so there's no collision
if not files_all[fname] then
files_all[fname] = {}
collision = false
-- Returns node for given path. If node contains "pkg" field then it is not directory. If it contains "nodes" field, then it is directory. If it has neither then it was newly created.
local function files_tree_node(path)
local node = files_tree
local ppath = ""
for n in path:gmatch("[^/]+") do
ppath = ppath .. "/" .. n
if node.pkg then -- Node is file. We can't continue.
return false, node
else -- Node is not file or unknown
if not node.nodes then node.nodes = {} end
if not node.nodes[n] then node.nodes[n] = {} end
node = node.nodes[n]
node.path = ppath
end
end
return true, node
end
local function set_node(node)
node.pkg = pkg_name
node.when = when
return node
end
local function add_collision(path, coll)
if collisions[path] then
utils.table_merge(collisions[path], coll)
else
collisions[path] = coll
end
end
local function set_early_remove(node)
if not early_remove[pkg_name] then
early_remove[pkg_name] = {}
end
for _, n in files_tree_iterate(node) do
early_remove[pkg_name][n.path] = true
n.pkg = nil -- Drop package name. This effectively makes it to not appear in "remove" list
end
node.nodes = nil -- Drop whole tree. It should be freed by GC except some nodes that might be in remove_candidates list.
end
local ok, node = files_tree_node(fname)
if not ok then -- We collided to file
-- We are trying to replace file with directory
if node.when == "to-remove" then
set_early_remove(node)
return file_insert(fname, pkg_name, when)
else
add_collision(node.path, {
[pkg_name] = when,
[node.pkg] = node.when
})
return nil
end
else -- Required node returned
if node.nodes then
-- Trying replace directory with file.
local coll = {}
for _, snode in files_tree_iterate(node) do
if snode.when ~= "to-remove" then
coll[snode.pkg] = snode.when
end
end
if next(coll) then
coll[pkg_name] = when
add_collision(node.path, coll)
return nil
else
-- We can remove this directory
set_early_remove(node)
return set_node(node)
end
else
if node.pkg and node.pkg ~= pkg_name and node.when ~= "to-remove" then
-- File with file collision
add_collision(node.path, {
[pkg_name] = when,
[node.pkg] = node.when
})
return nil
else
-- This is new non-directory node or node of same package or previous node was marked as to-remove
return set_node(node)
end
end
end
files_all[fname][pkg_name] = when
return collision
end
-- Build the structure for the current state
-- Non-directory nodes that might disappear (but we need to check if another package claims them as well)
local remove_candidates = {}
-- Build tree of current state.
for name, status in pairs(current_status) do
if remove_pkgs[name] then
-- If we remove the package, all its files might disappear
for f in pairs(status.files or {}) do
remove_candidates[f] = true
remove_candidates[f] = file_insert(f, name, "to-remove")
end
else
-- Otherwise, the file is in the OS
......@@ -602,24 +718,23 @@ function collision_check(current_status, remove_pkgs, add_pkgs)
end
end
end
local collisions = {}
-- No go through the new packages and check if there are any new collisions
-- No collisions should happen until this point. If it does, we ignore it (it shouldn't be caused by us)
collisions = {}
early_remove = {}
-- Now go through the new packages
for name, files in pairs(add_pkgs) do
for f in pairs(files) do
if file_insert(f, name, 'new') then
-- In the end, there'll be the newest version of the table with all the collisions
collisions[f] = files_all[f]
end
file_insert(f, name, "new")
end
end
-- Files that shall really disappear
local remove = {}
for f in pairs(remove_candidates) do
if not files_all[f] then
for f, node in pairs(remove_candidates) do
if node.pkg and node.when == "to-remove" then
remove[f] = true
end
end
return collisions, remove
return collisions, early_remove, remove
end
--[[
......
......@@ -122,7 +122,7 @@ local function pkg_unpack(operations, status)
end
local function pkg_collision_check(status, to_remove, to_install)
local collisions, removes = backend.collision_check(status, to_remove, to_install)
local collisions, early_remove, removes = backend.collision_check(status, to_remove, to_install)
if next(collisions) then
--[[
Collisions:
......@@ -135,20 +135,27 @@ local function pkg_collision_check(status, to_remove, to_install)
end)), ", "), true
end)), "\n"))
end
return removes
return removes, early_remove
end
local function pkg_move(status, plan, errors_collected)
local function pkg_move(status, plan, early_remove, errors_collected)
-- Prepare table of not installed confs for config stealing
local not_installed_confs = backend.not_installed_confs(status)
local all_configs = {}
-- Build list of all configs and steal from not-installed
for _, op in ipairs(plan) do
if op.op == "install" then
local steal = backend.steal_configs(status, not_installed_confs, op.configs)
utils.table_merge(op.old_configs, steal)
utils.table_merge(all_configs, op.old_configs)
end
end
-- Go through the list once more and perform the prepared operations
for _, op in ipairs(plan) do
if op.op == "install" then
state_dump("install")
log_event("I", op.control.Package .. " " .. op.control.Version)
utils.table_merge(all_configs, op.old_configs)
-- Unfortunately, we need to merge the control files first, otherwise the maintainer scripts won't run. They expect to live in the info dir when they are run. And we need to run the preinst script before merging the files.
backend.pkg_merge_control(op.dir .. "/control", op.control.Package, op.control.files)
if utils.multi_index(status, op.control.Package, "Status", 3) == "installed" then
......@@ -157,8 +164,9 @@ local function pkg_move(status, plan, errors_collected)
else
script(errors_collected, op.control.Package, "preinst", "install", op.control.Version)
end
local steal = backend.steal_configs(status, not_installed_confs, op.configs)
utils.table_merge(op.old_configs, steal)
if early_remove[op.control.Package] then
backend.pkg_cleanup_files(early_remove[op.control.Package], all_configs)
end
backend.pkg_merge_files(op.dir .. "/data", op.dirs, op.files, op.old_configs)
status[op.control.Package] = op.control
end
......@@ -215,7 +223,7 @@ local function perform_internal(operations, journal_status, run_state)
- journal_type: One of the constants from journal module. This is the type
of record written into the journal.
- fun: The function performing the actual step.
- sync: If true, the file system is synced before marking the journal.
- flush: If true, the file system is synced before marking the journal.
- ...: Parameters for the function.
All the results from the step are stored in the journal and also returned.
......@@ -252,9 +260,9 @@ local function perform_internal(operations, journal_status, run_state)
-- Drop the operations. This way, if we are tail-called, then the package buffers may be garbage-collected
operations = nil
-- Check for collisions
local removes = step(journal.CHECKED, pkg_collision_check, false, status, to_remove, to_install)
local removes, early_remove = step(journal.CHECKED, pkg_collision_check, false, status, to_remove, to_install)
local all_configs
status, errors_collected, all_configs = step(journal.MOVED, pkg_move, true, status, plan, errors_collected)
status, errors_collected, all_configs = step(journal.MOVED, pkg_move, true, status, plan, early_remove, errors_collected)
status, errors_collected = step(journal.SCRIPTS, pkg_scripts, true, status, plan, removes, to_install, errors_collected, all_configs)
end)
-- Make sure the temporary dirs are removed even if it fails. This will probably be slightly different with working journal.
......
......@@ -421,20 +421,22 @@ end
function test_collisions()
local status = B.status_parse()
-- Just remove a package - no collisions, but files should disappear
local col, rem = B.collision_check(status, {['kmod-usb-storage'] = true}, {})
local col, erem, rem = B.collision_check(status, {['kmod-usb-storage'] = true}, {})
assert_table_equal({}, col)
assert_table_equal({}, erem)
assert_table_equal({
["/lib/modules/3.18.21-70ea6b9a4b789c558ac9d579b5c1022f-10/usb-storage.ko"] = true,
["/etc/modules-boot.d/usb-storage"] = true,
["/etc/modules.d/usb-storage"] = true
}, rem)
-- Add a new package, but without any collisions
local col, rem = B.collision_check(status, {}, {
local col, erem, rem = B.collision_check(status, {}, {
['package'] = {
['/a/file'] = true
}
})
assert_table_equal({}, col)
assert_table_equal({}, erem)
assert_table_equal({}, rem)
local test_pkg = {
['package'] = {
......@@ -442,25 +444,27 @@ function test_collisions()
}
}
-- Add a new package, collision isn't reported, because the original package owning it gets removed
local col, rem = B.collision_check(status, {['kmod-usb-storage'] = true}, test_pkg)
local col, erem, rem = B.collision_check(status, {['kmod-usb-storage'] = true}, test_pkg)
assert_table_equal({}, col)
assert_table_equal({}, erem)
assert_table_equal({
["/lib/modules/3.18.21-70ea6b9a4b789c558ac9d579b5c1022f-10/usb-storage.ko"] = true,
["/etc/modules-boot.d/usb-storage"] = true
-- The usb-storage file is taken over, it doesn't disappear
}, rem)
-- A collision
local col, rem = B.collision_check(status, {}, test_pkg)
local col, erem, rem = B.collision_check(status, {}, test_pkg)
assert_table_equal({
["/etc/modules.d/usb-storage"] = {
["kmod-usb-storage"] = "existing",
["package"] = "new"
}
}, col)
assert_table_equal({}, erem)
assert_table_equal({}, rem)
-- A collision between two new packages
test_pkg['another'] = test_pkg['package']
local col, rem = B.collision_check(status, {['kmod-usb-storage'] = true}, test_pkg)
local col, erem, rem = B.collision_check(status, {['kmod-usb-storage'] = true}, test_pkg)
assert_not_equal({
["/etc/modules.d/usb-storage"] = true
}, utils.map(col, function (k) return k, true end))
......@@ -468,11 +472,92 @@ function test_collisions()
["package"] = "new",
["another"] = "new"
}, col["/etc/modules.d/usb-storage"])
assert_table_equal({}, erem)
assert_table_equal({
["/lib/modules/3.18.21-70ea6b9a4b789c558ac9d579b5c1022f-10/usb-storage.ko"] = true,
["/etc/modules-boot.d/usb-storage"] = true
-- The usb-storage file is taken over, it doesn't disappear
}, rem)
-- Collision resolved with early file remove in favor of new directory
local test_pkg = {
["package"] = {
["/etc/modules.d/usb-storage/other-file"] = true,
["/etc/modules.d/usb-storage/new-file"] = true,
["/etc/test-package"] = true
}
}
local col, erem, rem = B.collision_check(status, {['kmod-usb-storage'] = true}, test_pkg)
assert_table_equal({}, col)
assert_table_equal({
["package"] = {
["/etc/modules.d/usb-storage"] = true,
}
}, erem)
assert_table_equal({
["/lib/modules/3.18.21-70ea6b9a4b789c558ac9d579b5c1022f-10/usb-storage.ko"] = true,
["/etc/modules-boot.d/usb-storage"] = true,
}, rem)
-- Collision resolved with early directory remove in favor of new file
local test_pkg = {
["package"] = {
["/usr/share/terminfo"] = true,
["/etc/modules.d/usb-storage"] = true
}
}
local col, erem, rem = B.collision_check(status, {['terminfo'] = true}, test_pkg)
assert_table_equal({
["/etc/modules.d/usb-storage"] = {
["package"] = "new",
["kmod-usb-storage"] = "existing"
}
}, col)
assert_table_equal({
["package"] = {
["/usr/share/terminfo/x/xterm"] = true,
["/usr/share/terminfo/r/rxvt-unicode"] = true,
["/usr/share/terminfo/d/dumb"] = true,
["/usr/share/terminfo/a/ansi"] = true,
["/usr/share/terminfo/x/xterm-color"] = true,
["/usr/share/terminfo/r/rxvt"] = true,
["/usr/share/terminfo/s/screen"] = true,
["/usr/share/terminfo/x/xterm-256color"] = true,
["/usr/share/terminfo/l/linux"] = true,
["/usr/share/terminfo/v/vt100"] = true,
["/usr/share/terminfo/v/vt102"] = true
} }, erem)
assert_table_equal({}, rem)
-- Collision that could be resolved by removing directory but new package requires it.
test_pkg["package"]["/etc/modules.d/usb-storage"] = nil
test_pkg["another"] = {
["/usr/share/terminfo/test"] = true,
}
local col, erem, rem = B.collision_check(status, {['terminfo'] = true}, test_pkg)
assert_table_equal({
["/usr/share/terminfo"] = {
["another"] = "new",
["package"] = "new"
}
}, col)
-- Note that we don't care about erem and rem. Their content depends on order packages are processed.
-- Collision that could be resolved by removing file, but existing and new package requires it.
local test_pkg = {
["package"] = {
["/etc/modules.d/usb-storage/other-file"] = true,
["/etc/modules.d/usb-storage/new-file"] = true,
},
["another"] = {
["/etc/modules.d/usb-storage"] = true,
}
}
local col, erem, rem = B.collision_check(status, {}, test_pkg)
assert_table_equal({
["/etc/modules.d/usb-storage"] = {
["package"] = "new",
["another"] = "new",
["kmod-usb-storage"] = "existing"
}
}, col)
-- For "erem" and "rem" see note few lines before this one.
end
-- Test config_steal and not_installed_confs function
......
......@@ -132,7 +132,7 @@ local function mocks_install()
end)
mock_gen("backend.pkg_unpack", function () return "pkg_dir" end)
mock_gen("backend.pkg_examine", function () return {f = true}, {d = true}, {c = "1234567890123456"}, {Package = "pkg-name", files = {f = true}, Conffiles = {c = "1234567890123456"}, Version = "1", Status = {"install", "user", "installed"}} end)
mock_gen("backend.collision_check", function () return {}, {} end)
mock_gen("backend.collision_check", function () return {}, {}, {} end)
mock_gen("backend.pkg_merge_files")
mock_gen("backend.pkg_cleanup_files")
mock_gen("backend.control_cleanup")
......@@ -171,7 +171,7 @@ function test_perform_empty()
},
{
f = "journal.write",
p = {journal.CHECKED, {}}
p = {journal.CHECKED, {}, {}}
},
{
f = "journal.write",
......@@ -192,7 +192,7 @@ end
-- Test a transaction when it goes well
function test_perform_ok()
mocks_install()
mock_gen("backend.collision_check", function () return {}, {d2 = true} end)
mock_gen("backend.collision_check", function () return {}, {}, {d2 = true} end)
local result = T.perform({
{
op = "install",
......@@ -266,7 +266,7 @@ function test_perform_ok()
},
{
f = "journal.write",
p = {journal.CHECKED, {["d2"] = true}}
p = {journal.CHECKED, {["d2"] = true}, { }}
},
{
f = "backend.pkg_merge_control",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment