...
 
Commits (12)
......@@ -9,7 +9,7 @@ RUN \
busybox ca-certificates curl git \
make pkg-config gcc \
check cppcheck lua-check valgrind \
libcurl4-openssl-dev libevent-dev libssl-dev \
libcurl4-openssl-dev libevent-dev libssl-dev libarchive-dev\
lua5.1 liblua5.1-0-dev \
asciidoc lcov markdown libcommon-sense-perl \
wget procps && \
......
......@@ -11,6 +11,7 @@ Binary dependencies:
* Lua 5.1
* libcurl
* libevent2
* libarchive
Runtime dependencies:
* usign (for signatures validation)
......
......@@ -21,6 +21,7 @@ libupdater_MODULES := \
arguments \
inject \
interpreter \
unpacker \
lautoload.embed \
embed_types \
events \
......@@ -39,7 +40,7 @@ endif
libupdater_MODULES_3RDPARTY := picosat-965/picosat
libupdater_PKG_CONFIGS := $(LUA_NAME) libevent libcurl libcrypto
libupdater_PKG_CONFIGS := $(LUA_NAME) libevent libcurl libcrypto libarchive
# Workaround, lua.pc doesn't containd -ldl, even when it uses dlopen
libupdater_SO_LIBS += dl
......
......@@ -36,6 +36,11 @@ local stat = stat
local events_wait = events_wait
local run_util = run_util
local print = print
module "utils"
-- luacheck: globals lines2set map set2arr arr2set cleanup_dirs dir_ensure mkdirp read_file write_file clone shallow_copy table_merge arr_append exception multi_index private filter_best strip table_overlay randstr arr_prune arr_inv file_exists
......@@ -106,6 +111,9 @@ end
-- Run rm -rf on all dirs in the provided table
function cleanup_dirs(dirs)
print("cleanup " .. type(dirs))
if next(dirs) then
events_wait(run_util(function (ecode, _, _, stderr)
if ecode ~= 0 then
......
......@@ -27,6 +27,7 @@ local next = next
local tostring = tostring
local tonumber = tonumber
local assert = assert
local string = string
local unpack = unpack
local io = io
local os = os
......@@ -41,12 +42,17 @@ local LST_PKG_SCRIPT = LST_PKG_SCRIPT
local events_wait = events_wait
local stat = stat
local lstat = lstat
local mkdir = mkdir
-- local mkdir = mkdir
local move = move
local copy = copy
local ls = ls
local md5_file = md5_file
local sha256_file = sha256_file
-- local test_extract = test_extract
local extract_inner_archive = extract_inner_archive
local get_file_size = get_file_size
local archive_read_file = archive_read_file
local archive_file_present = archive_file_present
local DBG = DBG
local WARN = WARN
local ERROR = ERROR
......@@ -54,6 +60,9 @@ local syscnf = require "syscnf"
local utils = require "utils"
local locks = require "locks"
local print = print
module "backend"
-- Variables that we want to access from outside (ex. for testing purposes)
......@@ -417,6 +426,7 @@ function status_dump(status)
end
end
--[[
local function rmrf(dir)
-- TODO: Would it be better to remove from within our code, without calling rm?
events_wait(run_util(function (ecode, _, _, stderr)
......@@ -425,6 +435,8 @@ local function rmrf(dir)
end
end, nil, nil, cmd_timeout, cmd_kill_timeout, "rm", "-rf", dir))
end
]]
--[[
Take the .ipk package and unpack it into a temporary location somewhere under
......@@ -438,7 +450,35 @@ TODO:
• Sanity checking of the package.
• Less calling of external commands.
]]
-- First we duplicate old functionality, to keep the rewrite simple
function pkg_unpack(package_path)
print("====================pkg_unpack: " .. package_path)
-- We do not need temp directory, so let's just use s2dir (renamed to just dir)
utils.mkdirp(syscnf.pkg_unpacked_dir)
local dir = mkdtemp(syscnf.pkg_unpacked_dir)
extract_inner_archive(package_path, "control", dir)
extract_inner_archive(package_path, "data", dir)
local fs = get_file_size(package_path, "control", "conffiles")
print("file_size:" .. fs)
local data = get_file_content(package_path, "control", "conffiles")
print("(((in Lua, data)))\n" .. data .. "---")
print("dir: " .. dir)
return dir
end
--[[
function pkg_unpack(package_path)
INFO("a_08/pkg_unpack")
-- The first unpack goes into the /tmp
-- We assume s1dir returs sane names of directories ‒ no spaces or strange chars in them
local s1dir = mkdtemp()
......@@ -459,8 +499,10 @@ function pkg_unpack(package_path)
end
-- Unpack the control.tar.gz and data.tar.gz under respective subdirs in s2dir
local function unpack_archive(what)
INFO("a_08/unpack_archive")
local archive = s1dir .. "/" .. what .. ".tar.gz"
local dir = s2dir .. "/" .. what
test_extract(archive, dir)
mkdir(dir)
return run_util(function (ecode, _, _, stderr)
if ecode ~= 0 then
......@@ -486,6 +528,7 @@ function pkg_unpack(package_path)
-- Everything went well. So return path to the directory where the package is unpacked
return s2dir
end
]]--
--[[
Look into the dir with unpacked package (the one containing control and data subdirs).
......@@ -501,7 +544,7 @@ In all three cases, the file names are keys, not values.
In case of errors, it raises error()
]]
function pkg_examine(dir)
function orig_pkg_examine(dir)
local data_dir = dir .. "/data"
-- Events to wait for
local events = {}
......@@ -533,6 +576,7 @@ function pkg_examine(dir)
-- Get list of config files, if there are any
local control_dir = dir .. "/control"
local cidx = io.open(control_dir .. "/conffiles")
-- BB: use extract_file_to_memory_here
local conffiles = {}
if cidx then
for l in cidx:lines() do
......@@ -564,6 +608,75 @@ function pkg_examine(dir)
return files, dirs, conffiles, control
end
function pkg_examine(package_path, dir)
local data_dir = dir .. "/data"
-- Events to wait for
local events = {}
local err = nil
-- Launch scans of the data directory
local function launch(postprocess, ...)
local function cback(ecode, _, stdout, stderr)
if ecode == 0 then
postprocess(stdout)
else
err = stderr
end
end
print("-------launch() called--------")
local event = run_util(cback, function () chdir(data_dir) end, nil, cmd_timeout, cmd_kill_timeout, ...)
table.insert(events, event)
end
local function find_result(text)
--[[
Split into „lines“ separated by 0-char. Then eat leading dots and, in case
there was only a dot, replace it by /.
]]
return utils.map(utils.lines2set(text, "%z"), function (f) return f:gsub("^%.", ""):gsub("^$", "/"), true end)
end
local files, dirs
-- One for non-directories
launch(function (text) files = slashes_sanitize(find_result(text)) end, "find", "!", "-type", "d", "-print0")
-- One for directories
launch(function (text) dirs = slashes_sanitize(find_result(text)) end, "find", "-type", "d", "-print0")
-- Get list of config files, if there are any
local cidx = archive_read_file(package_path, "control", "conffiles")
print("BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB\n" .. cidx)
local conffiles = {}
if cidx then
for l in string.gmatch(cidx, "[^\n]+") do
local fname = l:match("^%s*(/.*%S)%s*")
if archive_file_present(package_path, "data", fname) then
conffiles[fname] = sha256(archive_read_file(package_path, "data", fname))
else
error("File " .. fname .. " does not exist.")
end
end
end
conffiles = slashes_sanitize(conffiles)
-- Load the control file of the package and parse it
local control = package_postprocess(block_parse(archive_read_file(package_path, "control", "control")))
-- Wait for all asynchronous processes to finish
events_wait(unpack(events))
-- How well did it go?
if err then
error(err)
end
-- Complete the control structure
control.files = files
if next(conffiles) then -- Don't store empty config files
control.Conffiles = conffiles
end
control["Installed-Time"] = tostring(os.time())
control.Status = {"install", "user", "installed"}
return files, dirs, conffiles, control
end
--[[
Check if we can perform installation of packages and no files
of other packages would get overwritten. It checks both the
......@@ -850,6 +963,9 @@ function pkg_merge_files(dir, dirs, files, configs)
end
end
-- Remove the original directory
print("cleanup from merge_files")
utils.cleanup_dirs({dir})
return true
end
......
......@@ -24,6 +24,7 @@ interrupted and the dangerous parts already started.
This is a fairly high-level module, connecting many things together.
]]
local tostring = tostring
local ipairs = ipairs
local next = next
......@@ -53,6 +54,9 @@ local sync = sync
local log_event = log_event
local system_reboot = system_reboot
local print = print
local type = type
module "transaction"
-- luacheck: globals perform recover perform_queue recover_pretty queue_remove queue_install queue_install_downloaded cleanup_actions
......@@ -68,6 +72,7 @@ end
-- Stages of the transaction. Each one is written into the journal, with its results.
local function pkg_unpack(operations, status)
print("+++pkg_unpack+++")
update_state(LS_UNPACK)
INFO("Unpacking download packages")
local dir_cleanups = {}
......@@ -93,7 +98,9 @@ local function pkg_unpack(operations, status)
elseif op.op == "install" then
local pkg_dir = backend.pkg_unpack(op.file)
table.insert(dir_cleanups, pkg_dir)
local files, dirs, configs, control = backend.pkg_examine(pkg_dir)
-- local files, dirs, configs, control = backend.pkg_examine(pkg_dir)
print("xxx Calling pkg_examine(" .. op.file .. ", " .. pkg_dir .. ")")
local files, dirs, configs, control = backend.pkg_examine(op.file, pkg_dir)
to_remove[control.Package] = true
to_install[control.Package] = files
--[[
......@@ -296,6 +303,13 @@ local function perform_internal(operations, journal_status, run_state)
-- Look at what the current status looks like.
local to_remove, to_install, plan
to_remove, to_install, plan, dir_cleanups, cleanup_actions = step(journal.UNPACKED, pkg_unpack, true, operations, status)
print(syscnf.pkg_download_dir)
print("cleanup from perrfom_internals1" .. type({syscnf.pkg_download_dir}))
utils.cleanup_dirs({syscnf.pkg_download_dir})
cleanup_actions = cleanup_actions or {} -- just to handle if journal contains no cleanup actions (journal from previous version)
-- Drop the operations. This way, if we are tail-called, then the package buffers may be garbage-collected
......@@ -307,6 +321,11 @@ local function perform_internal(operations, journal_status, run_state)
status, errors_collected = step(journal.SCRIPTS, pkg_scripts, true, status, plan, removes, to_install, errors_collected, all_configs)
end)
-- Make sure the temporary dirs are removed even if it fails. This will probably be slightly different with working journal.
print("cleanup from perrfom_internals2 " .. tostring(dir_cleanups))
utils.cleanup_dirs(dir_cleanups)
if not ok then
--[[
......
......@@ -408,6 +408,7 @@ function new(context, uri, verification)
sigval(stdout)
end
end
INFO("\n***GZIP called from `dispatch@10`***\n")
events_wait(run_util(gzip_done, nil, result.content, -1, -1, 'gzip', '-c', '-d'))
end
if not found then
......
......@@ -31,6 +31,7 @@ local string = string
local events_wait = events_wait
local run_util = run_util
local DBG = DBG
local INFO = INFO
local WARN = WARN
local ERROR = ERROR
local utils = require "utils"
......@@ -115,6 +116,7 @@ function get_repos()
elseif answer:sub(1, 2) == string.char(0x1F, 0x8B) then
-- It starts with gzip magic - we want to decompress it
DBG("Index " .. name .. " is compressed, decompressing")
INFO("\n***GZIP called from `get_repos@13`***\n")
table.insert(extract_events, run_util(decompressed, nil, answer, -1, -1, 'gzip', '-dc'))
else
parse(answer)
......
......@@ -26,6 +26,7 @@
#include "locks.h"
#include "arguments.h"
#include "picosat.h"
#include "unpacker.h"
#include <lua.h>
#include <lualib.h>
......@@ -632,6 +633,71 @@ static int lua_copy(lua_State *L) {
return 0;
}
static int lua_extract_inner_archive(lua_State *L) {
const char *arc_name = luaL_checkstring(L, 1);
const char *subarc_name = luaL_checkstring(L, 2);
const char *path = luaL_checkstring(L, 3);
int r = extract_inner_archive(arc_name, subarc_name, path);
/* TODO: error handling */
/* TODO: return something sensible to lua? */
return 0;
}
static int lua_get_file_size(lua_State *L) {
/* Returns -1 if file is not found in archive */
const char *arc_name = luaL_checkstring(L, 1);
const char *subarc_name = luaL_checkstring(L, 2);
const char *path = luaL_checkstring(L, 3);
int size = get_file_size(arc_name, subarc_name, path);
printf("File size of %s in %d.\n", path, size);
lua_pushinteger(L, size);
return 1;
}
static int lua_archive_read_file(lua_State *L) {
const char *arc_name = luaL_checkstring(L, 1);
const char *subarc_name = luaL_checkstring(L, 2);
const char *path = luaL_checkstring(L, 3);
int size = get_file_size(arc_name, subarc_name, path);
if (size > 0) {
char buffer[size];
extract_file_to_memory(buffer, arc_name, subarc_name, path, size);
lua_pushlstring(L, buffer, size);
} else {
lua_pushlstring(L, "", 0);
}
return 1;
}
static int lua_archive_file_present(lua_State *L) {
const char *arc_name = luaL_checkstring(L, 1);
const char *subarc_name = luaL_checkstring(L, 2);
const char *path = luaL_checkstring(L, 3);
printf("lua_calling fpia\n");
int ret = archive_file_present(arc_name, subarc_name, path);
if (ret == 0) {
lua_pushboolean(L, 1);
} else {
lua_pushboolean(L, 0);
}
printf("returned: %d\n", ret);
return 1;
}
void l_pushtablestring(lua_State* L , char* key , char* value) {
lua_pushstring(L, key);
lua_pushstring(L, value);
lua_settable(L, -3);
}
static int lua_archive_list_files(lua_State *L) {
const char *arc_name = luaL_checkstring(L, 1);
const char *subarc_name = luaL_checkstring(L, 2);
printf("lua_archive_list_names");
lua_newtable(L);
l_pushtablestring(L,
}
static const char *stat2str(const struct stat *buf) {
switch (buf->st_mode & S_IFMT) {
case S_IFSOCK:
......@@ -914,6 +980,11 @@ static const struct injected_func injected_funcs[] = {
{ lua_mkdir, "mkdir" },
{ lua_move, "move" },
{ lua_copy, "copy" },
{ lua_extract_inner_archive, "extract_inner_archive" },
{ lua_get_file_size, "get_file_size" },
{ lua_archive_read_file, "archive_read_file" },
{ lua_archive_file_present, "archive_file_present" },
{ lua_archive_list_files, "archive_list_files" },
{ lua_ls, "ls" },
{ lua_stat, "stat" },
{ lua_lstat, "lstat" },
......
This diff is collapsed.
#include <archive.h>
#include <archive_entry.h>
int unpacker_test();
/*
*
*/
/*int extract_file(struct archive *a, const char *filename);*/
/*
*
*/
int extract_files(struct archive *a, char *files[], int count);
/*
*
*/
int extract_to_disk(const char *arc_name, const char *subarc_name, char *files[], int count);
int extract_inner_archive(const char* arcname, const char* subarcname, const char *path);
int get_file_size(const char *arcname, const char *subarcname, const char *filename);
int extract_file_to_memory(char *buff, const char *arcname, const char *subarcname, const char *filename, int size);
int test_extract(const char *arc_name, const char *subarc_name, char *files[], int count);
int archive_file_present(const char *arcname, const char *subarcname, const char *filename);
......@@ -298,7 +298,7 @@ function test_pkg_unpack()
./data/usr/share/updater/keys/standby.pem
]]), lines2set(stdout))
end, function () chdir(path) end, nil, -1, -1, "/usr/bin/find"))
local files, dirs, conffiles, control = B.pkg_examine(path)
local files, dirs, conffiles, control = B.pkg_examine(datadir .. "updater.ipk", path)
assert_table_equal(lines2set(
[[
/etc/init.d/updater
......
......@@ -12,6 +12,7 @@ globals = {
-- From interpreter.c
"log", "state_log_enabled", "update_state", "cleanup_register_handle",
"cleanup_unregister_handle", "run_command", "run_util", "download",
"extract_inner_archive", "get_file_size",
"events_wait", "mkdtemp", "chdir", "getcwd", "mkdir", "move", "copy", "ls",
"stat", "lstat", "sync", "setenv", "md5", "sha256", "md5_file", "sha256_file",
"reexec", "uri_internal_get", "system_reboot", "get_updater_version",
......