Commit faa3c376 authored by Karel Koci's avatar Karel Koci 🤘

Generate test coverage information

Defining "COV' for make will enable generation of coverage data. Then
running tests will create those data and html can be generated with
"COV=y make coverage".

Coverage is generated for both C and Lua. C uses gcov. Lua is our hacky
implementation.
parent 92952531
......@@ -5,12 +5,16 @@
/.deps/
/.objs/
/.gen/
/.lua_coverage/
# The desired results
*.html
*.so
/src/opkg-trans/opkg-trans
/src/pkgupdate/pkgupdate
/src/migrator/pkgmigrate
# Some stuff for debugging
/.coverage.info
/coverage/
# Stuff dump core all over the place.
core
vgcore.*
......
Subproject commit 4f7843d28501e0ce66f812bd7079ebca923c22f7
Subproject commit 4ad75c84674f90e31fc9c7be0d5f59509f90b020
......@@ -9,6 +9,9 @@ $(O)/.gen/src/%.embedlist: $(S)/src/lib/gen_embed.sh $(S)/src/lib/embed_types.h
$(O)/.gen/src/lib/lautoload.embedlist: $(wildcard $(S)/src/lib/autoload/*.lua)
$(O)/.gen/src/lib/lautoload.embedlist: SUFFIX := .lua
$(O)/.gen/src/lib/lcoverage.embedlist: $(S)/src/lib/coverage.lua
$(O)/.gen/src/lib/lcoverage.embedlist: SUFFIX := .lua
libupdater_MODULES := \
arguments \
inject \
......@@ -20,6 +23,9 @@ libupdater_MODULES := \
locks \
picosat \
util
ifdef COV
libupdater_MODULES += lcoverage.embed
endif
libupdater_MODULES_3RDPARTY := \
md5 \
......
--[[
Copyright 2016, CZ.NIC z.s.p.o. (http://www.nic.cz/)
This file is part of the turris updater.
Updater is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Updater is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Updater. If not, see <http://www.gnu.org/licenses/>.
]]--
local getmetatable = getmetatable
local setmetatable = setmetatable
local tostring = tostring
local pairs = pairs
local pcall = pcall
local io = io
local os = os
local mkdir = mkdir
local debug = debug
module "coverage"
coverage_data = {}
--[[
Notes that given line from given source was executed. source is string. It is name
of module for preloaded modules and path for real files. If it's path then has
prepended '@'.
]]
local function line(event, line)
local info = debug.getinfo(2, 'S')
local source = info.source;
-- ignore lines outside of lua (C = line is -1) and chunks outside of any file (starting with "Chunk")
if line == -1 or source:match('^Chunk') then return end
if not coverage_data[source] then coverage_data[source] = {} end
if not coverage_data[source][line] then coverage_data[source][line] = 0 end
coverage_data[source][line] = coverage_data[source][line] + 1
end
--[[
Dumps all coverage data collected so far. It creates or appends data to file named
after source. If source is module, then it's directly name of file with added
postfix ".lua_lines". If source is path (starts with @), then all "/" are replaced
to "-" and same postfix is added as for module.
]]
function dump(dir)
pcall(mkdir, dir) -- ignore all errors, this is just to ensure existence of this directory
for mod, lines in pairs(coverage_data) do
local fname = mod .. ".lua_lines"
if fname:sub(1, 1) == '@' then
fname = fname:gsub('/', '-')
end
fname = dir .. '/' .. fname
file = io.open(fname, 'a')
for line, hits in pairs(lines) do
file:write(tostring(line) .. ":" .. tostring(hits) .. "\n")
end
end
end
--[[
We want to be called when program exits automatically. We use Lua garbage
collector for that. But we can hook on it only on user data. We create one when
coverage scan is started and hold it in this variable until program exit or at
least when lua is not going to be used any more.
]]
gc_udata = nil
-- Setup line hook to lua
debug.sethook(line, 'l')
......@@ -728,6 +728,45 @@ static const struct injected_func injected_funcs[] = {
{ lua_uri_internal_get, "uri_internal_get" }
};
#ifdef COVERAGE
// From the embed file. Coverage lua code.
extern struct file_index_element lcoverage[];
static int interpreter_coverage_dump(lua_State *L) {
char *out_dir = getenv("COVERAGEDIR");
if (!out_dir) {
WARN("COVERAGEDIR variable not specified. Skipping coverage dump");
return 0;
}
DBG("Executing coverage data dump.");
int handler = push_err_handler(L);
lua_getfield(L, LUA_GLOBALSINDEX, "coverage");
lua_getfield(L, -1, "dump"); // called function
lua_pushstring(L, out_dir); // argument
if (lua_pcall(L, 1, 0, handler))
ERROR("Coverage data dump failed: %s", interpreter_error_result(L));
lua_pop(L, 1); // pop coverage module from stack
lua_remove(L, handler);
return 0;
}
static void interpreter_load_coverage(struct interpreter *interpreter) {
lua_State *L = interpreter->state;
DBG("Initializing Lua code coverage");
if (!interpreter_include(interpreter, (const char *) lcoverage->data, lcoverage->size, "coverage")) {
lua_getfield(L, LUA_GLOBALSINDEX, "coverage"); // get this module
lua_newuserdata(L, 1); // push to stack dummy user data. They are freed by Lua it self not our code.
lua_newtable(L); // new meta table for user data
lua_pushcfunction(L, interpreter_coverage_dump);
lua_setfield(L, -2, "__gc"); // set function to to new table
lua_setmetatable(L, -2); // set new table as user data meta table
lua_setfield(L, -2, "gc_udata"); // Set dummy user data to coverage.gc_udata
lua_pop(L, 1); // Pop coverage module from stack
} else
WARN("Loading of Lua coverage code failed.");
}
#endif
struct interpreter *interpreter_create(struct events *events, const struct file_index_element *uriinter) {
uriinternal = uriinter;
struct interpreter *result = malloc(sizeof *result);
......@@ -754,6 +793,9 @@ struct interpreter *interpreter_create(struct events *events, const struct file_
journal_mod_init(L);
locks_mod_init(L);
picosat_mod_init(L);
#ifdef COVERAGE
interpreter_load_coverage(result);
#endif
return result;
}
......
.PHONY: check-clean test valgrind luacheck check test-locks valgrind-locks luac-autoload luacheck
.PHONY: check-clean test valgrind luacheck check test-locks valgrind-locks luac-autoload luacheck coverage $(O)/.coverage.info
BINARIES_NOTARGET += tests/locks
locks_MODULES += locks
......@@ -40,6 +40,26 @@ LUA_TESTS := \
uri \
picosat
ifdef COV
LUA_TESTS += coverage
# Aggregates coverage to info file for genhtml.
# It uses lcov for coverage from C sources and our script for coverage from Lua
# No dependencies are intentional, this way coverage can be run for all tests or
# just for small subset of them. It's on user to prepare data by launching tests.
$(O)/.coverage.info:
lcov --capture --directory $(O) --base-directory $(S) --output-file $@
$(S)/tests/lua_coverage2info.pl $(O)/.lua_coverage $@ $(S)
coverage: $(O)/.coverage.info
genhtml $< --output-directory $(O)/coverage
endif
clean: clean-coverage
clean-coverage:
rm -rf $(O)/.lua_coverage
rm -f $(O)/.coverage.info
rm -rf $(O)/coverage
# Ignore stacktraceplus and dumper, not our creation.
LUA_AUTOLOAD := $(filter-out 01_stacktraceplus 05_dumper,$(patsubst a_%.lua,%,$(notdir $(wildcard $(S)/src/lib/autoload/a_*.lua))))
......@@ -74,21 +94,23 @@ luacheck: $(addprefix luacheck-,$(LUA_AUTOLOAD))
# Use the FORCE target instead of .PHONY, since .PHONY doesn't work well
# with patterned recipes. The FORCE comes from our shared build system.
TESTS_ENV = SUPPRESS_LOG=1 S=$(S) TMPDIR=$(abspath $(O)/testdir) COVERAGEDIR=$(abspath $(O)/.lua_coverage)
test-c-%: $(O)/bin/ctest-% FORCE
mkdir -p $(O)/testdir
SUPPRESS_LOG=1 S=$(S) TMPDIR=$(abspath $(O)/testdir) $<
$(TESTS_ENV) $<
valgrind-c-%: $(O)/bin/ctest-% FORCE
mkdir -p $(O)/testdir
CK_FORK=no SUPPRESS_LOG=1 S=$(S) TMPDIR=$(abspath $(O)/testdir) $(VALGRIND) $<
CK_FORK=no $(TESTS_ENV) $(VALGRIND) $<
test-lua-%: $(S)/tests/%.lua $(O)/bin/lulaunch FORCE
mkdir -p $(O)/testdir
SUPPRESS_LOG=1 S=$(S) TMPDIR=$(abspath $(O)/testdir) $(O)/bin/lulaunch $<
$(TESTS_ENV) $(O)/bin/lulaunch $<
valgrind-lua-%: $(S)/tests/%.lua $(O)/bin/lulaunch FORCE
mkdir -p $(O)/testdir
SUPPRESS_LOG=1 S=$(S) TMPDIR=$(abspath $(O)/testdir) $(VALGRIND) $(O)/bin/lulaunch $<
$(TESTS_ENV) $(VALGRIND) $(O)/bin/lulaunch $<
luac-autoload-%: $(S)/src/lib/autoload/a_%.lua FORCE
luac -p $<
......
--[[
Copyright 2016, CZ.NIC z.s.p.o. (http://www.nic.cz/)
This file is part of the turris updater.
Updater is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
Updater is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with Updater. If not, see <http://www.gnu.org/licenses/>.
]]--
require 'lunit'
local C = require 'coverage'
local debug = debug
local io = io
module("sandbox-tests", package.seeall, lunit.testcase)
function test_line()
-- This expect specific lines so we do some dummy operations
local a = 1
local b = 2
a = b + a
for i = 1, 3 do
b = b + 1
end
-- Now lets check that they were recorded (we are coverage module)
local source = debug.getinfo(1, 'S').source -- What we are?
local cov = C.coverage_data[source]
assert_equal(1, cov[29])
assert_equal(1, cov[30])
assert_equal(1, cov[31])
assert_equal(4, cov[32])
assert_equal(3, cov[33])
end
-- Check that when we call dump, file for this module is created
function test_dump()
local source = debug.getinfo(1, 'S').source -- What we are?
local fname = source:gsub('/', '-') .. '.lua_lines'
local dir = os.getenv("COVERAGEDIR")
-- Call dump
C.dump(dir)
-- Check if file exists
local f = io.open(dir .. "/" .. fname, "r")
assert(f)
io.close(f)
end
#!/usr/bin/env perl
# Copyright 2016, CZ.NIC z.s.p.o. (http://www.nic.cz/)
#
# This file is part of the turris updater.
#
# Updater is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Updater is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Updater. If not, see <http://www.gnu.org/licenses/>.
# Appends coverage info from lua to given output
# Usage: ./lua_coverage2info.sh COVERAGE_DIR OUT_INFO SOURCE
# Where COVERAGE_DIR is directory with coverage files from Lua, OUT_INFO is
# output file and SOURCE is original source directory
# TODO support functions, currently we only handle executed lines.
use common::sense;
use Cwd 'abs_path';
my @cfs = <$ARGV[0]/*>;
open my $outf, '>>', $ARGV[1] or die "Couldn't append to file $ARGV[1]: $!\n";
my $source = $ARGV[2];
# We intensionally ignore some modules here to not show them in output.
# TODO probably implement some search here instead
my %module2path = (
coverage => "coverage.lua",
testing => "autoload/a_02_testing.lua",
logging => "autoload/a_03_logging.lua",
utils => "autoload/a_04_utils.lua",
backend => "autoload/a_06_backend.lua",
transaction => "autoload/a_07_transaction.lua",
uri => "autoload/a_08_uri.lua",
requests => "autoload/a_09_requests.lua",
sandbox => "autoload/a_10_sandbox.lua",
postprocess => "autoload/a_11_postprocess.lua",
planner => "autoload/a_12_planner.lua",
updater => "autoload/a_13_updater.lua",
migrator => "autoload/a_14_migrator.lua"
);
foreach my $module (keys %module2path) {
$module2path{$module} = abs_path($source . '/src/lib/' . $module2path{$module});
}
# Collects hits from file and writes them to output
# first argument is source file, second one is file with coverage data
# This implementation is very naive, specially when we are reading source and adding unexecuted lines.
sub add_file($$) {
my ($source, $lines) = @_;
print $outf "TN:\n";
print $outf "SF:" . abs_path($source) . "\n"; # absolute path to source file
open my $inf, '<', $lines or die "Couldn't read $lines: $!\n";
my %dt;
while (<$inf>) {
my ($line, $count) = /^([\d]+):([\d]+)$/;
$dt{$line} += $count;
}
close $inf;
# Read source and found lines with code without execution history
open my $inf, $source or die "Couldn't read $lines: $!\n";
my $i = 1;
my $multiline = 0; # ignore multi-line comments
while (<$inf>) {
$multiline = 1 if /--\[\[/;
$dt{$i} //= 0 unless
$multiline or # we are in multi-line comment
/^[\s]*$/ or # ignore empty lines
/^[\s]*--/ or # Ignore single line comments
/^(end|else|\)|\}|[\s])*$/ # Ignore lines just with end, else, ) or }
;
$multiline = 0 if /\]\]/;
$i++;
}
close $inf;
foreach my $l (sort { $a <=> $b } keys %dt) {
print $outf "DA:$l,$dt{$l}\n";
}
my $lines = scalar keys %dt;
print $outf "LH:$lines\n";
# We print that we executed all lines we know about, but it seems make no difference, we receive correct percent coverage anyway.
print $outf "LF:$lines\n";
print $outf "end_of_record\n";
}
foreach (@cfs) {
chomp;
my $cfsource = s/^$ARGV[0]\///r =~ s/-/\//gr =~ s/\.lua_lines//r;
if ($cfsource =~ s/^@//) { # We should have path
$cfsource = $source . '/' . $cfsource; # relative to source
} else { # We have module name
$cfsource = $module2path{$cfsource} if defined $module2path{$cfsource};
}
if (-f $cfsource) {
add_file $cfsource, $_;
} else {
warn "$cfsource ignored. Can't locate file.\n";
}
}
close $outf;
......@@ -47,6 +47,7 @@ if [ -f "$DEFINITION"/setup ] ; then
fi
find "$TMP_DIR" -type f -name .keep -exec rm {} \;
# Launch it
export COVERAGEDIR="$O/.lua_coverage"
eval $LAUNCHER "$O"/bin/"$1" $(cat "$DEFINITION"/params)
# Do we need to de-randomize the output somehow?
if [ -f "$DEFINITION"/cleanup ] ; then
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment