Verified Commit b2497b13 authored by Petr Špaček's avatar Petr Špaček

CI coverage: merge test coverage data from parallel runs

We run tests in paralell so have to make sure that coverage tools
do not overwrite results from each run.

This is hacky because lcov tool insists on having gcno and gcda files
in the same place as original source, so we have to copy files
to workaround this.
parent 77af3465
......@@ -127,6 +127,7 @@ test:linux:amd64:valgrind:
respdiff:iter:udp:linux:amd64:
stage: test
script:
- source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/udp" --export)
- PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
- ./ci/respdiff/run-respdiff-tests.sh udp
- cat results/respdiff.txt
......@@ -150,6 +151,7 @@ respdiff:iter:udp:linux:amd64:
respdiff:iter:tcp:linux:amd64:
stage: test
script:
- source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/tcp" --export)
- PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
- ./ci/respdiff/run-respdiff-tests.sh tcp
- cat results/respdiff.txt
......@@ -173,6 +175,7 @@ respdiff:iter:tcp:linux:amd64:
respdiff:iter:tls:linux:amd64:
stage: test
script:
- source <(./scripts/coverage_env.sh "$(pwd)" "$(pwd)/coverage.stats/respdiff" "iter/tls" --export)
- PREFIX=$(pwd)/.local ./ci/respdiff/start-resolvers.sh
- ./ci/respdiff/run-respdiff-tests.sh tls
- cat results/respdiff.txt
......
......@@ -6,33 +6,12 @@ all: info lib daemon client modules etc
install: lib-install daemon-install client-install modules-install etc-install
check: all tests
clean: contrib-clean lib-clean daemon-clean client-clean modules-clean \
tests-clean doc-clean bench-clean
tests-clean doc-clean bench-clean coverage-clean
doc: doc-html
lint: $(patsubst %.lua.in,%.lua,$(wildcard */*/*.lua.in))
luacheck --codes --formatter TAP .
coverage-c:
@echo "# C coverage in $(COVERAGE_STAGE).c.info"
@$(LCOV) --no-external --capture -d lib -d daemon -d modules -o $(COVERAGE_STAGE).c.info > /dev/null
coverage-lua: $(shell find -type f -name 'luacov.stats.out')
@# map install paths to source paths
@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua --sed > .luacov_path_map
@find -type f -name 'luacov.stats.out' | xargs sed -i -f .luacov_path_map
@rm .luacov_path_map
@# add missing Lua files
@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua | cut -f 2 | grep '\.lua$$' | scripts/luacov_gen_empty.sh > luacov.empty_stats.out
@echo "# Lua coverage in $(COVERAGE_STAGE).lua.info"
@scripts/luacov_to_info.lua luacov.empty_stats.out $^ > $(COVERAGE_STAGE).lua.info
@rm luacov.empty_stats.out
coverage:
@$(LCOV) $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STAGE)*.info)) --output-file coverage.info
@$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p $(realpath $(CURDIR)) -t "Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report" --legend coverage.info
.PHONY: all install check clean doc info coverage
# Options
ifdef COVERAGE
BUILD_CFLAGS += --coverage
endif
.PHONY: all install check clean doc info lint
# Dependencies
KNOT_MINVER := 2.4.0
......@@ -194,6 +173,7 @@ $(DESTDIR)$(ETCDIR):
# Sub-targets
include contrib/contrib.mk
include coverage.mk
include lib/lib.mk
include client/client.mk
include daemon/daemon.mk
......
-- measure code coverage: kresd must run from $GIT_DIR
require('tests.config.coverage')
-- Refer to manual: https://knot-resolver.readthedocs.io/en/latest/daemon.html#configuration
-- Listen on localhost and external interface
net.listen('127.0.0.1', 5353)
......
......@@ -21,6 +21,8 @@ MODULEDIR ?= $(LIBDIR)/kdns_modules
ETCDIR ?= $(PREFIX)/etc/kresd
ROOTHINTS ?= $(ETCDIR)/root.hints
COVERAGE_STAGE ?= gcov
COVERAGE_STATSDIR ?= $(CURDIR)/coverage.stats
TOPSRCDIR := $(CURDIR)
# Tools
CC ?= cc
......
# Measure code coverage using luacov and gcov
# C and Lua code is measured separately and resutls are combined together
# Define COVERAGE=1 during build *and* test runs to enable measurement.
#
# Beware: Tests are typically run in parallel and neither luacov not gcov
# support that, so we have to store results from each run separatelly
# and combine them.
coverage-c-combine-gcda:
@# combine trees of gcda files into one info file per tree
@mkdir -p "$(COVERAGE_STATSDIR)/tmp.c"
@LCOV=$(LCOV) ./scripts/coverage_c_combine.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)" "$(COVERAGE_STATSDIR)/tmp.c"
coverage-c: coverage-c-combine-gcda
@# combine info files for each tree into resulting c.info file
@$(LCOV) -q $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STATSDIR)/tmp.c/*.info)) --output-file "$(COVERAGE_STAGE).c.info"
@$(RM) -r "$(COVERAGE_STATSDIR)/tmp.c"
coverage-lua-fix-paths: $(shell find -type f -name 'luacov.stats.out')
@# map Lua install paths to source paths
@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua --sed > .luacov_path_map
@sed -i -f .luacov_path_map $^
@rm .luacov_path_map
coverage-lua: coverage-lua-fix-paths
@# add missing Lua files to stats
@$(MAKE) PREFIX=$(PREFIX) install --dry-run --always-make | scripts/map_install_src.lua | cut -f 2 | grep '\.lua$$' | scripts/luacov_gen_empty.sh > luacov.empty_stats.out
@echo "# Lua coverage in $(COVERAGE_STAGE).lua.info"
@scripts/luacov_to_info.lua luacov.empty_stats.out $(shell find -type f -name 'luacov.stats.out') > $(COVERAGE_STAGE).lua.info
@rm luacov.empty_stats.out
coverage:
@$(LCOV) $(addprefix --add-tracefile ,$(wildcard $(COVERAGE_STAGE)*.info)) --output-file coverage.info
@$(GENHTML) --no-function-coverage --no-branch-coverage -q -o coverage -p $(realpath $(CURDIR)) -t "Knot DNS Resolver $(VERSION)-$(PLATFORM) coverage report" --legend coverage.info
coverage-clean:
@rm -rf "$(COVERAGE_STATSDIR)"
.PHONY: coverage-c-combine-gcda coverage-c coverage-lua-fix-paths coverage-lua coverage coverage-clean
# Options
ifdef COVERAGE
BUILD_CFLAGS += --coverage
endif
#!/bin/bash
# $1 = top source directory
# $2 = coverage data directory path
# $3 = output directory for *.info files
set -o errexit -o nounset
shopt -s nullglob
IFS=$'\n'
TOPSRCDIR="$1"
DATAROOT="$2"
OUTDIR="$3"
cd "${TOPSRCDIR}"
for COVNAME in $(find "${DATAROOT}" -name .topdir_kresd_coverage)
do
find "${DATAROOT}" -name '*.gcda' -not -path "${DATAROOT}/*" -delete
COVDIR="$(dirname "${COVNAME}")"
COVDATA_FILENAMES=("${COVDIR}"/*) # filenames in BASH array
(( ${#COVDATA_FILENAMES[*]} )) || continue # skip empty dirs
cp -r -t ${TOPSRCDIR} "${COVDIR}"/*
${LCOV} -q --no-external --capture -d lib -d daemon -d modules -o "$(mktemp -p "${OUTDIR}" -t XXXXXXXX.c.info)" > /dev/null
done
#!/bin/bash
# generate variables for coverage testing
# $1 = top source directory
# $2 = coverage data directory path
# $3 = name of test/new subdirectory name
# $4 = [optional] --export to generate export commands
set -o errexit -o nounset
shopt -s nullglob
test -z "${COVERAGE:-}" && exit 0 # not enabled, do nothing
test ! -z "${V:-}" && set -o xtrace # verbose mode
EXPORT=""
test "${4:-}" == "--export" && EXPORT="export "
TOPSRCDIR="$1"
DATAROOT="$2"
OUTPATH="$2/$3"
# check that output directory is empty
# beware: Makefile will always call coverage_env.sh for all targets
# so directories get created but not populated
# i.e. test -d is not sufficient check
OUTPATH_FILENAMES=("${OUTPATH}"/*) # filenames in BASH array
(( ${#OUTPATH_FILENAMES[*]} )) && echo "false" && >&2 echo "fatal: output directory ${OUTPATH} must be empty (or non-existent)" && exit 1
mkdir -p "${OUTPATH}"
# convert paths to absolute
pushd "${OUTPATH}" &> /dev/null
touch .topdir_kresd_coverage
OUTPATH="$(pwd -P)"
popd &> /dev/null
# determine GCOV_PREFIX_STRIP value for current source directory
TOPSRCDIR_SLASHES="${TOPSRCDIR//[^\/]/}" # remove everything except /
GCOV_PREFIX_STRIP="${#TOPSRCDIR_SLASHES}" # numer of / == number of components
KRESD_COVERAGE_STATS="${OUTPATH}/luacov.stats.out"
GCOV_PREFIX="${OUTPATH}"
echo "${EXPORT}KRESD_COVERAGE_STATS=\"${KRESD_COVERAGE_STATS}\" ${EXPORT}GCOV_PREFIX=\"${GCOV_PREFIX}\" ${EXPORT}GCOV_PREFIX_STRIP=\"${GCOV_PREFIX_STRIP}\""
-- optional code coverage
-- include this file into config if you want to generate coverage data
local ok, runner = pcall(require, 'luacov.runner')
if ok then
runner.init({
savestepsize = 2, -- TODO
statsfile = 'luacov.stats.out',
exclude = {'test', 'tapered'},
})
jit.off()
end
package.path = package.path .. ';' .. env.SOURCE_PATH .. '/?.lua'
TEST_DIR = env.TEST_FILE:match('(.*/)')
require('coverage')
-- export testing module in globals
local tapered = require('tapered.src.tapered')
for k, v in pairs(tapered) do
......
......@@ -11,13 +11,10 @@ tests_config := \
define make_config_test
$(1): check-install-precond
@$(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(abspath $(1))
$(1)-clean:
@$(RM) $(dir $(1))/luacov.stats.out
@$(shell ./scripts/coverage_env.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)/tests_config" "$(1)") $(preload_syms) ./tests/config/runtest.sh $(abspath $(SBINDIR)/kresd) $(abspath $(1))
.PHONY: $(1)
endef
$(foreach test,$(tests_config),$(eval $(call make_config_test,$(test))))
check-config: $(tests_config)
check-config-clean: $(foreach test,$(tests_config),$(test)-clean)
.PHONY: check-config
Subproject commit 8e533979fe667fdc0bf6975d643d9e98c37f3b41
Subproject commit 387e8845a45a7bab6ab1c8e5be55a8d88d0315ae
......@@ -39,7 +39,7 @@ check-install-precond:
check-integration: check-install-precond $(deckard_DIR)/Makefile
$(if $(SUBMODULES_DIRTY), $(warning Warning: Git submodules are not up-to-date),)
@mkdir -p $(deckard_DIR)/contrib/libswrap/obj
+TESTS=$(TESTS) DAEMON=$(abspath $(SBINDIR)/kresd) TEMPLATE=$(TEMPLATE) $(preload_syms) $(deckard_DIR)/kresd_run.sh
+TESTS=$(TESTS) DAEMON=$(abspath $(SBINDIR)/kresd) TEMPLATE=$(TEMPLATE) COVERAGE_ENV_SCRIPT=$(TOPSRCDIR)/scripts/coverage_env.sh DAEMONSRCDIR=$(TOPSRCDIR) COVERAGE_STATSDIR=$(COVERAGE_STATSDIR)/deckard $(preload_syms) $(deckard_DIR)/kresd_run.sh
deckard: check-integration
......
......@@ -30,7 +30,7 @@ $(1)_LIBS := $(tests_LIBS)
$(1)_DEPEND := $(tests_DEPEND)
$(call make_bin,$(1),tests)
$(1): $$($(1))
@$(preload_syms) $(DEBUGGER) $$<
@$(shell ./scripts/coverage_env.sh "$(TOPSRCDIR)" "$(COVERAGE_STATSDIR)/tests_unit" "$(1)") $(preload_syms) $(DEBUGGER) $$<
.PHONY: $(1)
endef
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment