Commit 3cd8f3cf authored by Štěpán Balážik's avatar Štěpán Balážik Committed by Tomas Krizek

Migrate running of Deckard to use py.test

parent 94ed593b
......@@ -73,10 +73,8 @@ test:comparative:kresd:
expire_in: '1 hour'
paths:
- modified_tests
- base.log
- base.tests
- head.log
- head.tests
- base.xml
- head.xml
tags:
- docker
- linux
......@@ -92,7 +90,7 @@ test:latest:kresd:
- ( cd kresd-local-build ; git submodule update --init --recursive )
- PREFIX=$(pwd)/.local make -C kresd-local-build -k all
- PREFIX=$(pwd)/.local make -C kresd-local-build install
- TMPDIR=$(pwd) LD_LIBRARY_PATH=$(pwd)/.local/lib DAEMON=$(pwd)/.local/sbin/kresd MAKEFLAGS="-j $(nproc) --keep-going" ./kresd_run.sh
- TMPDIR=$(pwd) LD_LIBRARY_PATH=$(pwd)/.local/lib DAEMON=$(pwd)/.local/sbin/kresd ./kresd_run.sh -n $(nproc)
artifacts:
when: on_failure
expire_in: 1 week
......@@ -109,8 +107,8 @@ test:latest:kresd:
test:sanity:unbound:
stage: test
script:
- TESTS=sets/resolver/iter_hint_lame.rpl TMPDIR=$(pwd) ./unbound_run.sh
- TESTS=sets/resolver/iter_lame_root.rpl TMPDIR=$(pwd) ./unbound_run.sh
- TMPDIR=$(pwd) ./unbound_run.sh -k sets/resolver/iter_hint_lame.rpl
- TMPDIR=$(pwd) ./unbound_run.sh -k sets/resolver/iter_lame_root.rpl
# these do not work with Unbound 1.5.8 which is in CI container
#- TESTS=sets/resolver/nsec_wildcard_answer_response.rpl ./unbound_run.sh
#- TESTS=sets/resolver/world_cz_lidovky_www.rpl ./unbound_run.sh
......@@ -130,8 +128,8 @@ test:sanity:unbound:
test:sanity:pdnsrecursor:
stage: test
script:
- TESTS=sets/resolver/iter_recurse.rpl TMPDIR=$(pwd) ./pdns_run.sh
- TESTS=sets/resolver/iter_tcbit.rpl TMPDIR=$(pwd) ./pdns_run.sh
- TMPDIR=$(pwd) ./pdns_run.sh -k sets/resolver/iter_recurse.rpl
- TMPDIR=$(pwd) ./pdns_run.sh -k sets/resolver/iter_tcbit.rpl
artifacts:
when: on_failure
expire_in: 1 week
......
# Defaults
TESTS ?= sets/resolver
DAEMON ?= kresd
TEMPLATE ?= template/kresd.j2
CONFIG ?= config
ADDITIONAL ?= -f 1
OPTS ?=
PYTHON ?= python3
LIBEXT := .so
PLATFORM := $(shell uname -s)
......@@ -13,14 +6,6 @@ ifeq ($(PLATFORM),Darwin)
LIBEXT := .dylib
endif
# Find all sub-targets
TARGETS := $(TESTS)
ifeq (,$(findstring .rpl,$(TESTS)))
TARGETS := $(wildcard $(TESTS)/*.rpl)
endif
SOURCES := $(TARGETS)
TARGETS := $(sort $(patsubst %.rpl,%.out-qmin,$(SOURCES))) $(sort $(patsubst %.rpl,%.out-noqmin,$(SOURCES)))
# Dependencies
include platform.mk
libcwrap_DIR := contrib/libswrap
......@@ -40,37 +25,13 @@ else
preload_syms := LD_PRELOAD="$(libfaketime):$(libcwrap)"
endif
# Test coverage measurement
# User has to provide own coverage_env.sh to generate environment variables for daemon under test
ifdef COVERAGE
ifndef COVERAGE_ENV_SCRIPT
$(error COVERAGE requires COVERAGE_ENV_SCRIPT with path to scripts/coverage_env.sh for given daemon)
endif
ifndef DAEMONSRCDIR
$(error COVERAGE requires DAEMONSRCDIR pointing to source directory of daemon under test)
endif
ifndef COVERAGE_STATSDIR
$(error COVERAGE requires COVERAGE_STATSDIR pointing to output directory)
endif
define set_coverage_env
$(shell "$(COVERAGE_ENV_SCRIPT)" "$(DAEMONSRCDIR)" "$(COVERAGE_STATSDIR)" "$(1)")
endef
endif
# Targets
all: $(TARGETS)
all:
@echo "Deckard is now run using *run.sh scripts in its root directory.\n\
To build the dependencies (libfaketime and libcwrap) run 'make depend'."
depend: $(libfaketime) $(libcwrap)
# Generic rule to run test
$(SOURCES): depend
%.out-qmin: %.rpl
@test "$${QMIN:-true}" = "true" || exit 0 && \
$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin true $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
%.out-noqmin: %.rpl
@test "$${QMIN:-false}" = "false" || exit 0 && \
$(call set_coverage_env,$@) $(preload_syms) $(PYTHON) $(abspath ./deckard.py) --qmin false $(OPTS) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
@echo "export $(preload_syms)" > env.sh
# Synchronize submodules
submodules: .gitmodules
......@@ -92,4 +53,4 @@ $(libcwrap): $(libcwrap_cmake_DIR)/Makefile
check:
@echo Running unittests using pytest
${PYTHON} -m pytest
${PYTHON} -m pytest --ignore=tests/test_runner.py
......@@ -32,22 +32,11 @@ MODIFIED_TESTS_FILE="/tmp/modified_tests"
find_modified_tests
LOGS[0]="${MODIFIED_TESTS_FILE}"
: get results from all tests, including the failing ones
export MAKEFLAGS="--output-sync=target --keep-going -j$(nproc)"
: get test results from version under test
PYTHON=${PYTHON} "${TESTRUNNER}" &> /tmp/head.log || :
LOGS[1]="/tmp/head.log"
extract_test_results /tmp/head.log | filter_test_results &> /tmp/head.tests || (: "no tests left, huh?" && cat /tmp/head.log)
LOGS[2]="/tmp/head.tests"
"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/head.xml
: get test results from common ancestor with master branch
git checkout --force --detach "${MERGEBASE}"
git clean -xdf
PYTHON=${PYTHON} "${TESTRUNNER}" &> /tmp/base.log || :
LOGS[3]="/tmp/base.log"
extract_test_results /tmp/base.log | filter_test_results &> /tmp/base.tests || (: "no tests left, huh?" && cat /tmp/base.log)
LOGS[4]="/tmp/base.tests"
: tests which were not modified should produce the same results
diff -U0 /tmp/base.tests /tmp/head.tests && echo "OK, no differences found"
"${TESTRUNNER}" -n $(nproc) --junit-xml=/tmp/base.xml
test -z "$(./junit_compare.py /tmp/head.xml /tmp/base.xml /tmp/modified_tests)" && echo "OK, no differences found"
\ No newline at end of file
#!/usr/bin/python3
import xml.etree.ElementTree as xml
import sys
def parse_xml(filename):
results = set()
root = xml.parse(filename).getroot()
for case in root:
if case.find("failure") is not None: # Because empty XML elements are falsey
results.add((case.get("name"), case.get("name").split("'")[1], "FAILED"))
elif case.find("skipped") is not None:
results.add((case.get("name"), case.get("name").split("'")[1], "SKIPPED"))
else:
results.add((case.get("name"), case.get("name").split("'")[1], "PASSED"))
return results
new = sys.argv[1]
old = sys.argv[2]
modified_tests = [line.strip() for line in open(sys.argv[3]).readlines()]
for diff in parse_xml(old) ^ parse_xml(new):
print(diff[1])
if diff[1] not in modified_tests:
print(diff)
programs:
- name: getdns
binary: getdns_query
additional:
- -C
- getdns.conf
templates:
- template/getdns.j2
- template/dnssec_getdns.j2
configs:
- getdns.conf
- getdns-root.key
programs:
- name: knotd
binary: knotd
additional:
- -c
- ./knotd.conf
templates:
- "template/knotd_master.j2"
configs:
- "knotd.conf"
programs:
- name: knotd
binary: knotd
additional:
- -c
- ./knotd.conf
templates:
- "template/knotd_slave.j2"
configs:
- "knotd.conf"
\ No newline at end of file
programs:
- name: kresd
binary: kresd
additional:
- -f
- "1"
templates:
- template/kresd.j2
configs:
- config
programs:
- name: pdns
binary: pdns_recursor
additional:
- --config-dir=./
templates:
- template/pdns_recursor.j2
- template/hints_zone.j2
- template/pdns_dnssec.j2
configs:
- recursor.conf
- hints.pdns
- dnssec.lua
# SIGTERM leads to return code -15 instead of clean 0 so we have to ignore it
ignore_exit_code: true
programs:
- name: unbound
binary: unbound
additional:
- -d
- -c
- unbound.conf
templates:
- template/unbound.j2
- template/hints_zone.j2
configs:
- unbound.conf
- hints.zone
#!/usr/bin/env python3
import argparse
from datetime import datetime
import errno
import logging
......@@ -8,14 +7,12 @@ import os
import shutil
import socket
import subprocess
import sys
import tempfile
import time
import jinja2
import yaml
from pydnstest import scenario, testserver, test
from pydnstest import scenario, testserver
# path to Deckard files
......@@ -86,18 +83,6 @@ class IfaceManager(object):
for name in self.name2iface}
def find_objects(path):
""" Recursively scan file/directory for scenarios. """
result = []
if os.path.isdir(path):
for e in os.listdir(path):
result += find_objects(os.path.join(path, e))
elif os.path.isfile(path):
if path.endswith('.rpl'):
result.append(path)
return result
def write_timestamp_file(path, tst):
time_file = open(path, 'w')
time_file.write(datetime.fromtimestamp(tst).strftime('@%Y-%m-%d %H:%M:%S'))
......@@ -287,11 +272,11 @@ def conncheck_daemon(process, cfg, sockfamily):
sock.close()
def process_file(path, args, prog_cfgs):
def process_file(path, qmin, prog_cfgs):
"""Parse scenario from a file object and create workdir."""
# Parse scenario
case, cfg_text = scenario.parse_file(os.path.realpath(path))
cfg_ctx, ta_files = scenario.parse_config(cfg_text, args.qmin, INSTALLDIR)
cfg_ctx, ta_files = scenario.parse_config(cfg_text, qmin, INSTALLDIR)
template_ctx = setup_network(cfg_ctx['_SOCKET_FAMILY'], prog_cfgs)
# merge variables from scenario with generated network variables (scenario has priority)
template_ctx.update(cfg_ctx)
......@@ -352,143 +337,10 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
with open(daemon['cfg']['log']) as logf:
for line in logf:
daemon_logger_log.debug(line.strip())
ignore_exit = bool(os.environ.get('IGNORE_EXIT_CODE', 0))
ignore_exit = daemon["cfg"].get('ignore_exit_code', False)
if daemon['proc'].returncode != 0 and not ignore_exit:
raise ValueError('process %s terminated with return code %s'
% (daemon['cfg']['name'], daemon['proc'].returncode))
# Do not clear files if the server crashed (for analysis)
if server.undefined_answers > 0:
raise ValueError('the scenario does not define all necessary answers (see error log)')
def test_platform():
if sys.platform == 'windows':
raise NotImplementedError('not supported at all on Windows')
def deckard():
"""Entrypoint for script"""
# auxilitary classes for argparse
class ColonSplitter(argparse.Action): # pylint: disable=too-few-public-methods
"""Split argument string into list holding items separated by colon."""
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, values.split(':'))
class EnvDefault(argparse.Action): # pylint: disable=too-few-public-methods
"""Get default value for parameter from environment variable."""
def __init__(self, envvar, required=True, default=None, **kwargs):
if envvar and envvar in os.environ:
default = os.environ[envvar]
if required and default is not None:
required = False
super(EnvDefault, self).__init__(default=default, required=required, **kwargs)
def __call__(self, parser, namespace, values, option_string=None):
setattr(namespace, self.dest, values)
def loglevel2number(level):
"""Convert direct log level number or symbolic name to a number."""
try:
return int(level)
except ValueError:
pass # not a number, try if it is a named constant from logging module
try:
return getattr(logging, level.upper())
except AttributeError:
raise ValueError('unknown log level %s' % level)
test_platform()
argparser = argparse.ArgumentParser()
argparser.add_argument('--qmin', help='query minimization (default: enabled)', default=True,
action=EnvDefault, envvar='QMIN', type=scenario.str2bool)
argparser.add_argument('--loglevel', help='verbosity (default: errors + test results)',
action=EnvDefault, envvar='VERBOSE',
type=loglevel2number, required=False)
argparser.add_argument('scenario', help='path to test scenario')
argparser.add_argument('--noclean', action='store_true',
help='don\'t delete working directory')
subparsers = argparser.add_subparsers(
dest='cmd', title='sub-commands',
description='run scenario with one binary specified on command line '
'or multiple binaries specified in config file')
run_one = subparsers.add_parser('one', help='run single binary inside single scenario')
run_one.add_argument('binary', help='executable to test')
run_one.add_argument('templates', help='colon-separated list of jinja2 template files',
action=ColonSplitter)
run_one.add_argument('configs',
help='colon-separated list of files to be generated from templates',
action=ColonSplitter)
run_one.add_argument('additional', help='additional parameters for the binary', nargs='*')
run_cfg = subparsers.add_parser(
'multiple',
help='run all binaries specified in YaML file; '
'all binaries will be executed inside single scenario')
run_cfg.add_argument('yaml', help='YaML specifying binaries and their parameter',
type=open)
args = argparser.parse_args()
if not args.loglevel:
# default verbosity: errors + test results
args.loglevel = logging.ERROR
logging.config.dictConfig(
{
'version': 1,
'incremental': True,
'loggers': {
'deckard.hint': {'level': 'INFO'},
'pydnstest.test.Test': {'level': 'INFO'}
}
})
if args.loglevel <= logging.DEBUG: # include message origin
logging.basicConfig(level=args.loglevel)
else:
logging.basicConfig(level=args.loglevel, format='%(message)s')
log = logging.getLogger('deckard')
if args.cmd == 'multiple':
config = yaml.load(args.yaml)
else:
assert args.cmd == 'one'
config = {
'programs': [{
'binary': args.binary,
'templates': args.templates,
'configs': args.configs,
'additional': args.additional,
'name': os.path.basename(args.binary),
}],
'noclean': args.noclean,
}
mandatory_keys = {'name', 'binary', 'templates', 'configs', 'additional'}
for cfg in config['programs']:
missing_keys = mandatory_keys - set(cfg.keys())
if missing_keys:
log.critical('Mandatory fields in configuration are missing: %s', missing_keys)
sys.exit(1)
# sanity check templates vs. configs
if len(cfg['templates']) != len(cfg['configs']):
log.critical('Number of jinja2 template files is not equal '
'to number of config files to be generated for '
'program "%s", i.e. len(templates) != len(configs)',
cfg['name'])
sys.exit(1)
# Scan for scenarios
testset = test.Test()
objects = find_objects(args.scenario)
for path in objects:
testset.add(path, process_file, args, config)
sys.exit(testset.run())
if __name__ == '__main__':
# this is done to avoid creating global variables
deckard()
......@@ -59,22 +59,6 @@ set -o errexit -o nounset
## iter_minim_a_nxdomain.rpl nsec3_wildcard_no_data_response.rpl
## iter_minim_a.rpl val_nsec3_optout_unsec_cache.rpl
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/getdns.yaml "$@"
# Path to scenario files
TESTS=${TESTS:-"sets/resolver"}
# Path to daemon
DAEMON=${DAEMON:-"getdns_query"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/getdns.j2:template/dnssec_getdns.j2"}
# Config file name
CONFIG=${CONFIG:-"getdns.conf:getdns-root.key"}
ADDITIONAL=${ADDITIONAL:-"-C getdns.conf"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
#!/bin/bash
set -o errexit -o nounset
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/knotd_master.yaml "$@"
# Path to scenario files
TESTS=${TESTS:-"sets/knotd/master"}
# Path to daemon
DAEMON=${DAEMON:-"knotd"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/knotd_master.j2"}
# Config file name
CONFIG=${CONFIG:-"knotd.conf"}
# Additional parameter for knotd
# it means configuration file can be found in working directory
ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
#!/bin/bash
set -o errexit -o nounset
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/knotd_slave.yaml "$@"
# Path to scenario files
TESTS=${TESTS:-"sets/knotd/slave"}
# Path to daemon
DAEMON=${DAEMON:-"knotd"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/knotd_slave.j2"}
# Config file name
CONFIG=${CONFIG:-"knotd.conf"}
# Additional parameter for knotd
# it means configuration file can be found in working directory
ADDITIONAL=${ADDITIONAL:-"-c ./knotd.conf"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
echo 'INFO: Tests require Knot compiled with ./configure --enable-recvmmsg=no'
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
#!/bin/bash
set -o errexit -o nounset
# Path to scenario files
TESTS=${TESTS:-"sets/resolver"}
# Path to daemon
DAEMON=${DAEMON:-"kresd"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/kresd.j2"}
# Config file name
CONFIG=${CONFIG:-"config"}
# Additional parameters for kresd - non-interactive mode
ADDITIONAL=${ADDITIONAL:-"-f 1"}
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/kresd.yaml "$@"
#!/bin/bash
set -o errexit -o nounset
# Path to scenario files
TESTS=${TESTS:-"sets/resolver"}
# Path to daemon
DAEMON=${DAEMON:-"pdns_recursor"}
# Template file name
TEMPLATE=${TEMPLATE:-"template/pdns_recursor.j2:template/hints_zone.j2:template/pdns_dnssec.j2"}
# Config file name
CONFIG=${CONFIG:-"recursor.conf:hints.pdns:dnssec.lua"}
# Additional parameter for pdns_recursor
# it means configuration file can be found in working directory
ADDITIONAL=${ADDITIONAL:-"--config-dir=./"}
# SIGTERM leads to return code -15 instead of clean 0 so we have to ignore it
IGNORE_EXIT_CODE=1
export TESTS DAEMON TEMPLATE CONFIG ADDITIONAL IGNORE_EXIT_CODE
MAKEDIR="$(dirname "$0")"
make -C "${MAKEDIR}"
RUNDIR="$(dirname "$0")"
cd $RUNDIR && ./run.sh --config configs/pdns.yaml "$@"
#!/usr/bin/env python3
import logging
class Test:
""" Small library to imitate CMocka output. """
log = logging.getLogger('pydnstest.test.Test')
def __init__(self):
self.tests = []
def add(self, name, test, args, config):
""" Add named test to set. """
self.tests.append((name, test, args, config))
def run(self):
""" Run planned tests. """
planned = len(self.tests)
passed = 0
if planned == 0:
return
for name, test_callback, args, config in self.tests:
try:
test_callback(name, args, config)
passed += 1
self.log.info('[ OK ] %s', name)
except Exception as ex:
self.log.error('[ FAIL ] %s', name)
self.log.exception(ex)
# Clear test set
self.tests = []
if passed == planned:
return 0
else:
return 1
[pytest]
log_print = False
\ No newline at end of file
......@@ -2,4 +2,5 @@ dnspython>=1.15
Jinja2>=2.8
PyYAML
python-augeas
pytest
\ No newline at end of file
pytest
pytest-xdist
MAKEDIR="$(dirname "$0")"
LDPRELOAD=$(make depend -C "${MAKEDIR}")
source ${MAKEDIR}/env.sh
py.test -q tests/test_runner.py "$@"
RETVAL=$?
if [[ $RETVAL -eq 4 && "$@" =~ .*-n.* ]]
then
echo "Running on multiple cores failed. Is pytest-xdist module installed?"
fi
exit $RETVAL
\ No newline at end of file
from collections import namedtuple
import glob
import os
import pytest
import re
import yaml
Scenario = namedtuple("Scenario", ["path", "qmin"])
def config_sanity_check(config_dict, config_name):
"""Checks if parsed configuration is valid"""
mandatory_keys = {'name', 'binary', 'templates', 'configs', 'additional'}
for cfg in config_dict['programs']:
missing_keys = mandatory_keys - set(cfg.keys())
assert not missing_keys, 'Mandatory fields in configuration are missing: %s' % missing_keys
# sanity check templates vs. configs
assert len(cfg['templates']) == len(cfg['configs']),\
('Number of jinja2 template files is not equal '
'to number of config files to be generated for '
'program "%s" (%s), i.e. len(templates) != len(configs)'
% (cfg['name'], config_name))
for additional in cfg["additional"]:
assert type(additional) is str,\
"All additional arguments in yaml should be strings. (%s, %s)"\
% (cfg['name'], config_name)
def get_qmin_config(path):
"""Reads configuration from the *.rpl file and determines query-minimization setting."""
with open(path) as f:
for line in f:
if re.search(r"^CONFIG_END", line) or re.search(r"^SCENARIO_BEGIN", line):
return None
if re.search(r"^\s*query-minimization:\s*(on|yes)", line):
return True
if re.search(r"^\s*query-minimization:\s*(off|no)", line):
return False
def scenarios(scenarios_path):
"""Returns list of *.rpl files from given path and packs them with their minimization setting"""
return [Scenario(scenario, get_qmin_config(scenario))
for scenario in sorted(glob.glob(os.path.join(scenarios_path, "*.rpl")))]
def pytest_addoption(parser):
parser.addoption("--config", action="store")
parser.addoption("--scenarios", action="store")
def pytest_generate_tests(metafunc):
"""This is pytest weirdness to parametrize the test over all the *.rpl files."""
if 'scenario' in metafunc.fixturenames:
if metafunc.config.option.scenarios is not None:
paths = metafunc.config.option.scenarios
metafunc.parametrize("scenario", scenarios(paths[0]), ids=str)
else:
# If no --config option is given, we use the default from Deckard repository
metafunc.parametrize("scenario", scenarios("sets/resolver"), ids=str)
@pytest.fixture
def config(request):
"""Parses and checks the config given"""
config_file = request.config.getoption("--config")
configuration = yaml.safe_load(open(config_file))
config_sanity_check(configuration, config_file)
return configuration
"""Usage: run `LD_PRELOAD="«paths to libfaketime and libsocket_wrapper»"
py.test tests/test_experiment` in deckard's root directory.
Alternatively add `-n «number of cores»` with `pytest-xdist` module installed.
"""
import logging
import os
import pytest