Commit ea667265 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'multiple_resolvers' into 'master'

support multiple resolvers inside single Deckard instance

Closes #6

See merge request !51
parents 43061fa2 d15776c1
Pipeline #3862 passed with stage
in 1 minute and 1 second
......@@ -121,3 +121,36 @@ test:python3:latest:kresd:
- docker
- linux
- amd64
# sanity check that Unbound under Deckard still works
# I've selected the only tests which are working
# on kresd and Unbound 1.5.8 as well as 1.6.0
test:python3:sanity:unbound:
image: cznic/deckard-ci
stage: test
script:
- export PYTHON=python3
- TESTS=sets/resolver/iter_hint_lame.rpl ./unbound_run.sh
- TESTS=sets/resolver/iter_lame_root.rpl ./unbound_run.sh
# these do not work with Unbound 1.5.8 which is in CI container
#- TESTS=sets/resolver/nsec_wildcard_answer_response.rpl ./unbound_run.sh
#- TESTS=sets/resolver/world_cz_lidovky_www.rpl ./unbound_run.sh
tags:
- docker
- linux
- amd64
# sanity check that PowerDNS recursor under Deckard still works
# I've selected couple tests which are working
# on kresd and PowerDNS recursor 4.0.0~alpha2 as well as 4.0.4
test:python3:sanity:pdnsrecursor:
image: cznic/deckard-ci
stage: test
script:
- export PYTHON=python3
- TESTS=sets/resolver/iter_recurse.rpl ./pdns_run.sh
- TESTS=sets/resolver/iter_tcbit.rpl ./pdns_run.sh
tags:
- docker
- linux
- amd64
......@@ -45,7 +45,7 @@ depend: $(libfaketime) $(libcwrap)
# Generic rule to run test
$(SOURCES): depend
%.out: %.rpl
@$(preload_syms) $(PYTHON) $(abspath ./deckard.py) $< $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
@$(preload_syms) $(PYTHON) $(abspath ./deckard.py) $< one $(DAEMON) $(TEMPLATE) $(CONFIG) -- $(ADDITIONAL)
# Synchronize submodules
submodules: .gitmodules
......
This diff is collapsed.
This diff is collapsed.
......@@ -9,5 +9,7 @@ CMD ["/bin/bash"]
RUN apt-get update
RUN apt-get upgrade -y
RUN apt-get install -y knot-resolver
RUN apt-get install -y python-pep8 pylint python-dnspython python-jinja2
RUN apt-get install -y python3-pep8 pylint3 python3-dnspython python3-jinja2
RUN apt-get install -y unbound
RUN apt-get install -y pdns-recursor
RUN apt-get install -y python-pep8 pylint python-dnspython python-jinja2 python-yaml
RUN apt-get install -y python3-pep8 pylint3 python3-dnspython python3-jinja2 python3-yaml
......@@ -10,7 +10,7 @@ PYFILES=$(find . \
-type f -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print)
: check if version under test does not produce critical errors
${PYTHON} -m pylint -E ${PYFILES}
${PYTHON} -m pylint -j $(nproc) -E ${PYFILES}
: no critical errors, compare score between versions
rm -rf ~/.pylint.d
......@@ -22,7 +22,7 @@ PYFILES=$(find . \
-type d -exec test -e '{}/__init__.py' \; -print -prune -o \
-name '*.py' -print -o \
-type f -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print)
${PYTHON} -m pylint ${PYFILES} &> /tmp/base.log || : old version is not clear
${PYTHON} -m pylint -j $(nproc) ${PYFILES} &> /tmp/base.log || : old version is not clear
LOGS[0]="/tmp/base.log"
echo ==================== merge base ====================
cat /tmp/base.log
......@@ -31,7 +31,7 @@ echo ==================== merge base end ====================
: get test results from version under test
git checkout --force --detach "${HEAD}"
git clean -xdf
${PYTHON} -m pylint ${PYFILES} &> /tmp/head.log || : version under test is not clear
${PYTHON} -m pylint -j $(nproc) ${PYFILES} &> /tmp/head.log || : version under test is not clear
LOGS[1]="/tmp/head.log"
echo ==================== candidate version ====================
cat /tmp/head.log
......
This diff is collapsed.
Notes for Deckard developers
============================
socket wrapper library (cwrap)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Detailed instructions on using cwrap you can be found here_
cwrap environment is managed by Deckard. Default values are sufficient, do not touch the environment unless you are trying to debug something. Variables available for direct use are:
- ``SOCKET_WRAPPER_DIR`` is a generic working directory. It defaults
to a new temporary directory with randomly generated name,
prefixed by ``tmpdeckard``. When a test fails, the work directory can contain useful
information for post-mortem analysis. You can explicitly set ``SOCKET_WRAPPER_DIR``
to a custom path for more convenient analysis.
- ``SOCKET_WRAPPER_DEBUGLEVEL`` is not set by default.
Deckard automatically sets ``SOCKET_WRAPPER_PCAP_FILE`` to create separate PCAP files in working directory for Deckard itself and each daemon. Feel free to inspect them.
.. _here: https://git.samba.org/?p=socket_wrapper.git;a=blob;f=doc/socket_wrapper.1.txt;hb=HEAD
libfaketime
^^^^^^^^^^^
Run-time changes to ``FAKETIME_`` environment variables might not be picked up by running process if ``FAKETIME_NO_CACHE=1`` variable is not set before the process starts.
This diff is collapsed.
This diff is collapsed.
from __future__ import absolute_import
import logging
import dns.message
import dns.rrset
import dns.rcode
import dns.dnssec
import dns.tsigkeyring
import binascii
import socket
import struct
import os
import sys
import calendar
from datetime import datetime
import errno
import itertools
import logging
import os
import random
import socket
import string
import struct
import time
from datetime import datetime
import dns.dnssec
import dns.message
import dns.rcode
import dns.rrset
import dns.tsigkeyring
def str2bool(v):
""" Return conversion of JSON-ish string value to boolean. """
return v.lower() in ('yes', 'true', 'on', '1')
# Global statistics
......@@ -147,7 +152,6 @@ def replay_rrs(rrs, nqueries, destination, args=[]):
msg.want_dnssec(True)
queries.append(msg.to_wire())
# Make a UDP connected socket to the destination
tstart = datetime.now()
family = socket.AF_INET6 if ':' in destination[0] else socket.AF_INET
sock = socket.socket(family, socket.SOCK_DGRAM)
sock.connect(destination)
......@@ -277,7 +281,7 @@ class Entry:
match_fields += ['flags'] + ['rcode'] + self.sections
for code in match_fields:
try:
res = self.match_part(code, msg)
self.match_part(code, msg)
except Exception as e:
errstr = '%s in the response:\n%s' % (str(e), msg.to_text())
raise Exception("line %d, \"%s\": %s" % (self.lineno, code, errstr))
......@@ -362,9 +366,9 @@ class Entry:
opts.append(dns.edns.GenericOption(dns.edns.NSID, '' if v is True else v))
if k.lower() == 'subnet':
net = v.split('/')
family = socket.AF_INET6 if ':' in net[0] else socket.AF_INET
subnet_addr = net[0]
addr = socket.inet_pton(family, net[0])
family = socket.AF_INET6 if ':' in subnet_addr else socket.AF_INET
addr = socket.inet_pton(family, subnet_addr)
prefix = len(addr) * 8
if len(net) > 1:
prefix = int(net[1])
......@@ -477,7 +481,7 @@ class Range:
self.sent += 1
candidate.fired += 1
return resp
except Exception as e:
except Exception:
pass
return None
......@@ -555,7 +559,7 @@ class Step:
return self.__check_answer(ctx)
elif self.type == 'TIME_PASSES':
self.log.info('')
return self.__time_passes(ctx)
return self.__time_passes()
elif self.type == 'REPLY' or self.type == 'MOCK':
self.log.info('')
elif self.type == 'LOG':
......@@ -583,7 +587,7 @@ class Step:
self.log.debug("answer: %s", ctx.last_answer.to_text())
expected.match(ctx.last_answer)
def __replay(self, ctx, chunksize=8):
def __replay(self, ctx):
nqueries = len(self.queries)
if len(self.args) > 0 and self.args[0].isdigit():
nqueries = int(self.args.pop(0))
......@@ -669,7 +673,7 @@ class Step:
self.answer = None
ctx.last_answer = self.answer
def __time_passes(self, ctx):
def __time_passes(self):
""" Modify system time. """
time_file = open(os.environ["FAKETIME_TIMESTAMP_FILE"], 'r')
line = time_file.readline().strip()
......@@ -708,7 +712,6 @@ class Scenario:
self.steps = []
self.current_step = None
self.client = {}
self.sockfamily = socket.AF_INET
def reply(self, query, address=None):
"""
......@@ -918,6 +921,99 @@ def parse_scenario(op, args, file_in):
return out
def parse_config(scn_cfg, qmin, installdir):
"""
Transform scene config (key, value) pairs into dict filled with defaults.
"""
# defaults
do_not_query_localhost = True
harden_glue = True
sockfamily = 0 # auto-select value for socket.getaddrinfo
trust_anchor_list = []
stub_addr = None
override_timestamp = None
features = {}
feature_list_delimiter = ';'
feature_pair_delimiter = '='
for k, v in scn_cfg:
# Enable selectively for some tests
if k == 'do-not-query-localhost':
do_not_query_localhost = str2bool(v)
if k == 'harden-glue':
harden_glue = str2bool(v)
if k == 'query-minimization':
qmin = str2bool(v)
elif k == 'trust-anchor':
trust_anchor_list.append(v.strip('"\''))
elif k == 'val-override-timestamp':
override_timestamp_str = v.strip('"\'')
override_timestamp = int(override_timestamp_str)
elif k == 'val-override-date':
override_date_str = v.strip('"\'')
ovr_yr = override_date_str[0:4]
ovr_mnt = override_date_str[4:6]
ovr_day = override_date_str[6:8]
ovr_hr = override_date_str[8:10]
ovr_min = override_date_str[10:12]
ovr_sec = override_date_str[12:]
override_date_str_arg = '{0} {1} {2} {3} {4} {5}'.format(
ovr_yr, ovr_mnt, ovr_day, ovr_hr, ovr_min, ovr_sec)
override_date = time.strptime(override_date_str_arg, "%Y %m %d %H %M %S")
override_timestamp = calendar.timegm(override_date)
elif k == 'stub-addr':
stub_addr = v.strip('"\'')
elif k == 'features':
feature_list = v.split(feature_list_delimiter)
try:
for f_item in feature_list:
if f_item.find(feature_pair_delimiter) != -1:
f_key, f_value = [x.strip()
for x
in f_item.split(feature_pair_delimiter, 1)]
else:
f_key = f_item.strip()
f_value = ""
features[f_key] = f_value
except Exception as e:
raise Exception("can't parse features (%s) in config section (%s)" % (v, str(e)))
elif k == 'feature-list':
try:
f_key, f_value = [x.strip() for x in v.split(feature_pair_delimiter, 1)]
if f_key not in features:
features[f_key] = []
f_value = f_value.replace("{{INSTALL_DIR}}", installdir)
features[f_key].append(f_value)
except Exception as e:
raise Exception("can't parse feature-list (%s) in config section (%s)"
% (v, str(e)))
elif k == 'force-ipv6' and v.upper() == 'TRUE':
sockfamily = socket.AF_INET6
ctx = {
"DO_NOT_QUERY_LOCALHOST": str(do_not_query_localhost).lower(),
"FEATURES": features,
"HARDEN_GLUE": str(harden_glue).lower(),
"INSTALL_DIR": installdir,
"QMIN": str(qmin).lower(),
"TRUST_ANCHORS": trust_anchor_list,
}
if stub_addr:
ctx['ROOT_ADDR'] = stub_addr
# determine and verify socket family for specified root address
gai = socket.getaddrinfo(stub_addr, 53, sockfamily, 0,
socket.IPPROTO_UDP, socket.AI_NUMERICHOST)
assert len(gai) == 1
sockfamily = gai[0][0]
if not sockfamily:
sockfamily = socket.AF_INET # default to IPv4
ctx['_SOCKET_FAMILY'] = sockfamily
if override_timestamp:
ctx['_OVERRIDE_TIMESTAMP'] = override_timestamp
return ctx
def parse_file(file_in):
""" Parse scenario from a file. """
try:
......
#!/usr/bin/env python
import logging
import os
import time
class Test:
......@@ -11,9 +9,9 @@ class Test:
def __init__(self):
self.tests = []
def add(self, name, test, args):
def add(self, name, test, args, config):
""" Add named test to set. """
self.tests.append((name, test, args))
self.tests.append((name, test, args, config))
def run(self):
""" Run planned tests. """
......@@ -22,9 +20,9 @@ class Test:
if planned == 0:
return
for name, test_callback, args in self.tests:
for name, test_callback, args, config in self.tests:
try:
test_callback(name, args)
test_callback(name, args, config)
passed += 1
self.log.info('[ OK ] %s', name)
except Exception as e:
......
......@@ -2,43 +2,24 @@ from __future__ import absolute_import
import argparse
import fileinput
import itertools
import logging
import threading
import os
import select
import socket
import os
import threading
import time
import dns.message
import dns.rdatatype
import itertools
from pydnstest import scenario
def get_local_addr_str(family, iface):
""" Returns pattern string for localhost address """
if family == socket.AF_INET:
addr_local_pattern = "127.0.0.{}"
elif family == socket.AF_INET6:
addr_local_pattern = "fd00::5357:5f{:02X}"
else:
raise NotImplementedError("[get_local_addr_str] family not supported '%i'" % family)
return addr_local_pattern.format(iface)
class AddrMapInfo:
""" Saves mapping info between adresses from rpl and cwrap adresses """
def __init__(self, family, local, external):
self.family = family
self.local = local
self.external = external
class TestServer:
""" This simulates UDP DNS server returning scripted or mirror DNS responses. """
def __init__(self, test_scenario, config, d_iface):
def __init__(self, test_scenario, root_addr, addr_family):
""" Initialize server instance. """
self.thread = None
self.srv_socks = []
......@@ -46,14 +27,11 @@ class TestServer:
self.connections = []
self.active = False
self.scenario = test_scenario
self.config = config
self.addr_map = []
self.start_iface = 2
self.cur_iface = self.start_iface
self.kroot_local = None
self.addr_family = None
self.default_iface = d_iface
self.set_initial_address()
self.kroot_local = root_addr
self.addr_family = addr_family
def __del__(self):
""" Cleanup after deletion. """
......@@ -85,38 +63,6 @@ class TestServer:
self.connections = []
self.scenario = None
def check_family(self, addr, family):
""" Determines if address matches family """
test_addr = None
try:
n = socket.inet_pton(family, addr)
test_addr = socket.inet_ntop(family, n)
except socket.error:
return False
return True
def set_initial_address(self):
""" Set address for starting thread """
if self.config is None:
self.addr_family = socket.AF_INET
self.kroot_local = get_local_addr_str(self.addr_family, self.default_iface)
return
# Default address is localhost
kroot_addr = None
for k, v in self.config:
if k == 'stub-addr':
kroot_addr = v
if kroot_addr is not None:
if self.check_family(kroot_addr, socket.AF_INET):
self.addr_family = socket.AF_INET
self.kroot_local = kroot_addr
elif self.check_family(kroot_addr, socket.AF_INET6):
self.addr_family = socket.AF_INET6
self.kroot_local = kroot_addr
else:
self.addr_family = socket.AF_INET
self.kroot_local = get_local_addr_str(self.addr_family, self.default_iface)
def address(self):
""" Returns opened sockets list """
addrlist = []
......@@ -170,7 +116,7 @@ class TestServer:
for sock in to_read:
if sock in self.srv_socks:
if sock.proto == socket.IPPROTO_TCP:
conn, addr = sock.accept()
conn, _ = sock.accept()
self.connections.append(conn)
else:
self.handle_query(sock)
......@@ -184,20 +130,18 @@ class TestServer:
for sock in to_error:
raise Exception("[query_io] Socket IO error {}, exit".format(sock.getsockname()))
def start_srv(self, address=None, family=socket.AF_INET, proto=socket.IPPROTO_UDP):
def start_srv(self, address, family, proto=socket.IPPROTO_UDP):
""" Starts listening thread if necessary """
assert address
assert address[0] # host
assert address[1] # port
assert family
assert proto
if family == socket.AF_INET:
if address[0] is None:
address = (get_local_addr_str(family, self.default_iface), 53)
elif family == socket.AF_INET6:
if socket.has_ipv6 is not True:
if family == socket.AF_INET6:
if not socket.has_ipv6:
raise NotImplementedError("[start_srv] IPv6 is not supported by socket {0}"
.format(socket))
if address[0] is None:
address = (get_local_addr_str(family, self.default_iface), 53)
else:
elif family != socket.AF_INET:
raise NotImplementedError("[start_srv] unsupported protocol family {0}".format(family))
if proto == socket.IPPROTO_TCP:
......@@ -251,8 +195,7 @@ class TestServer:
self.start_srv((rd.address, 53), socket.AF_INET6)
def play(self, subject_addr):
paddr = get_local_addr_str(self.scenario.sockfamily, subject_addr)
self.scenario.play({'': (paddr, 53)})
self.scenario.play({'': (subject_addr, 53)})
def empty_test_case():
......@@ -275,7 +218,8 @@ def empty_test_case():
test_scenario.steps.append(step)
test_scenario.current_step = step
test_config = [('stub-addr', '127.0.0.10')]
test_config = {'ROOT_ADDR': '127.0.0.10',
'_SOCKET_FAMILY': socket.AF_INET}
return (test_scenario, test_config)
......@@ -295,7 +239,8 @@ def standalone_self_test():
required=False, type=int)
args = argparser.parse_args()
if args.scenario:
test_scenario, test_config = scenario.parse_file(fileinput.input(args.scenario))
test_scenario, test_config_text = scenario.parse_file(fileinput.input(args.scenario))
test_config = scenario.parse_config(test_config_text, True, os.getcwd())
else:
test_scenario, test_config = empty_test_case()
......@@ -308,15 +253,7 @@ def standalone_self_test():
else:
test_scenario.current_step = test_scenario.steps[0]
DEFAULT_IFACE = 0
CHILD_IFACE = 0
if "SOCKET_WRAPPER_DEFAULT_IFACE" in os.environ:
DEFAULT_IFACE = int(os.environ["SOCKET_WRAPPER_DEFAULT_IFACE"])
if DEFAULT_IFACE < 2 or DEFAULT_IFACE > 254:
DEFAULT_IFACE = 10
os.environ["SOCKET_WRAPPER_DEFAULT_IFACE"] = "{}".format(DEFAULT_IFACE)
server = TestServer(test_scenario, test_config, DEFAULT_IFACE)
server = TestServer(test_scenario, test_config['ROOT_ADDR'], test_config['_SOCKET_FAMILY'])
server.start()
logging.info("[==========] Mirror server running at %s", server.address())
......
......@@ -19,7 +19,8 @@ kwargs = {
'packages': ['pydnstest'],
'install_requires': [
'dnspython',
'jinja2'
'jinja2',
'PyYAML'
],
'classifiers': [
"Intended Audience :: Developers",
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment