Commit 93843c37 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'augeas_merge' into 'master'

Move to Augeas as replacement for our hand-made parser

See merge request !54
parents dc16b197 b311113b
Pipeline #6000 passed with stage
in 1 minute and 21 seconds
......@@ -5,11 +5,18 @@ WORKDIR /root
CMD ["/bin/bash"]
# knot-resolver used for comparative tests
# we do not care that much about particular version
# we do not care that much about particular version of resolvers
# we want to install newer version of Augeas because those from Ubuntu repositories are incredibly slow on some operations
# context: https://www.redhat.com/archives/augeas-devel/2017-June/msg00000.html
RUN LC_ALL=C.UTF-8 add-apt-repository ppa:raphink/augeas -y
RUN apt-get update
RUN apt-get upgrade -y
RUN apt-get install -y knot-resolver
RUN apt-get install -y unbound
RUN apt-get install -y pdns-recursor
RUN apt-get install -y python-pep8 pylint python-dnspython python-jinja2 python-yaml
RUN apt-get install -y python3-pep8 pylint3 python3-dnspython python3-jinja2 python3-yaml
RUN apt-get install -y python-augeas python-pep8 pylint python-pip python-jinja2 python-yaml
RUN apt-get install -y python3-augeas python3-pep8 pylint3 python3-pip python3-jinja2 python3-yaml
# version of dnspython in Ubuntu repository is f**ked up
RUN pip install --upgrade dnspython
RUN pip3 install --upgrade dnspython
#!/usr/bin/env python
import argparse
from datetime import datetime
import fileinput
import logging
import logging.config
import os
......@@ -250,7 +249,7 @@ def conncheck_daemon(process, cfg, sockfamily):
raise subprocess.CalledProcessError(process.returncode, cfg['args'], msg)
try:
sock.connect((cfg['ipaddr'], 53))
except:
except socket.error:
continue
break
sock.close()
......@@ -261,7 +260,7 @@ def play_object(path, args, prog_cfgs):
daemon_logger_log = logging.getLogger('deckard.daemon.log')
# Parse scenario
case, cfg_text = scenario.parse_file(fileinput.input(path))
case, cfg_text = scenario.parse_file(os.path.realpath(path))
cfg_ctx = scenario.parse_config(cfg_text, args.qmin, INSTALLDIR)
# get working directory and environment variables
......
#!/usr/bin/python3
# Copyright (C) 2017
import posixpath
import logging
import os
import collections
import sys
from augeas import Augeas
AUGEAS_LOAD_PATH = '/augeas/load/'
AUGEAS_FILES_PATH = '/files/'
AUGEAS_ERROR_PATH = '//error'
log = logging.getLogger('augeas')
def join(*paths):
"""
join two Augeas tree paths
FIXME: Beware: // is normalized to /
"""
norm_paths = [posixpath.normpath(path) for path in paths]
# first path must be absolute
assert norm_paths[0][0] == '/'
new_paths = [norm_paths[0]]
# relativize all other paths so join works as expected
for path in norm_paths[1:]:
if path.startswith('/'):
path = path[1:]
new_paths.append(path)
new_path = posixpath.join(*new_paths)
log.debug("join: new_path %s", new_path)
return posixpath.normpath(new_path)
class AugeasWrapper(object):
"""python-augeas higher-level wrapper.
Load single augeas lens and configuration file.
Exposes configuration file as AugeasNode object with dict-like interface.
AugeasWrapper can be used in with statement in the same way as file does.
"""
def __init__(self, confpath, lens, root=None, loadpath=None,
flags=Augeas.NO_MODL_AUTOLOAD | Augeas.NO_LOAD | Augeas.ENABLE_SPAN):
"""Parse configuration file using given lens.
Params:
confpath (str): Absolute path to the configuration file
lens (str): Name of module containing Augeas lens
root: passed down to original Augeas
flags: passed down to original Augeas
loadpath: passed down to original Augeas
flags: passed down to original Augeas
"""
log.debug('loadpath: %s', loadpath)
log.debug('confpath: %s', confpath)
self._aug = Augeas(root=root, loadpath=loadpath, flags=flags)
# /augeas/load/{lens}
aug_load_path = join(AUGEAS_LOAD_PATH, lens)
# /augeas/load/{lens}/lens = {lens}.lns
self._aug.set(join(aug_load_path, 'lens'), '%s.lns' % lens)
# /augeas/load/{lens}/incl[0] = {confpath}
self._aug.set(join(aug_load_path, 'incl[0]'), confpath)
self._aug.load()
errors = self._aug.match(AUGEAS_ERROR_PATH)
if errors:
err_msg = '\n'.join(
["{}: {}".format(e, self._aug.get(e)) for e in errors]
)
raise RuntimeError(err_msg)
path = join(AUGEAS_FILES_PATH, confpath)
paths = self._aug.match(path)
if len(paths) != 1:
raise ValueError('path %s did not match exactly once' % path)
self.tree = AugeasNode(self._aug, path)
self._loaded = True
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.save()
self.close()
def save(self):
"""Save Augeas tree to its original file."""
assert self._loaded
try:
self._aug.save()
except IOError as exc:
log.exception(exc)
for err_path in self._aug.match('//error'):
log.error('%s: %s', err_path,
self._aug.get(os.path.join(err_path, 'message')))
raise
def close(self):
"""
close Augeas library
After calling close() the object must not be used anymore.
"""
assert self._loaded
self._aug.close()
del self._aug
self._loaded = False
def match(self, path):
"""Yield AugeasNodes matching given expression."""
assert self._loaded
assert path
log.debug('tree match %s', path)
for matched_path in self._aug.match(path):
yield AugeasNode(self._aug, matched_path)
class AugeasNode(collections.MutableMapping):
"""One Augeas tree node with dict-like interface."""
def __init__(self, aug, path):
"""
Args:
aug (AugeasWrapper or Augeas): Augeas library instance
path (str): absolute path in Augeas tree matching single node
BEWARE: There are no sanity checks of given path for performance reasons.
"""
assert aug
assert path
assert path.startswith('/')
self._aug = aug
self._path = path
self._span = None
@property
def path(self):
"""canonical path in Augeas tree, read-only"""
return self._path
@property
def value(self):
"""
get value of this node in Augeas tree
"""
value = self._aug.get(self._path)
log.debug('tree get: %s = %s', self._path, value)
return value
@property
def span(self):
if self._span is None:
self._span = "char position %s" % self._aug.span(self._path)[5]
return self._span
@value.setter
def value(self, value):
"""
set value of this node in Augeas tree
"""
log.debug('tree set: %s = %s', self._path, value)
self._aug.set(self._path, value)
def __len__(self):
"""
number of items matching this path
It is always 1 after __init__() but it may change
as Augeas tree changes.
"""
return len(self._aug.match(self._path))
def __getitem__(self, key):
if isinstance(key, int):
# int is a shortcut to write [int]
target_path = '%s[%s]' % (self._path, key)
else:
target_path = self._path + key
log.debug('tree getitem: target_path %s', target_path)
paths = self._aug.match(target_path)
if len(paths) != 1:
raise KeyError('path %s did not match exactly once' % target_path)
return AugeasNode(self._aug, target_path)
def __delitem__(self, key):
log.debug('tree delitem: %s + %s', self._path, key)
target_path = self._path + key
log.debug('tree delitem: target_path %s', target_path)
self._aug.remove(target_path)
def __setitem__(self, key, value):
assert isinstance(value, AugeasNode)
target_path = self.path + key
self._aug.copy(value.path, target_path)
def __iter__(self):
self_path_len = len(self._path)
assert self_path_len > 0
log.debug('tree iter: %s', self._path)
for new_path in self._aug.match(self._path):
if len(new_path) == self_path_len:
yield ''
else:
yield new_path[self_path_len - 1:]
def match(self, subpath):
"""Yield AugeasNodes matching given sub-expression."""
assert subpath.startswith("/")
match_path = "%s%s" % (self._path, subpath)
log.debug('tree match %s: %s', match_path, self._path)
for matched_path in self._aug.match(match_path):
yield AugeasNode(self._aug, matched_path)
def __repr__(self):
return 'AugeasNode(%s)' % self._path
module Deckard =
autoload xfm
let del_str = Util.del_str
let space = del /[ \t]+/ " "
let tab = del /[ \t]+/ "\t"
let ws = del /[\t ]*/ ""
let word = /[^\t\n\/#; ]+/
let comment = del /[;#]/ ";" . [label "comment" . store /[^\n]+/]
let eol = del /([ \t]*([;#][^\n]*)?\n)+/ "\n" . Util.indent
let comment_or_eol = ws . comment? . del_str "\n" . del /([ \t]*([;#][^\n]*)?\n)*/ "\n" . Util.indent
(*let comment_or_eol = [ label "#comment" . counter "comment" . (ws . [del /[;#]/ ";" . label "" . store /[^\n]*/ ]? . del_str "\n")]+ . Util.indent
*)
let domain_re = (/[^.\t\n\/#; ]+(\.[^.\t\n\/#; ]+)*\.?/ | ".") - "SECTION" (*quick n dirty, sorry to whoever will ever own SECTION TLD*)
let class_re = /CLASS[0-9]+/ | "IN" | "CH" | "HS" | "NONE" | "ANY"
let domain = [ label "domain" . store domain_re ]
let ttl = [label "ttl" . store /[0-9]+/]
let class = [label "class" . store class_re ]
let type = [label "type" . store ((/[^0-9#;\n \t][^\t\n\/#; ]*/) - class_re) ]
let data_re = /([^ \t\n#;][^\n#;]*[^ \t\n#;])|[^ \t\n#;]/ (*Can not start nor end with whitespace but can have whitespace in the middle. Disjunction is there so we match strings of length one.*)
let data = [label "data" . store data_re ]
let ip_re = /[0-9a-f.:]+/
let hex_re = /[0-9a-fA-F]+/
let match_option = "opcode" | "qtype" | "qcase" | "qname" | "subdomain" | "flags" | "rcode" | "question" | "answer" | "authority" | "additional" | "all" | "TCP" | "ttl"
let adjust_option = "copy_id" | "copy_query"
let reply_option = "QR" | "TC" | "AA" | "AD" | "RD" | "RA" | "CD" | "DO" | "NOERROR" | "FORMERR" | "SERVFAIL" | "NXDOMAIN" | "NOTIMP" | "REFUSED" | "YXDOMAIN" | "YXRRSET" | "NXRRSET" | "NOTAUTH" | "NOTZONE" | "BADVERS"
let step_option = "REPLY" | "QUERY" | "CHECK_ANSWER" | "CHECK_OUT_QUERY" | /TIME_PASSES[ \t]+ELAPSE/
let mandatory = [del_str "MANDATORY" . label "mandatory" . value "true" . comment_or_eol]
let tsig = [del_str "TSIG" . label "tsig" . space . [label "keyname" . store word] . space . [label "secret" . store word] . comment_or_eol]
let match = (mandatory | tsig)* . del_str "MATCH" . [space . label "match" . store match_option ]+ . comment_or_eol
let adjust = (mandatory | tsig)* . del_str "ADJUST" . [space . label "adjust" . store adjust_option ]+ . comment_or_eol
let reply = (mandatory | tsig)* . del ("REPLY" | "FLAGS") "REPLY" . [space . label "reply" . store reply_option ]+ . comment_or_eol
let question = [label "record" . domain . tab . (class . tab)? . type . comment_or_eol ]
let record = [label "record" . domain . tab . (ttl . tab)? . (class . tab)? . type . tab . data . comment_or_eol]
let section_question = [ label "question" . del_str "SECTION QUESTION" .
comment_or_eol . question? ]
let section_answer = [ label "answer" . del_str "SECTION ANSWER" .
comment_or_eol . record* ]
let section_authority = [ label "authority" . del_str "SECTION AUTHORITY" .
comment_or_eol . record* ]
let section_additional = [ label "additional" . del_str "SECTION ADDITIONAL" .
comment_or_eol . record* ]
let sections = [label "section" . section_question? . section_answer? . section_authority? . section_additional?]
let raw = [del_str "RAW" . comment_or_eol . label "raw" . store hex_re ] . comment_or_eol
(* This is quite dirty hack to match every combination of options given to entry since 'let normal = ((match | adjust | reply | mandatory | tsig)* . sections)' just is not possible *)
let normal = (match . (adjust . reply? | reply . adjust?)? | adjust . (match . reply? | reply . match?)? | reply . (match . adjust? | adjust . match?)?)? . (mandatory | tsig)* . sections
let entry = [label "entry" . del_str "ENTRY_BEGIN" . comment_or_eol . ( normal | raw ) . del_str "ENTRY_END" . eol]
let single_address = [ label "address" . space . store ip_re ]
let addresses = [label "address" . counter "address" . [seq "address" . del_str "ADDRESS" . space . store ip_re . comment_or_eol]+]
let range = [label "range" . del_str "RANGE_BEGIN" . space . [ label "from" . store /[0-9]+/] . space .
[ label "to" . store /[0-9]+/] . single_address? . comment_or_eol . addresses? . entry* . del_str "RANGE_END" . eol]
let step = [label "step" . del_str "STEP" . space . store /[0-9]+/ . space . [label "type" . store step_option] . [space . label "timestamp" . store /[0-9]+/]? . comment_or_eol .
entry? ]
let config_record = /[^\n]*/ - ("CONFIG_END" | /STEP.*/ | /SCENARIO.*/ | /RANGE.*/ | /ENTRY.*/)
let config = [ label "config" . counter "config" . [seq "config" . store config_record . del_str "\n"]* . del_str "CONFIG_END" . comment_or_eol ]
let guts = (step | range )*
let scenario = [label "scenario" . del_str "SCENARIO_BEGIN" . space . store data_re . comment_or_eol . guts . del_str "SCENARIO_END" . eol]
let lns = config? . scenario
(* TODO: REPLAY step *)
(* TODO: store all comments into the tree instead of ignoring them *)
(*let filter = incl "/home/test/*.rpl"*)
let filter = incl "/media/test/27159fa1-67d4-4162-8707-cd67900f3b36/stepan/nic/deckard_stable/deckard/sets/resolver/*.rpl"
let xfm = transform lns filter
stub-addr: 127.0.0.10
CONFIG_END
SCENARIO_BEGIN empty replies
RANGE_BEGIN 0 100
ADDRESS 127.0.0.10
ENTRY_BEGIN
MATCH subdomain
ADJUST copy_id copy_query
SECTION QUESTION
. IN A
ENTRY_END
RANGE_END
STEP 1 QUERY
ENTRY_BEGIN
ENTRY_END
SCENARIO_END
This diff is collapsed.
......@@ -25,9 +25,9 @@ class Test:
test_callback(name, args, config)
passed += 1
self.log.info('[ OK ] %s', name)
except Exception as e:
except Exception as ex:
self.log.error('[ FAIL ] %s', name)
self.log.exception(e)
self.log.exception(ex)
# Clear test set
self.tests = []
......
......@@ -26,6 +26,8 @@ class TestServer:
self.client_socks = []
self.connections = []
self.active = False
self.active_lock = threading.Lock()
self.condition = threading.Condition()
self.scenario = test_scenario
self.addr_map = []
self.start_iface = 2
......@@ -35,21 +37,26 @@ class TestServer:
def __del__(self):
""" Cleanup after deletion. """
if self.active is True:
with self.active_lock:
active = self.active
if active:
self.stop()
def start(self, port=53):
""" Synchronous start """
if self.active is True:
raise Exception('TestServer already started')
self.active = True
with self.active_lock:
if self.active:
raise Exception('TestServer already started')
with self.active_lock:
self.active = True
self.addr, _ = self.start_srv((self.kroot_local, port), self.addr_family)
self.start_srv(self.addr, self.addr_family, socket.IPPROTO_TCP)
self._bind_sockets()
def stop(self):
""" Stop socket server operation. """
self.active = False
with self.active_lock:
self.active = False
if self.thread:
self.thread.join()
for conn in self.connections:
......@@ -86,7 +93,7 @@ class TestServer:
log.debug('server %s received query from %s: %s', server_addr, client_addr, query)
response, is_raw_data = self.scenario.reply(query, server_addr)
if response:
if is_raw_data is False:
if not is_raw_data:
data_to_wire = response.to_wire(max_size=65535)
log.debug('response: %s', response)
else:
......@@ -108,9 +115,15 @@ class TestServer:
def query_io(self):
""" Main server process """
self.undefined_answers = 0
if self.active is False:
raise Exception("[query_io] Test server not active")
while self.active is True:
with self.active_lock:
if not self.active:
raise Exception("[query_io] Test server not active")
while True:
with self.condition:
self.condition.notify()
with self.active_lock:
if not self.active:
break
objects = self.srv_socks + self.connections
to_read, _, to_error = select.select(objects, [], objects, 0.1)
for sock in to_read:
......@@ -154,6 +167,8 @@ class TestServer:
if self.thread is None:
self.thread = threading.Thread(target=self.query_io)
self.thread.start()
with self.condition:
self.condition.wait()
for srv_sock in self.srv_socks:
if (srv_sock.family == family
......@@ -203,25 +218,10 @@ def empty_test_case():
Return (scenario, config) pair which answers to any query on 127.0.0.10.
"""
# Mirror server
entry = scenario.Entry()
entry.set_match([]) # match everything
entry.set_adjust(['copy_id', 'copy_query'])
rng = scenario.Range(0, 100)
rng.add(entry)
rng.addresses.add('127.0.0.10')
step = scenario.Step(1, 'QUERY', [])
test_scenario = scenario.Scenario('empty replies')
test_scenario.ranges.append(rng)
test_scenario.steps.append(step)
test_scenario.current_step = step
empty_test_path = os.path.dirname(os.path.realpath(__file__)) + "/empty.rpl"
test_config = {'ROOT_ADDR': '127.0.0.10',
'_SOCKET_FAMILY': socket.AF_INET}
return (test_scenario, test_config)
return scenario.parse_file(empty_test_path)[0], test_config
def standalone_self_test():
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment