Commit 7a86af1f authored by Štěpán Balážik's avatar Štěpán Balážik Committed by Štěpán Balážik

Replace the old parser with a new one using Augeas lens

parent d37abd2e
......@@ -11,5 +11,5 @@ RUN apt-get upgrade -y
RUN apt-get install -y knot-resolver
RUN apt-get install -y unbound
RUN apt-get install -y pdns-recursor
RUN apt-get install -y python-pep8 pylint python-dnspython python-jinja2 python-yaml
RUN apt-get install -y python3-pep8 pylint3 python3-dnspython python3-jinja2 python3-yaml
RUN apt-get install -y python-augeas python-pep8 pylint python-dnspython python-jinja2 python-yaml
RUN apt-get install -y python3-augeas python3-pep8 pylint3 python3-dnspython python3-jinja2 python3-yaml
\ No newline at end of file
......@@ -261,7 +261,7 @@ def play_object(path, args, prog_cfgs):
daemon_logger_log = logging.getLogger('deckard.daemon.log')
# Parse scenario
case, cfg_text = scenario.parse_file(fileinput.input(path))
case, cfg_text = scenario.parse_file(os.path.realpath(path))
cfg_ctx = scenario.parse_config(cfg_text, args.qmin, INSTALLDIR)
# get working directory and environment variables
......@@ -289,6 +289,7 @@ def play_object(path, args, prog_cfgs):
try:
server.play(prog_under_test_ip)
finally:
name = server.scenario.file
server.stop()
for daemon in daemons:
daemon['proc'].terminate()
......
#!/usr/bin/python3
# Copyright (C) 2017
import posixpath
import logging
import os
import collections
import sys
from augeas import Augeas
#from IPython.core.debugger import Tracer
AUGEAS_LOAD_PATH = '/augeas/load/'
AUGEAS_FILES_PATH = '/files/'
AUGEAS_ERROR_PATH = '//error'
log = logging.getLogger('augeas')
def join(*paths):
"""
join two Augeas tree paths
FIXME: Beware: // is normalized to /
"""
norm_paths = [posixpath.normpath(path) for path in paths]
# first path must be absolute
assert norm_paths[0][0] == '/'
new_paths = [norm_paths[0]]
# relativize all other paths so join works as expected
for path in norm_paths[1:]:
if path.startswith('/'):
path = path[1:]
new_paths.append(path)
new_path = posixpath.join(*new_paths)
log.debug("join: new_path %s", new_path)
return posixpath.normpath(new_path)
class AugeasWrapper(object):
"""python-augeas higher-level wrapper.
Load single augeas lens and configuration file.
Exposes configuration file as AugeasNode object with dict-like interface.
AugeasWrapper can be used in with statement in the same way as file does.
"""
def __init__(self, confpath, lens, root=None, loadpath=None,
flags=Augeas.NO_MODL_AUTOLOAD | Augeas.NO_LOAD):
"""Parse configuration file using given lens.
Params:
confpath (str): Absolute path to the configuration file
lens (str): Name of module containing Augeas lens
root: passed down to original Augeas
flags: passed down to original Augeas
loadpath: passed down to original Augeas
flags: passed down to original Augeas
"""
log.debug('loadpath: %s', loadpath)
log.debug('confpath: %s', confpath)
self._aug = Augeas(root=root, loadpath=loadpath, flags=flags)
# /augeas/load/{lens}
aug_load_path = join(AUGEAS_LOAD_PATH, lens)
# /augeas/load/{lens}/lens = {lens}.lns
self._aug.set(join(aug_load_path, 'lens'), '%s.lns' % lens)
#print(join(aug_load_path, 'lens'), '%s.lns' % lens)
# /augeas/load/{lens}/incl[0] = {confpath}
self._aug.set(join(aug_load_path, 'incl[0]'), confpath)
#print(join(aug_load_path, 'incl[0]'), confpath)
self._aug.load()
errors = self._aug.match(AUGEAS_ERROR_PATH)
if errors:
err_msg = '\n'.join(
["{}: {}".format(e, self._aug.get(e)) for e in errors]
)
raise RuntimeError(err_msg)
path = join(AUGEAS_FILES_PATH, confpath)
paths = self._aug.match(path)
if len(paths) != 1:
raise ValueError('path %s did not match exactly once' % path)
self.tree = AugeasNode(self._aug, path)
self._loaded = True
def __enter__(self):
return self
def __exit__(self, exc_type, exc_value, traceback):
self.save()
self.close()
def save(self):
"""Save Augeas tree to its original file."""
assert self._loaded
try:
self._aug.save()
except IOError as exc:
log.exception(exc)
for err_path in self._aug.match('//error'):
log.error('%s: %s', err_path,
self._aug.get(os.path.join(err_path, 'message')))
raise
def close(self):
"""
close Augeas library
After calling close() the object must not be used anymore.
"""
assert self._loaded
self._aug.close()
del self._aug
self._loaded = False
def match(self, path):
"""Yield AugeasNodes matching given expression."""
assert self._loaded
assert len(path) > 0
log.debug('tree match %s', path)
for matched_path in self._aug.match(path):
yield AugeasNode(self._aug, matched_path)
class AugeasNode(collections.MutableMapping):
"""One Augeas tree node with dict-like interface."""
def __init__(self, aug, path, exists=True):
"""
Args:
aug (AugeasWrapper or Augeas): Augeas library instance
path (str): absolute path in Augeas tree matching single node
BEWARE: There are no sanity checks of given path for performance reasons.
"""
assert aug
assert path
assert path.startswith('/')
self._aug = aug
self._path = path
@property
def path(self):
"""canonical path in Augeas tree, read-only"""
return self._path
@property
def value(self):
"""
get value of this node in Augeas tree
"""
value = self._aug.get(self._path)
log.debug('tree get: %s = %s', self._path, value)
return value
@value.setter
def value(self, value):
"""
set value of this node in Augeas tree
"""
log.debug('tree set: %s = %s', self._path, value)
self._aug.set(self._path, value)
def __len__(self):
"""
number of items matching this path
It is always 1 after __init__() but it may change
as Augeas tree changes.
"""
return len(self._aug.match(self._path))
def __getitem__(self, key):
if isinstance(key, int):
# int is a shortcut to write [int]
target_path = '%s[%s]' % (self._path, key)
else:
target_path = self._path + key
log.debug('tree getitem: target_path %s', target_path)
paths = self._aug.match(target_path)
if len(paths) != 1:
raise KeyError('path %s did not match exactly once' % target_path)
return AugeasNode(self._aug, target_path)
def __delitem__(self, key):
log.debug('tree delitem: %s + %s', self._path, key)
target_path = self._path + key
log.debug('tree delitem: target_path %s', target_path)
self._aug.remove(target_path)
def __setitem__(self, key, value):
assert isinstance(value, AugeasNode)
target_path = self.path + key
self._aug.copy(value.path, target_path)
def __iter__(self):
self_path_len = len(self._path)
assert self_path_len > 0
log.debug('tree iter: %s', self._path)
for new_path in self._aug.match(self._path):
if len(new_path) == self_path_len:
yield ''
else:
yield new_path[self_path_len - 1:]
def match(self, subpath):
"""Yield AugeasNodes matching given sub-expression."""
assert subpath.startswith("/")
match_path = "%s%s" % (self._path, subpath)
log.debug('tree match %s: %s', match_path, self._path)
for matched_path in self._aug.match(match_path):
yield AugeasNode(self._aug, matched_path)
def __repr__(self):
return 'AugeasNode(%s)' % self._path
......@@ -24,12 +24,14 @@ let domain = [ label "domain" . store domain_re ]
let ttl = [label "ttl" . store /[0-9]+/]
let class = [label "class" . store class_re ]
let type = [label "type" . store ((/[^0-9#;\n \t][^\t\n\/#; ]*/) - class_re) ]
let data = [label "data" . store /[^\n\t ;#][^\n;#]+[^\n\t;# ]/]
let data_re = /([^ \t\n#;][^\n#;]*[^ \t\n#;])|[^ \t\n#;]/ (*Can not start nor end with whitespace but can have whitespace in the middle. Disjunction is there so we match strings of length one.*)
let data = [label "data" . store data_re ]
let ip_re = /[0-9a-f.:]+/
let hex_re = /[0-9a-fA-F]+/
let match_option = "opcode" | "qtype" | "qcase" | "qname" | "subdomain" | "flags" | "rcode" | "question" | "answer" | "authority" | "additional" | "all" | "TCP" | "ttl"
let match_option = "opcode" | "qtype" | "qcase" | "qname" | "subdomain" | "flags" | "rcode" | "question" | "answer" | "authority" | "additional" | "all" | "TCP" | "ttl"
let adjust_option = "copy_id" | "copy_query"
let reply_option = "QR" | "TC" | "AA" | "AD" | "RD" | "RA" | "CD" | "DO" | "NOERROR" | "FORMERR" | "SERVFAIL" | "NXDOMAIN" | "NOTIMP" | "REFUSED" | "YXDOMAIN" | "YXRRSET" | "NXRRSET" | "NOTAUTH" | "NOTZONE" | "BADVERS"
let step_option = "REPLY" | "QUERY" | "CHECK_ANSWER" | "CHECK_OUT_QUERY" | /TIME_PASSES[ \t]+ELAPSE/
......@@ -42,26 +44,26 @@ let adjust = (mandatory | tsig)* . del_str "ADJUST" . [space . label "adjust" .
let reply = (mandatory | tsig)* . del ("REPLY" | "FLAGS") "REPLY" . [space . label "reply" . store reply_option ]+ . comment_or_eol
let question = domain . tab . (class . tab)? . type . comment_or_eol
let record = [seq "record" . domain . tab . (ttl . tab)? . (class . tab)? . type . tab . data . comment_or_eol]
let question = [label "record" . domain . tab . (class . tab)? . type . comment_or_eol ]
let record = [label "record" . domain . tab . (ttl . tab)? . (class . tab)? . type . tab . data . comment_or_eol]
let section_question = [ label "question" . del_str "SECTION QUESTION" .
comment_or_eol . question? ]
let section_answer = [ label "answer" . counter "record" . del_str "SECTION ANSWER" .
let section_answer = [ label "answer" . del_str "SECTION ANSWER" .
comment_or_eol . record* ]
let section_authority = [ label "authority" . counter "record" . del_str "SECTION AUTHORITY" .
let section_authority = [ label "authority" . del_str "SECTION AUTHORITY" .
comment_or_eol . record* ]
let section_additional = [ label "additional" . counter "record" . del_str "SECTION ADDITIONAL" .
let section_additional = [ label "additional" . del_str "SECTION ADDITIONAL" .
comment_or_eol . record* ]
let sections = section_question? . section_answer? . section_authority? . section_additional?
let sections = [label "section" . section_question? . section_answer? . section_authority? . section_additional?]
let raw = [del_str "RAW" . comment_or_eol . label "raw" . store word ] . comment_or_eol
let raw = [del_str "RAW" . comment_or_eol . label "raw" . store hex_re ] . comment_or_eol
let normal = (match . (adjust . reply? | reply . adjust?)? | adjust . (match . reply? | reply . match?)? | reply . (match . adjust? | adjust . match?)?) . (mandatory | tsig)* . sections
(* This is quite dirty hack to match every combination of options given to entry since 'let normal = ((match | adjust | reply | mandatory | tsig)* . sections)' just is not possible *)
(*let normal = ((match | adjust | reply | mandatory | tsig)* . sections)*)
let normal = (match . (adjust . reply? | reply . adjust?)? | adjust . (match . reply? | reply . match?)? | reply . (match . adjust? | adjust . match?)?)? . (mandatory | tsig)* . sections
let entry = [label "entry" . del_str "ENTRY_BEGIN" . comment_or_eol . ( normal | raw )? . del_str "ENTRY_END" . eol]
let entry = [label "entry" . del_str "ENTRY_BEGIN" . comment_or_eol . ( normal | raw ) . del_str "ENTRY_END" . eol]
let single_address = [ label "address" . space . store ip_re ]
......@@ -79,13 +81,14 @@ let config = [ label "config" . counter "config" . [seq "config" . store config_
let guts = (step | range )*
let scenario = [label "scenario" . del_str "SCENARIO_BEGIN" . (space . store /[^ \t\n#;][^\n#;]+[^\t\n #;]/)? . comment_or_eol . guts . del_str "SCENARIO_END" . eol]
let scenario = [label "scenario" . del_str "SCENARIO_BEGIN" . space . store data_re . comment_or_eol . guts . del_str "SCENARIO_END" . eol]
let lns = config? . scenario
(* TODO: REPLAY step *)
(* TODO: store all comments into the tree instead of ignoring them *)
let filter = incl "/home/stepan/nic/deckard/sets/resolver/*.rpl"
(*let filter = incl "/home/test/*.rpl"*)
let filter = incl "/media/test/27159fa1-67d4-4162-8707-cd67900f3b36/stepan/nic/deckard_stable/deckard/sets/resolver/*.rpl"
let xfm = transform lns filter
stub-addr: 127.0.0.10
CONFIG_END
SCENARIO_BEGIN empty replies
RANGE_BEGIN 0 100
ADDRESS 127.0.0.10
ENTRY_BEGIN
MATCH subdomain
ADJUST copy_id copy_query
SECTION QUESTION
. IN A
ENTRY_END
RANGE_END
STEP 1 QUERY
ENTRY_BEGIN
ENTRY_END
SCENARIO_END
This diff is collapsed.
......@@ -26,6 +26,8 @@ class TestServer:
self.client_socks = []
self.connections = []
self.active = False
self.active_lock = threading.Lock()
self.condition = threading.Condition()
self.scenario = test_scenario
self.addr_map = []
self.start_iface = 2
......@@ -35,21 +37,26 @@ class TestServer:
def __del__(self):
""" Cleanup after deletion. """
if self.active is True:
with self.active_lock:
active = self.active
if active:
self.stop()
def start(self, port=53):
""" Synchronous start """
if self.active is True:
raise Exception('TestServer already started')
self.active = True
with self.active_lock:
if self.active:
raise Exception('TestServer already started')
with self.active_lock:
self.active = True
self.addr, _ = self.start_srv((self.kroot_local, port), self.addr_family)
self.start_srv(self.addr, self.addr_family, socket.IPPROTO_TCP)
self._bind_sockets()
def stop(self):
""" Stop socket server operation. """
self.active = False
with self.active_lock:
self.active = False
if self.thread:
self.thread.join()
for conn in self.connections:
......@@ -86,7 +93,7 @@ class TestServer:
log.debug('server %s received query from %s: %s', server_addr, client_addr, query)
response, is_raw_data = self.scenario.reply(query, server_addr)
if response:
if is_raw_data is False:
if not is_raw_data:
data_to_wire = response.to_wire(max_size=65535)
log.debug('response: %s', response)
else:
......@@ -108,9 +115,15 @@ class TestServer:
def query_io(self):
""" Main server process """
self.undefined_answers = 0
if self.active is False:
raise Exception("[query_io] Test server not active")
while self.active is True:
with self.active_lock:
if not self.active:
raise Exception(self.scenario.file+" [query_io] Test server not active")
while True:
with self.condition:
self.condition.notify()
with self.active_lock:
if not self.active:
break
objects = self.srv_socks + self.connections
to_read, _, to_error = select.select(objects, [], objects, 0.1)
for sock in to_read:
......@@ -154,6 +167,8 @@ class TestServer:
if self.thread is None:
self.thread = threading.Thread(target=self.query_io)
self.thread.start()
with self.condition:
self.condition.wait()
for srv_sock in self.srv_socks:
if (srv_sock.family == family
......@@ -203,25 +218,10 @@ def empty_test_case():
Return (scenario, config) pair which answers to any query on 127.0.0.10.
"""
# Mirror server
entry = scenario.Entry()
entry.set_match([]) # match everything
entry.set_adjust(['copy_id', 'copy_query'])
rng = scenario.Range(0, 100)
rng.add(entry)
rng.addresses.add('127.0.0.10')
step = scenario.Step(1, 'QUERY', [])
test_scenario = scenario.Scenario('empty replies')
test_scenario.ranges.append(rng)
test_scenario.steps.append(step)
test_scenario.current_step = step
empty_test_path = os.path.dirname(os.path.realpath(__file__)) + "/empty.rpl"
test_config = {'ROOT_ADDR': '127.0.0.10',
'_SOCKET_FAMILY': socket.AF_INET}
return (test_scenario, test_config)
return scenario.parse_file(empty_test_path)[0],test_config
def standalone_self_test():
......@@ -263,6 +263,7 @@ def standalone_self_test():
except KeyboardInterrupt:
logging.info("[==========] Shutdown.")
pass
name = server.scenario.file
server.stop()
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment