Commit 8132f134 authored by Daniel Salzman's avatar Daniel Salzman

func-test: improve launching of tests + valgrind output processing

parent ffaa52b3
Prerequisites:
Python >=3.3
lsof
python3-dnspython
dnssec-keygen
dnssec-signzone
Bind 9.9
lsof
sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.named
\ No newline at end of file
#!/usr/bin/env python3
import os, sys, tempfile, time
import subprocess
def run_test(test_file, out_dir):
return subprocess.call([test_file, out_dir])
import importlib, os, sys, tempfile, time
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/tools")
from dnstest import log, err
import params
outs_dir = tempfile.mkdtemp(prefix="knottest-%s-" % int(time.time()))
tests_dir = "tests"
test_cnt = 0
fail_cnt = 0
tests_dir = "./cases"
outs_dir = tempfile.mkdtemp(prefix="knottest-%s-" % int(time.time()))
print("Starting Knot test suite %s" % outs_dir)
log("Starting Knot test suite %s" % outs_dir)
for test in sorted(os.listdir(tests_dir)):
test_dir = tests_dir + "/" + test
for test in sorted(os.listdir("./" + tests_dir)):
test_dir = "./%s/%s" % (tests_dir, test)
if not os.path.isdir(test_dir):
continue
test_file = test_dir + "/test.py"
if not os.path.isfile(test_file):
print("Missing test file %s" % test_file)
continue
log("Test \'%s\'" % test)
for case in sorted(os.listdir(test_dir)):
case_dir = test_dir + "/" + case
test_file = case_dir + "/test.py"
if not os.path.isfile(test_file):
continue
log(" * case \'%s\'" % case)
test_cnt += 1
try:
out_dir = outs_dir + "/" + test + "/" + case
os.makedirs(out_dir, exist_ok=True)
params.test_dir = case_dir
params.out_dir = out_dir
params.err = False
params.errmsg = ""
except OsError:
fail_cnt += 1
err("Can't create output directory %s" % out_dir)
continue
try:
importlib.import_module("%s.%s.%s.test" % (tests_dir, test, case))
try:
out_dir = outs_dir + "/" + test
os.mkdir(out_dir)
except Exception as exc:
params.err = True
params.errmsg = format(exc)
except KeyboardInterrupt:
log("Interrupted by user")
exit(1)
except:
params.err = True
params.errmsg = sys.exc_info()[0]
print("Test %s: " % test, end="")
if params.err:
fail_cnt += 1
if params.errmsg:
err(params.errmsg)
test_cnt = test_cnt + 1
if run_test(test_file, out_dir):
print("failed")
fail_cnt = fail_cnt + 1
else:
print("ok")
except (OSError, Exception) as err:
print(format(err))
exit(1)
except:
print("Unexpected error:", sys.exc_info()[0])
# Stop servers if still running.
if params.test:
params.test.stop()
if fail_cnt:
print("Failed tests: %i/%i" % (fail_cnt, test_cnt))
log("Failed test cases: %i/%i" % (fail_cnt, test_cnt))
exit(1)
else:
print("All tests passed")
log("All test cases passed")
exit(0)
#!/usr/bin/env python3
import os, sys
test_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(test_dir + "/../../tools")
from dnstest import *
import dnstest
################################################################################
t = DnsTest(test_dir, sys.argv[1])
t = dnstest.DnsTest()
master1 = t.server("knot", nsid="nsid", ident=True, version="Knot XXX")
master2 = t.server("bind", ident="ahoj", version="xx")
......@@ -25,4 +20,4 @@ t.link(z3, slave, master1)
t.start()
t.stop()
t.end()
#!/usr/bin/env python3
import base64
import re
import os
import random
import shutil
......@@ -9,8 +10,7 @@ import string
import sys
import time
from subprocess import Popen, PIPE, DEVNULL, check_call
import zone_generate
import zone_generate, params
knot_vars = [
["KNOT_TEST_KNOT", "knotd"],
......@@ -25,6 +25,13 @@ nsd_vars = [
["KNOT_TEST_NSDC", "nsdc"]
]
def log(text):
time_str = time.strftime("%H:%M:%S", time.localtime())
print("%s# %s" % (time_str, text))
def err(text):
log(" ERR> %s" % text)
class Tsig(object):
'''TSIG key generator'''
......@@ -260,7 +267,7 @@ class DnsServer(object):
stdout=self.fout, stderr=self.ferr)
p.communicate(timeout=DnsServer.COMPILE_TIMEOUT)
except:
print("Compile error")
err("Compile error")
def start(self):
try:
......@@ -278,7 +285,7 @@ class DnsServer(object):
else:
time.sleep(DnsServer.START_WAIT)
except OSError:
print("Server %s start error" % self.name)
err("Server %s start error" % self.name)
def reload(self):
try:
......@@ -286,7 +293,7 @@ class DnsServer(object):
stdout=DEVNULL, stderr=DEVNULL)
time.sleep(DnsServer.START_WAIT)
except OSError:
print("Server %s reload error" % self.name)
err("Server %s reload error" % self.name)
def flush(self):
try:
......@@ -295,7 +302,39 @@ class DnsServer(object):
stdout=DEVNULL, stderr=DEVNULL)
time.sleep(DnsServer.START_WAIT)
except OSError:
print("Server %s flush error" % self.name)
err("Server %s flush error" % self.name)
def _valgrind_check(self):
if not self.valgrind:
return
errcount = 0
reachable = -32
lost = 0
f = open(self.ferr, "r")
for line in f:
if re.search("Process terminating", line) or \
re.search("Invalid read", line) or \
re.search("Invalid write", line) or \
re.search("Assertion", line):
errcount += 1
lost_line = re.search("lost:", line)
if lost_line:
lost += int(line[lost_line.end():].lstrip().\
split(" ")[0].replace(",", ""))
reach_line = re.search("reachable:", line)
if reach_line:
reachable += int(line[reach_line.end():].lstrip().\
split(" ")[0].replace(",", ""))
f.close()
if errcount > 0 or reachable > 0 or lost > 0:
params.err = True
err("%s memcheck: lost(%i B), reachable(%i B), errcount(%i)" \
% (self.name, lost, reachable, errcount))
def stop(self):
if self.proc:
......@@ -303,7 +342,7 @@ class DnsServer(object):
self.proc.terminate()
self.proc.wait(DnsServer.STOP_TIMEOUT)
except:
print("killing")
err("killing")
self.proc.kill()
def gen_confile(self):
......@@ -346,6 +385,7 @@ class Bind(DnsServer):
s.item("listen-on-v6 port", "%i { %s; }" % (self.port, self.addr))
s.item("auth-nxdomain", "no")
s.item("recursion", "no")
s.item("masterfile-format", "text")
s.end()
s.begin("key", self.ctlkey.name)
......@@ -594,12 +634,12 @@ class DnsTest(object):
# Number of unsuccessful starts of servers. Recursion protection.
start_tries = 0
def __init__(self, test_dir, out_dir, ip=None, tsig=None):
if not os.path.exists(out_dir):
def __init__(self, ip=None, tsig=None):
if not os.path.exists(params.out_dir):
raise Exception("Output directory doesn't exist")
self.out_dir = str(out_dir)
self.data_dir = str(test_dir) + "/data/"
self.out_dir = params.out_dir
self.data_dir = params.test_dir + "/data/"
self.zones_dir = self.out_dir + "/zones/"
try:
os.mkdir(self.zones_dir)
......@@ -717,13 +757,17 @@ class DnsTest(object):
self.stop()
self.start()
params.test = self
def stop(self):
for server in self.servers:
server.stop()
params.test = None
def end(self):
self.stop()
pass
for server in self.servers:
server._valgrind_check()
def zone(self, name, filename):
try:
......@@ -763,7 +807,7 @@ class DnsTest(object):
file.close()
except OSError:
print("Can't create zone file %s" % filename)
err("Can't create zone file %s" % filename)
zones[name + "."] = filename
......
#!/usr/bin/env python3
'''
This module allows interchanging of running parameters between modules.
All variables are set automatically.
'''
# Current case relative directory.
test_dir = ""
# Current case absolute output directory.
out_dir = ""
# Current test object (for controling test from other modules).
test = None
# Test result.
err = False
# Error message text.
errmsg = ""
# Enable exception traceback.
debug = False
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment