Commit 23b42f28 authored by Daniel Salzman's avatar Daniel Salzman

func-test: improve logging

parent 04db17e2
#!/usr/bin/env python3
import argparse, importlib, os, re, sys, tempfile, time, traceback
import argparse, importlib, logging, os, re, sys, tempfile, time, traceback
current_dir = os.path.dirname(os.path.realpath(__file__))
sys.path.append(current_dir + "/tools")
from dnstest import log, err
import params
tests_dir = "tests"
params.common_data_dir = current_dir + "/tools/data/"
# Parse comman line arguments.
parser = argparse.ArgumentParser()
parser.add_argument("-d", dest="debug", action="store_true", \
help="enable exception traceback on stdout")
......@@ -13,12 +16,7 @@ parser.add_argument("tests", metavar="[:]test[/case]", nargs="*", \
help="([exclude] | run) specific (test set | [test case])")
args = parser.parse_args()
params.common_data_dir = current_dir + "/tools/data/"
outs_dir = tempfile.mkdtemp(prefix="knottest-%s-" % int(time.time()))
tests_dir = "tests"
test_cnt = 0
fail_cnt = 0
# Process tests/cases arguments.
excluded = dict()
included = dict()
for item in args.tests:
......@@ -49,24 +47,48 @@ if not included:
included[i] = list()
def save_traceback(outdir):
file = open(params.out_dir + "/traceback", mode="a")
file = open(params.out_dir + "/traceback.log", mode="a")
traceback.print_exc(file=file)
file.close()
log("Starting Knot test suite %s" % outs_dir)
def create_log(logger, filename="", level=logging.NOTSET):
if filename:
handler = logging.FileHandler(filename)
else:
handler = logging.StreamHandler()
handler.setLevel(level)
formatter = logging.Formatter('%(asctime)s# %(message)s', "%H:%M:%S")
handler.setFormatter(formatter)
logger.addHandler(handler)
return handler
timestamp = int(time.time())
today = time.strftime("%Y-%m-%d", time.localtime(timestamp))
outs_dir = tempfile.mkdtemp(prefix="knottest-%s-" % timestamp)
# Set up logging.
log = logging.getLogger()
log.setLevel(logging.NOTSET)
create_log(log)
create_log(log, outs_dir + "/summary.log", logging.NOTSET)
log.info("KNOT TESTING SUITE %s" % today)
log.info("Working directory %s" % outs_dir)
case_cnt = 0
fail_cnt = 0
for test in sorted(included):
# Skip excluded test set.
if test in excluded and not excluded[test]:
continue
# Check test directory.
test_dir = "./%s/%s" % (tests_dir, test)
test_dir = "%s/%s/%s" % (current_dir, tests_dir, test)
if not os.path.isdir(test_dir):
log("Invalid test name %s (ignored)" % test)
log.error("Invalid test name \'%s\': IGNORED" % test)
continue
log("Test \'%s\'" % test)
log.info("Test \'%s\'" % test)
# Set test cases to run.
if not included[test]:
......@@ -80,57 +102,63 @@ for test in sorted(included):
if test in excluded and case in excluded[test]:
continue
test_cnt += 1
case_cnt += 1
case_dir = test_dir + "/" + case
test_file = case_dir + "/test.py"
if not os.path.isfile(test_file):
log(" * case \'%s\': NOT EXECUTABLE!" % case)
log.error(" * case \'%s\':\tUNEXECUTABLE" % case)
fail_cnt += 1
continue
log(" * case \'%s\'" % case)
try:
out_dir = outs_dir + "/" + test + "/" + case
os.makedirs(out_dir, exist_ok=True)
params.test_dir = case_dir
params.out_dir = out_dir
params.err = False
params.errmsg = ""
params.case_log = open(out_dir + "/case.log", mode="a")
except OsError:
fail_cnt += 1
err("Can't create output directory %s" % out_dir)
log.error("Can't create output directory %s" % out_dir)
continue
try:
importlib.import_module("%s.%s.%s.test" % (tests_dir, test, case))
except Exception as exc:
params.err = True
params.errmsg = format(exc)
save_traceback(params.out_dir)
if args.debug:
traceback.print_exc()
else:
log.error(" EXCEPTION: " + format(exc))
except BaseException as exc:
log("Interrupted")
save_traceback(params.out_dir)
if args.debug:
traceback.print_exc()
else:
log.info("INTERRUPTED")
# Stop servers if still running.
if params.test:
params.test.stop()
exit(1)
# Stop servers if still running.
if params.err:
fail_cnt += 1
if params.errmsg:
err(params.errmsg)
if params.test:
params.test.stop()
log("TEST CASES: %i, FAILED: %i" % (test_cnt, fail_cnt))
params.case_log.close()
if params.err:
log.info(" * case \'%s\':\tFAILED" % case)
fail_cnt += 1
else:
log.info(" * case \'%s\':\tOK" % case)
if fail_cnt:
log.info("TEST CASES: %i, FAILED: %i" % (case_cnt, fail_cnt))
exit(1)
else:
log.info("TEST CASES: %i, SUCCESS" % (case_cnt))
exit(0)
......@@ -32,16 +32,16 @@ nsd_vars = [
["KNOT_TEST_NSDC", "nsdc"]
]
def log(text):
time_str = time.strftime("%H:%M:%S", time.localtime())
print("%s# %s" % (time_str, text))
def err(text):
frames = inspect.getouterframes(inspect.currentframe())
for frame in frames:
if params.test_dir == os.path.dirname(frame[1]):
log(" ERR> %s" % text)
log(" %s#%i" % (params.test_dir, frame[2]))
params.case_log.write("Error in %s#%i:\n" % (params.test_dir, frame[2]))
params.case_log.write(text + "\n")
params.case_log.write("====================================\n")
def detail(text):
params.case_log.write(str(text) + "\n")
def compare(value, expected, name):
if value != expected:
......@@ -896,6 +896,10 @@ class DnsTest(object):
self.servers = set()
Knot.count = 0
Bind.count = 0
Nsd.count = 0
def _check_port(self, port):
if not port:
return False
......@@ -1087,12 +1091,12 @@ class DnsTest(object):
if z1_diff:
params.err = True
for key in z1_diff:
print(key)
detail(key)
if z2_diff:
params.err = True
for key in z2_diff:
print(key)
detail(key)
if not z_keys:
return
......@@ -1100,8 +1104,8 @@ class DnsTest(object):
for key in z_keys:
if z1.nodes[key] != z2.nodes[key]:
params.err = True
print(z1.nodes[key].to_text(key))
print(z2.nodes[key].to_text(key))
detail(z1.nodes[key].to_text(key))
detail(z2.nodes[key].to_text(key))
err("differences in zone %s between %s and %s" % \
(zone, server1.name, server2.name))
......@@ -2,18 +2,17 @@
'''
This module allows interchanging of running parameters between modules.
All variables are set automatically.
'''
# Common data directory.
# Common data directory (e.g. zone files).
common_data_dir = ""
# Current case relative directory.
test_dir = ""
# Current case absolute output directory.
out_dir = ""
# Current case log file.
case_log = None
# Current test object (for controling test from other modules).
test = None
# Test result.
# Indication for failed test.
err = False
# Error message text.
errmsg = ""
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment