runtests.py 8.56 KB
Newer Older
1 2
#!/usr/bin/env python3

3 4 5 6 7 8 9 10 11 12 13
import argparse
import datetime
import importlib
import logging
import os
import re
import sys
import tempfile
import time
import traceback

14
current_dir = os.path.dirname(os.path.realpath(__file__))
15
sys.path.append(os.path.join(current_dir, "tools"))
16 17
import dnstest.params as params
import dnstest.utils
18

19
TESTS_DIR = "tests"
20

21
def save_traceback(outdir):
22 23 24
    path = os.path.join(params.out_dir, "traceback.log")
    with open(path, mode="a") as f:
        traceback.print_exc(file=f)
25

26 27 28 29 30 31 32 33 34 35 36
def create_log(logger, filename="", level=logging.NOTSET):
    if filename:
        handler = logging.FileHandler(filename)
    else:
        handler = logging.StreamHandler()
    handler.setLevel(level)
    formatter = logging.Formatter('%(asctime)s# %(message)s', "%H:%M:%S")
    handler.setFormatter(formatter)
    logger.addHandler(handler)
    return handler

37 38 39 40 41 42 43 44 45 46 47 48 49 50
def log_environment(filename):
    def want_log(key):
        return key in [ "CC", "CPP", "CFLAGS", "CPPFLAGS",
                        "LDFLAGS", "LIBS",
                        "PKG_CONFIG", "PKG_CONFIG_PATH", "PKG_CONFIG_LIBDIR",
                        "YAAC", "YFLAGS",
                        "MALLOC_PERTURB_", "MALLOC_CHECK_" ] or \
              re.match(r'.+_(CFLAGS|LIBS)$', key) or \
              re.match(r'^KNOT_TEST_', key)

    with open(filename, "w") as log:
        lines = ["%s=%s\n" % (k, v) for (k, v) in os.environ.items() if want_log(k)]
        log.writelines(lines)

51 52 53 54 55 56 57 58 59
def parse_args(cmd_args):
    parser = argparse.ArgumentParser()
    parser.add_argument("-d", dest="debug", action="store_true", \
                        help="enable exception traceback on stdout")
    parser.add_argument("tests", metavar="[:]test[/case]", nargs="*", \
                        help="([exclude] | run) specific (test set | [test case])")
    args = parser.parse_args(cmd_args)

    params.debug = True if args.debug else False
60
    params.common_data_dir = os.path.join(current_dir, "data")
61 62 63 64 65 66 67 68 69 70

    # Process tests/cases arguments.
    excluded = dict()
    included = dict()
    for item in args.tests:
        if re.match(":", item):
            item = item[1:]
            storage = excluded
        else:
            storage = included
71

72 73 74 75 76 77 78 79 80 81 82
        parts = item.split("/")
        if len(parts) == 1:
            case = list()
        elif len(parts) == 2:
            case = [parts[1]]
        else:
            print("Invalid argument %s" % item)
            sys.exit(1)
        test = parts[0]

        if test in storage:
83
            storage[test].extend(case)
84 85 86 87 88
        else:
            storage[test] = case

    # List all tests if nothing was specified.
    if not included:
89 90
        tests_path = os.path.join(current_dir, TESTS_DIR)
        for i in sorted(os.listdir(tests_path)):
91 92 93 94
            included[i] = list()

    return included, excluded

95
def log_failed(log_dir, msg, indent=True):
96
    fname = os.path.join(log_dir, "failed.log")
97 98 99 100 101
    first = False if os.path.isfile(fname) else True

    file = open(fname, mode="a")
    if first:
        print("Failed tests:", file=file)
102
    print("%s%s" % ("  " if indent else "", msg), file=file)
103 104
    file.close()

105 106 107 108 109
def main(args):
    included, excluded = parse_args(args)

    timestamp = int(time.time())
    today = time.strftime("%Y-%m-%d", time.localtime(timestamp))
110 111
    outs_dir = tempfile.mkdtemp(prefix="knottest-%s-" % timestamp,
                                dir=params.outs_dir)
112

113
    # Try to create symlink to the latest result.
114
    last_link = os.path.join(params.outs_dir, "knottest-last")
115
    try:
116 117 118
        if os.path.exists(last_link):
            os.remove(last_link)
        os.symlink(outs_dir, last_link)
119 120 121
    except:
        pass

122 123 124
    # Write down environment
    log_environment(os.path.join(outs_dir, "environment.log"))

125 126 127 128
    # Set up logging.
    log = logging.getLogger()
    log.setLevel(logging.NOTSET)
    create_log(log)
129
    create_log(log, os.path.join(outs_dir, "summary.log"), logging.NOTSET)
130

131 132 133
    log.info("KNOT TESTING SUITE %s" % today)
    log.info("Working directory %s" % outs_dir)

134 135
    ref_time = datetime.datetime.now().replace(microsecond=0)

136 137 138 139 140 141
    case_cnt = 0
    fail_cnt = 0
    skip_cnt = 0
    for test in sorted(included):
        # Skip excluded test set.
        if test in excluded and not excluded[test]:
142 143
            continue

144
        # Check test directory.
145
        test_dir = os.path.join(current_dir, TESTS_DIR, test)
146 147
        if not os.path.isdir(test_dir):
            log.error("Test \'%s\':\tIGNORED (invalid folder)" % test)
148 149
            continue

150 151 152 153 154 155 156 157 158 159 160 161 162 163
        log.info("Test \'%s\'" % test)

        # Set test cases to run.
        if not included[test]:
            # List all test cases.
            cases = sorted(os.listdir(test_dir))
        else:
            cases = included[test]

        for case in cases:
            # Skip excluded cases.
            if test in excluded and case in excluded[test]:
                continue

164
            case_str_err = (" * case \'%s\':" % case).ljust(31)
165
            case_str_fail = ("%s/%s" % (test, case)).ljust(28)
166 167
            case_cnt += 1

168 169
            case_dir = os.path.join(test_dir, case)
            test_file = os.path.join(case_dir, "test.py")
170
            if not os.path.isfile(test_file):
171 172
                log.error(case_str_err + "MISSING")
                skip_cnt += 1
173 174 175
                continue

            try:
176 177 178
                out_dir = os.path.join(outs_dir, test, case)
                log_file = os.path.join(out_dir, "case.log")

179
                os.makedirs(out_dir, exist_ok=True)
180
                params.module = "%s.%s.%s" % (TESTS_DIR, test, case)
181 182
                params.test_dir = case_dir
                params.out_dir = out_dir
183
                params.case_log = open(log_file, mode="a")
184 185 186 187
                params.test = None
                params.err = False
                params.err_msg = ""
            except OsError:
188 189 190
                msg = "EXCEPTION (no dir \'%s\')" % out_dir
                log.error(case_str_err + msg)
                log_failed(outs_dir, case_str_fail + msg)
191 192 193 194 195
                fail_cnt += 1
                continue

            try:
                importlib.import_module("%s.%s.%s.test" % (TESTS_DIR, test, case))
196
            except dnstest.utils.Skip as exc:
197
                log.error(case_str_err + "SKIPPED (%s)" % format(exc))
198
                skip_cnt += 1
199 200 201 202 203 204 205 206 207 208 209 210 211 212
            except dnstest.utils.Failed as exc:
                save_traceback(params.out_dir)

                desc = format(exc)
                msg = "FAILED (%s)" % (desc if desc else exc.__class__.__name__)
                if params.err and params.err_msg:
                    msg += " AND (" + params.err_msg + ")"
                log.error(case_str_err + msg)
                log_failed(outs_dir, case_str_fail + msg)

                if params.debug:
                    traceback.print_exc()

                fail_cnt += 1
213 214
            except Exception as exc:
                save_traceback(params.out_dir)
215 216

                desc = format(exc)
217 218 219
                msg = "EXCEPTION (%s)" % (desc if desc else exc.__class__.__name__)
                log.error(case_str_err + msg)
                log_failed(outs_dir, case_str_fail + msg)
220

221 222
                if params.debug:
                    traceback.print_exc()
223

224 225 226 227 228 229 230 231 232 233 234
                fail_cnt += 1
            except BaseException as exc:
                save_traceback(params.out_dir)
                if params.debug:
                    traceback.print_exc()
                else:
                    log.info("INTERRUPTED")
                # Stop servers if still running.
                if params.test:
                    params.test.end()
                sys.exit(1)
235
            else:
236
                if params.err:
237
                    msg = "FAILED" + \
238
                          ((" (" + params.err_msg + ")") if params.err_msg else "")
239 240
                    log.info(case_str_err + msg)
                    log_failed(outs_dir, case_str_fail + msg)
241 242
                    fail_cnt += 1
                else:
243
                    log.info(case_str_err + "OK")
244

245 246
            # Stop servers if still running.
            if params.test:
247
                params.test.end()
248

249
            params.case_log.close()
250

251 252
    time_diff = datetime.datetime.now().replace(microsecond=0) - ref_time
    msg_time = "TOTAL TIME: %s, " % time_diff
253 254 255
    msg_cases = "TEST CASES: %i" % case_cnt
    msg_skips = ", SKIPPED: %i" % skip_cnt if skip_cnt > 0 else ""
    msg_res = ", FAILED: %i" % fail_cnt if fail_cnt > 0 else ", SUCCESS"
256
    log.info(msg_time + msg_cases + msg_skips + msg_res)
257

258
    if fail_cnt:
259
        log_failed(outs_dir, "Total %i/%i" % (fail_cnt, case_cnt), indent=False)
260 261 262
        sys.exit(1)
    else:
        sys.exit(0)
263

264 265
if __name__ == "__main__":
    main(sys.argv[1:])