Commit 9b17273f authored by Tomas Krizek's avatar Tomas Krizek

msgdiff: generate new json datafromat

parent 93e1d9f7
......@@ -62,14 +62,6 @@ class DataMismatch(Exception):
return hash(self.key)
def to_json(python_object):
if isinstance(python_object, DataMismatch):
return {
'exp_val': python_object.exp_val,
'got_val': python_object.got_val,
def read_json(filename):
with open(filename, 'r') as f:
return json.load(f)
......@@ -10,6 +10,10 @@ def qid2key(qid):
return struct.pack('@I', qid) # native integer
def key2qid(key):
return struct.unpack('@I', key)[0]
class LMDB:
ANSWERS = b'answers'
DIFFS = b'diffs'
......@@ -4,38 +4,22 @@ import argparse
from functools import partial
import logging
import multiprocessing.pool as pool
import os
import pickle
import sys
from typing import Dict
import dns.message
import dns.exception
import cfg
import dataformat
from dbhelper import LMDB
from dataformat import Reply, DataMismatch, DiffReport, Disagreements, DisagreementsCounter
from dbhelper import LMDB, key2qid
lmdb = None
class DataMismatch(Exception):
def __init__(self, exp_val, got_val):
super(DataMismatch, self).__init__(exp_val, got_val)
self.exp_val = exp_val
self.got_val = got_val
def __str__(self):
return 'expected "{0.exp_val}" got "{0.got_val}"'.format(self)
def __eq__(self, other):
return (isinstance(other, DataMismatch)
and self.exp_val == other.exp_val
and self.got_val == other.got_val)
def __ne__(self, other):
return not self.__eq__(other)
def compare_val(exp_val, got_val):
""" Compare values, throw exception if different. """
if exp_val != got_val:
......@@ -153,7 +137,7 @@ def match(expected, got, match_fields):
yield (code, ex)
def decode_wire_dict(wire_dict: Dict[str, dataformat.Reply]) \
def decode_wire_dict(wire_dict: Dict[str, Reply]) \
-> Dict[str, dns.message.Message]:
answers = {} # type: Dict[str, dns.message.Message]
for k, v in wire_dict.items():
......@@ -231,12 +215,35 @@ def compare_lmdb_wrapper(criteria, target, qid):
txn.put(qid, blob)
def export_json(filename):
report = DiffReport.from_json(filename)
report.other_disagreements = DisagreementsCounter()
report.target_disagreements = Disagreements()
# get diff data
ddb = lmdb.get_db(LMDB.DIFFS)
with lmdb.env.begin(ddb) as txn:
with txn.cursor() as diffcur:
for key, diffblob in diffcur:
qid = key2qid(key)
others_agree, diff = pickle.loads(diffblob)
if not others_agree:
report.other_disagreements.count += 1
for field, mismatch in diff.items():
report.target_disagreements.add_mismatch(field, mismatch, qid)
def main():
global lmdb
logging.basicConfig(format='%(levelname)s %(message)s', level=logging.DEBUG)
parser = argparse.ArgumentParser(
description='compute diff from answers stored in LMDB and write diffs to LMDB')
parser.add_argument('-d', '--datafile', type=str, default='report.json',
help='JSON report file (default: report.json)')
parser.add_argument('-c', '--config', default='respdiff.cfg', dest='cfgpath',
help='config file (default: respdiff.cfg)')
parser.add_argument('envdir', type=str,
......@@ -247,6 +254,11 @@ def main():
criteria = config['diff']['criteria']
target = config['diff']['target']
# JSON report has to be created by orchestrator
if not os.path.exists(args.datafile):
logging.error("JSON report (%s) doesn't exist!", args.datafile)
with LMDB(args.envdir, fast=True) as lmdb_:
lmdb = lmdb_
......@@ -256,6 +268,7 @@ def main():
with pool.Pool() as p:
for _ in p.imap_unordered(func, qid_stream, chunksize=10):
if __name__ == '__main__':
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment