respdiff/dataformat: add QIDs for other_disagreements

parent 8573c119
Pipeline #39895 passed with stage
in 1 minute and 1 second
......@@ -64,6 +64,9 @@ Respdiff is conceptually chain of independent tools:
Changelog
---------
- 2018-08-30: JSON datafile format changed: ``other_disagreements`` now
includes a list of all affected queries. Older reports will incorrectly report
0 for this field. (Can be fixed by re-creating reports with ``msgdiff.py``).
- 2018-08-21: ``qname``, ``qcase`` and ``qtype`` field comparisons were removed.
``question`` comparison now performs all these check in a case sensitive manner.
Remove these checks from ``criteria`` and ``field_weights`` in ``respdiff.cfg``.
......
......@@ -62,7 +62,7 @@ def export_json(filename: str, report: DiffReport):
qid = key2qid(key)
others_agree, diff = pickle.loads(diffblob)
if not others_agree:
report.other_disagreements.count += 1
report.other_disagreements.queries.add(qid)
else:
for field, mismatch in diff.items():
report.target_disagreements.add_mismatch(field, mismatch, qid)
......
......@@ -239,20 +239,20 @@ class Disagreements(collections.abc.Mapping, JSONDataObject):
class DisagreementsCounter(JSONDataObject):
_ATTRIBUTES = {
'count': (None, None)
'queries': (set, list),
}
def __init__(self, _restore_dict: Mapping[str, int] = None) -> None:
super(DisagreementsCounter, self).__init__()
self.count = 0
self.queries = set() # type: Set[QID]
if _restore_dict is not None:
self.restore(_restore_dict)
def __len__(self):
return self.count
def __len__(self) -> int:
return len(self.queries)
def __eq__(self, other) -> bool:
return self.count == other.count
return self.queries == other.queries
class Summary(Disagreements):
......
......@@ -85,7 +85,7 @@ DIFF_REPORT_JSON = """
"total_answers": 23100,
"total_queries": 23122,
"other_disagreements": {
"count": 150
"queries": [7, 8, 9]
},
"target_disagreements": {
"count": 4,
......@@ -300,11 +300,14 @@ def test_disagreements():
def test_disagreements_counter():
dc = DisagreementsCounter()
assert dc.count == 0
dc.count = 4
assert len(dc) == 0 # pylint: disable=len-as-condition
dc.queries.add(1)
dc.queries.add(1)
dc.queries.add(2)
assert len(dc) == 2
dc_restored = DisagreementsCounter(_restore_dict=dc.save())
assert dc_restored.count == 4
assert len(dc_restored) == 4
assert len(dc_restored) == 2
assert dc_restored.queries == set([1, 2])
def test_diff_report():
......@@ -314,7 +317,8 @@ def test_diff_report():
assert report.duration == 1
assert report.total_queries == 23122
assert report.total_answers == 23100
assert len(report.other_disagreements) == 150
assert len(report.other_disagreements) == 3
assert set([7, 8, 9]) == report.other_disagreements.queries
assert len(report.target_disagreements) == 4
report_restored = DiffReport(_restore_dict=report.save())
......@@ -352,8 +356,8 @@ def test_summary():
report.reprodata = ReproData(_restore_dict=json.loads(REPRODATA_JSON))
summary = Summary.from_report(report, field_weights)
assert len(summary) == 2
assert summary.upstream_unstable == (150 + 1)
assert summary.usable_answers == (23100 - 150 - 2)
assert summary.upstream_unstable == (3 + 1)
assert summary.usable_answers == (23100 - 3 - 2)
assert summary.not_reproducible == 1
# JSON export/import
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment