diffrepro: capture different_failure

parent e93a5b44
......@@ -98,8 +98,11 @@ def process_answers(
if others_agree:
reprocounter.upstream_stable += 1
assert mismatches is not None
if Diff(qid, mismatches) == report.target_disagreements[qid]:
new_diff = Diff(qid, mismatches)
if new_diff == report.target_disagreements[qid]:
reprocounter.verified += 1
elif new_diff:
reprocounter.different_failure += 1
def main():
......
......@@ -324,9 +324,10 @@ class Summary(Disagreements):
class ReproCounter(JSONDataObject):
_ATTRIBUTES = {
'retries': (None, None),
'upstream_stable': (None, None),
'verified': (None, None),
'retries': (None, None), # total amount of attempts to reproduce
'upstream_stable': (None, None), # number of cases, where others disagree
'verified': (None, None), # the query fails, and the diff is same (reproduced)
'different_failure': (None, None) # the query fails, but the diff doesn't match
}
def __init__(
......@@ -334,12 +335,14 @@ class ReproCounter(JSONDataObject):
retries: int = 0,
upstream_stable: int = 0,
verified: int = 0,
different_failure: int = 0,
_restore_dict: Optional[Mapping[str, int]] = None
) -> None:
super().__init__()
self.retries = retries
self.upstream_stable = upstream_stable
self.verified = verified
self.different_failure = different_failure
if _restore_dict is not None:
self.restore(_restore_dict)
......@@ -352,7 +355,8 @@ class ReproCounter(JSONDataObject):
return (
self.retries == other.retries
and self.upstream_stable == other.upstream_stable
and self.verified == other.verified)
and self.verified == other.verified
and self.different_failure == other.different_failure)
class ReproData(collections.abc.Mapping, JSONDataObject):
......
......@@ -382,21 +382,25 @@ def test_repro_counter():
assert rc.retries == 0
assert rc.upstream_stable == 0
assert rc.verified == 0
assert rc.different_failure == 0
rc = ReproCounter(retries=3, upstream_stable=2, verified=1)
assert rc.retries == 3
rc = ReproCounter(retries=4, upstream_stable=2, verified=1, different_failure=1)
assert rc.retries == 4
assert rc.upstream_stable == 2
assert rc.verified == 1
assert rc.different_failure == 1
rc = ReproCounter(_restore_dict={'retries': 4})
assert rc.retries == 4
assert rc.upstream_stable == 0
assert rc.verified == 0
assert rc.different_failure == 0
data = rc.save()
assert data['retries'] == 4
assert data['upstream_stable'] == 0
assert data['verified'] == 0
assert data['different_failure'] == 0
assert ReproCounter().save() is None
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment