Commit 74c58888 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'retry-when-under-load' into 'master'

retry tests which failed due to load

See merge request !134
parents d51ca57a 7d5cc169
Pipeline #43527 passed with stage
in 2 minutes and 3 seconds
......@@ -106,6 +106,8 @@ def rpls(paths):
def pytest_addoption(parser):
parser.addoption("--config", action="append", help="path to Deckard configuration .yaml file")
parser.addoption("--scenarios", action="append", help="directory with .rpl files")
parser.addoption("--retries", action="store", help=("number of retries per"
"test when Deckard is under load"))
def pytest_generate_tests(metafunc):
......@@ -128,6 +130,11 @@ def pytest_generate_tests(metafunc):
if 'rpl_path' in metafunc.fixturenames:
paths = metafunc.config.option.scenarios
metafunc.parametrize("rpl_path", rpls(paths), ids=str)
if 'max_retries' in metafunc.fixturenames:
max_retries = metafunc.config.option.retries
if max_retries is None:
max_retries = 3
metafunc.parametrize("max_retries", [max_retries], ids=lambda id: "max-retries-"+str(id))
def check_log_level_xdist(level):
......
......@@ -309,7 +309,7 @@ def process_file(path, qmin, prog_cfgs):
else:
shutil.rmtree(tmpdir)
except Exception:
logging.getLogger('deckard.hint').info(
logging.getLogger('deckard.hint').error(
'test failed, inspect working directory %s', tmpdir)
raise
......@@ -333,7 +333,11 @@ def setup_daemons(tmpdir, prog_cfgs, template_ctx, ta_files):
def check_for_icmp():
""" Checks Deckards's PCAP for ICMP packets """
# Deckard's responses to resolvers might be delayed due to load which
# leads the resolver to close the port and to the test failing in the
# end. We partially detect these by checking the PCAP for ICMP packets.
path = os.environ["SOCKET_WRAPPER_PCAP_FILE"]
udp_seen = False
with open(path, "rb") as f:
pcap = dpkt.pcap.Reader(f)
for _, packet in pcap:
......@@ -341,8 +345,14 @@ def check_for_icmp():
ip = dpkt.ip.IP(packet)
except dpkt.dpkt.UnpackError:
ip = dpkt.ip6.IP6(packet)
if isinstance(ip.data, (dpkt.icmp.ICMP, dpkt.icmp6.ICMP6)):
return True
if isinstance(ip.data, dpkt.udp.UDP):
udp_seen = True
if udp_seen:
if isinstance(ip.data, dpkt.icmp.ICMP, dpkt.icmp6.ICMP6):
raise DeckardUnderLoadError("Deckard is under load. "
"Other errors might be false negatives. "
"Consider retrying the job later.")
return False
......@@ -353,6 +363,9 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
try:
server.play(prog_under_test_ip)
except ValueError as e:
if not check_for_icmp():
raise e
finally:
server.stop()
for daemon in daemons:
......@@ -368,11 +381,5 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
% (daemon['cfg']['name'], daemon['proc'].returncode))
# Do not clear files if the server crashed (for analysis)
if server.undefined_answers > 0:
# Deckard's responses to resolvers might be delayed due to load which
# leads the resolver to close the port and to the test failing in the
# end. We partially detect these by checking the PCAP for ICMP packets.
if check_for_icmp():
logging.error("Deckard is under load.\
Other errors might be false negatives.\
Consider retrying the job later.")
raise ValueError('the scenario does not define all necessary answers (see error log)')
if not check_for_icmp():
raise ValueError('the scenario does not define all necessary answers (see error log)')
import logging
import os
import subprocess
import random
import sys
import time
import pytest
......@@ -33,24 +35,33 @@ logging.getLogger("augeas").setLevel(logging.ERROR)
check_platform()
def run_test(path, qmin, config):
def run_test(path, qmin, config, max_retries, retries=0):
set_coverage_env(path, qmin)
try:
del os.environ["SOCKET_WRAPPER_DIR"]
except KeyError:
pass
deckard.process_file(path, qmin, config)
def test_passes_qmin_on(scenario):
try:
deckard.process_file(path, qmin, config)
except deckard.DeckardUnderLoadError as e:
if retries < max_retries:
logging.error("Deckard under load. Retrying…")
# Exponential backoff
time.sleep((2 ** retries) + random.random())
run_test(path, qmin, config, max_retries, retries + 1)
else:
raise e
def test_passes_qmin_on(scenario, max_retries):
if scenario.qmin is True or scenario.qmin is None:
run_test(scenario.path, True, scenario.config)
run_test(scenario.path, True, scenario.config, max_retries)
else:
pytest.skip("Query minimization is off in test config")
def test_passes_qmin_off(scenario):
def test_passes_qmin_off(scenario, max_retries):
if scenario.qmin is False or scenario.qmin is None:
run_test(scenario.path, False, scenario.config)
run_test(scenario.path, False, scenario.config, max_retries)
else:
pytest.skip("Query minimization is on in test config")
......@@ -898,7 +898,7 @@ class Scenario:
for e in r.stored:
if e.mandatory and e.fired == 0:
# TODO: cisla radku
raise RuntimeError('Mandatory section at %s not fired' % e.mandatory.span)
raise ValueError('Mandatory section at %s not fired' % e.mandatory.span)
def get_next(file_in, skip_empty=True):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment