Commit 74c58888 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'retry-when-under-load' into 'master'

retry tests which failed due to load

See merge request !134
parents d51ca57a 7d5cc169
Pipeline #43527 passed with stage
in 2 minutes and 3 seconds
...@@ -106,6 +106,8 @@ def rpls(paths): ...@@ -106,6 +106,8 @@ def rpls(paths):
def pytest_addoption(parser): def pytest_addoption(parser):
parser.addoption("--config", action="append", help="path to Deckard configuration .yaml file") parser.addoption("--config", action="append", help="path to Deckard configuration .yaml file")
parser.addoption("--scenarios", action="append", help="directory with .rpl files") parser.addoption("--scenarios", action="append", help="directory with .rpl files")
parser.addoption("--retries", action="store", help=("number of retries per"
"test when Deckard is under load"))
def pytest_generate_tests(metafunc): def pytest_generate_tests(metafunc):
...@@ -128,6 +130,11 @@ def pytest_generate_tests(metafunc): ...@@ -128,6 +130,11 @@ def pytest_generate_tests(metafunc):
if 'rpl_path' in metafunc.fixturenames: if 'rpl_path' in metafunc.fixturenames:
paths = metafunc.config.option.scenarios paths = metafunc.config.option.scenarios
metafunc.parametrize("rpl_path", rpls(paths), ids=str) metafunc.parametrize("rpl_path", rpls(paths), ids=str)
if 'max_retries' in metafunc.fixturenames:
max_retries = metafunc.config.option.retries
if max_retries is None:
max_retries = 3
metafunc.parametrize("max_retries", [max_retries], ids=lambda id: "max-retries-"+str(id))
def check_log_level_xdist(level): def check_log_level_xdist(level):
......
...@@ -309,7 +309,7 @@ def process_file(path, qmin, prog_cfgs): ...@@ -309,7 +309,7 @@ def process_file(path, qmin, prog_cfgs):
else: else:
shutil.rmtree(tmpdir) shutil.rmtree(tmpdir)
except Exception: except Exception:
logging.getLogger('deckard.hint').info( logging.getLogger('deckard.hint').error(
'test failed, inspect working directory %s', tmpdir) 'test failed, inspect working directory %s', tmpdir)
raise raise
...@@ -333,7 +333,11 @@ def setup_daemons(tmpdir, prog_cfgs, template_ctx, ta_files): ...@@ -333,7 +333,11 @@ def setup_daemons(tmpdir, prog_cfgs, template_ctx, ta_files):
def check_for_icmp(): def check_for_icmp():
""" Checks Deckards's PCAP for ICMP packets """ """ Checks Deckards's PCAP for ICMP packets """
# Deckard's responses to resolvers might be delayed due to load which
# leads the resolver to close the port and to the test failing in the
# end. We partially detect these by checking the PCAP for ICMP packets.
path = os.environ["SOCKET_WRAPPER_PCAP_FILE"] path = os.environ["SOCKET_WRAPPER_PCAP_FILE"]
udp_seen = False
with open(path, "rb") as f: with open(path, "rb") as f:
pcap = dpkt.pcap.Reader(f) pcap = dpkt.pcap.Reader(f)
for _, packet in pcap: for _, packet in pcap:
...@@ -341,8 +345,14 @@ def check_for_icmp(): ...@@ -341,8 +345,14 @@ def check_for_icmp():
ip = dpkt.ip.IP(packet) ip = dpkt.ip.IP(packet)
except dpkt.dpkt.UnpackError: except dpkt.dpkt.UnpackError:
ip = dpkt.ip6.IP6(packet) ip = dpkt.ip6.IP6(packet)
if isinstance(ip.data, (dpkt.icmp.ICMP, dpkt.icmp6.ICMP6)): if isinstance(ip.data, dpkt.udp.UDP):
return True udp_seen = True
if udp_seen:
if isinstance(ip.data, dpkt.icmp.ICMP, dpkt.icmp6.ICMP6):
raise DeckardUnderLoadError("Deckard is under load. "
"Other errors might be false negatives. "
"Consider retrying the job later.")
return False return False
...@@ -353,6 +363,9 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip): ...@@ -353,6 +363,9 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
try: try:
server.play(prog_under_test_ip) server.play(prog_under_test_ip)
except ValueError as e:
if not check_for_icmp():
raise e
finally: finally:
server.stop() server.stop()
for daemon in daemons: for daemon in daemons:
...@@ -368,11 +381,5 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip): ...@@ -368,11 +381,5 @@ def run_testcase(daemons, case, root_addr, addr_family, prog_under_test_ip):
% (daemon['cfg']['name'], daemon['proc'].returncode)) % (daemon['cfg']['name'], daemon['proc'].returncode))
# Do not clear files if the server crashed (for analysis) # Do not clear files if the server crashed (for analysis)
if server.undefined_answers > 0: if server.undefined_answers > 0:
# Deckard's responses to resolvers might be delayed due to load which if not check_for_icmp():
# leads the resolver to close the port and to the test failing in the raise ValueError('the scenario does not define all necessary answers (see error log)')
# end. We partially detect these by checking the PCAP for ICMP packets.
if check_for_icmp():
logging.error("Deckard is under load.\
Other errors might be false negatives.\
Consider retrying the job later.")
raise ValueError('the scenario does not define all necessary answers (see error log)')
import logging import logging
import os import os
import subprocess import subprocess
import random
import sys import sys
import time
import pytest import pytest
...@@ -33,24 +35,33 @@ logging.getLogger("augeas").setLevel(logging.ERROR) ...@@ -33,24 +35,33 @@ logging.getLogger("augeas").setLevel(logging.ERROR)
check_platform() check_platform()
def run_test(path, qmin, config): def run_test(path, qmin, config, max_retries, retries=0):
set_coverage_env(path, qmin) set_coverage_env(path, qmin)
try: try:
del os.environ["SOCKET_WRAPPER_DIR"] del os.environ["SOCKET_WRAPPER_DIR"]
except KeyError: except KeyError:
pass pass
deckard.process_file(path, qmin, config) try:
deckard.process_file(path, qmin, config)
except deckard.DeckardUnderLoadError as e:
def test_passes_qmin_on(scenario): if retries < max_retries:
logging.error("Deckard under load. Retrying…")
# Exponential backoff
time.sleep((2 ** retries) + random.random())
run_test(path, qmin, config, max_retries, retries + 1)
else:
raise e
def test_passes_qmin_on(scenario, max_retries):
if scenario.qmin is True or scenario.qmin is None: if scenario.qmin is True or scenario.qmin is None:
run_test(scenario.path, True, scenario.config) run_test(scenario.path, True, scenario.config, max_retries)
else: else:
pytest.skip("Query minimization is off in test config") pytest.skip("Query minimization is off in test config")
def test_passes_qmin_off(scenario): def test_passes_qmin_off(scenario, max_retries):
if scenario.qmin is False or scenario.qmin is None: if scenario.qmin is False or scenario.qmin is None:
run_test(scenario.path, False, scenario.config) run_test(scenario.path, False, scenario.config, max_retries)
else: else:
pytest.skip("Query minimization is on in test config") pytest.skip("Query minimization is on in test config")
...@@ -898,7 +898,7 @@ class Scenario: ...@@ -898,7 +898,7 @@ class Scenario:
for e in r.stored: for e in r.stored:
if e.mandatory and e.fired == 0: if e.mandatory and e.fired == 0:
# TODO: cisla radku # TODO: cisla radku
raise RuntimeError('Mandatory section at %s not fired' % e.mandatory.span) raise ValueError('Mandatory section at %s not fired' % e.mandatory.span)
def get_next(file_in, skip_empty=True): def get_next(file_in, skip_empty=True):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment