Commit da204c32 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'asan-build' into 'master'

job_manager: resperf test + ASAN build

See merge request !34
parents e407e22e c186cd18
Pipeline #40350 passed with stage
in 2 minutes and 20 seconds
......@@ -28,12 +28,13 @@ RUN \
# actual image
FROM registry.labs.nic.cz/knot/respdiff/knot-resolver-buildenv:knot-$KNOT_BRANCH
MAINTAINER Knot Resolver <knot-resolver@labs.nic.cz>
ARG CFLAGS
COPY --from=intermediate /var/opt/knot-resolver /var/opt/knot-resolver
WORKDIR /var/opt/knot-resolver
RUN mkdir .install
RUN make PREFIX=/var/opt/knot-resolver/.install LDFLAGS="-Wl,-rpath=/var/opt/knot-resolver/.install/lib"
RUN make PREFIX=/var/opt/knot-resolver/.install LDFLAGS="-Wl,-rpath=/var/opt/knot-resolver/.install/lib" CFLAGS="${CFLAGS}"
RUN make install PREFIX=/var/opt/knot-resolver/.install
RUN mkdir /log
......
FROM ubuntu:latest
MAINTAINER Knot Resolver <knot-resolver@labs.nic.cz>
CMD ["tail", "-f", "/dev/null"]
RUN apt-get update
RUN apt-get install -y -qqq git wget build-essential bind9utils libbind-dev libssl-dev libkrb5-dev libcap-dev libxml2-dev libjson-c-dev libgeoip-dev
RUN cd /var/opt; git clone https://github.com/akamai/dnsperf.git
WORKDIR /var/opt/dnsperf
RUN ./configure && make && make install
RUN wget -q https://gitlab.labs.nic.cz/knot/respdiff/uploads/9b95b53c04091849023b12b992edad0e/shortlist -O shortlist
......@@ -69,11 +69,7 @@ def load_test_case_config(test_case: str) -> Dict[str, Any]:
return yaml.load(f)
def create_template_files(directory: str, config: Dict[str, Any]):
create_file_from_template('run_respdiff.sh.j2', config, directory, executable=True)
create_file_from_template('restart-all.sh.j2', config, directory, executable=True)
create_file_from_template('docker-compose.yaml.j2', config, directory)
def create_resolver_configs(directory: str, config: Dict[str, Any]):
for name, resolver in config['resolvers'].items():
resolver['name'] = name
resolver['verbose'] = config['verbose']
......@@ -100,6 +96,19 @@ def create_template_files(directory: str, config: Dict[str, Any]):
raise NotImplementedError(
"unknown resolver type: '{}'".format(resolver['type']))
def create_resperf_files(directory: str, config: Dict[str, Any]):
create_file_from_template('run_resperf.sh.j2', config, directory, executable=True)
create_file_from_template('docker-compose.yaml.j2', config, directory)
create_resolver_configs(directory, config)
def create_respdiff_files(directory: str, config: Dict[str, Any]):
create_file_from_template('run_respdiff.sh.j2', config, directory, executable=True)
create_file_from_template('restart-all.sh.j2', config, directory, executable=True)
create_file_from_template('docker-compose.yaml.j2', config, directory)
create_resolver_configs(directory, config)
# omit resolvers without respdiff section from respdiff.cfg
config['resolvers'] = {
name: res for name, res
......@@ -108,6 +117,13 @@ def create_template_files(directory: str, config: Dict[str, Any]):
create_file_from_template('respdiff.cfg.j2', config, directory)
def create_template_files(directory: str, config: Dict[str, Any]):
if 'respdiff' in config:
create_respdiff_files(directory, config)
elif 'resperf' in config:
create_resperf_files(directory, config)
def get_test_case_list(nameglob: str = '') -> List[str]:
# test cases end with '*.jXXX' implying a number of jobs (less -> longer runtime)
# return them in ascending order, so more time consuming test cases run first
......@@ -138,6 +154,7 @@ def create_jobs(args: argparse.Namespace) -> None:
config['git_sha'] = git_sha
config['knot_branch'] = args.knot_branch
config['verbose'] = args.verbose
config['asan'] = args.asan
directory = os.path.join(args.jobs_dir, commit_dir, test_case)
prepare_dir(directory, clean=args.clean)
......@@ -175,6 +192,9 @@ def main() -> None:
parser.add_argument(
'-v', '--verbose', action='store_true',
help="Capture verbose logs from kresd")
parser.add_argument(
'--asan', action='store_true',
help="Build with Address Sanitizer")
args = parser.parse_args()
create_jobs(args)
......
......@@ -13,7 +13,8 @@ services:
GIT_SHA: {{ git_sha }}
SSH_PRIVATE_KEY: ${SSH_PRIVATE_KEY}
KNOT_BRANCH: ${KNOT_BRANCH}
image: knot-resolver:{{ git_sha }}
CFLAGS: {{ '-ggdb3 -O0 -fsanitize=address -fno-omit-frame-pointer' if asan else "''" }}
image: knot-resolver:{{ git_sha }}{{ '-asan' if asan else '' }}
volumes:
- "./{{ name }}.conf:/etc/knot-resolver/kresd.conf:ro"
- "./root.keys:/etc/knot-resolver/root.keys:ro"
......@@ -42,3 +43,9 @@ services:
{%- endif -%}
{%- endfor -%}
{%- if resperf %}
resperf:
network_mode: host
image: registry.labs.nic.cz/knot/respdiff/resperf:latest
{%- endif -%}
......@@ -21,6 +21,8 @@ if [ -n "$(docker ps -q)" ]; then
fi
# Ensure ports are empty, so we're not testing something else!
set +o xtrace
echo "Waiting for ports..."
MAX_RETRIES=60
{% for resolver in resolvers.values() -%}
retries=0
......@@ -46,12 +48,13 @@ while [ ! -z "$(ss --all --no-header '( dport = :{{ resolver["tls_port"] }} or s
done
{% endif %}
{% endfor %}
set -o xtrace
# Exit if any cmd fails from now on.
set -o errexit
cleanup() {
docker-compose rm -fsv || :
docker-compose --no-ansi rm -fsv || :
tar --use-compress-program=pigz -cf ${JOBNAME}_logs.tar.gz logs &>/dev/null || :
{%if database['remove_after'] %}
rm -rf ./*.mdb || :
......@@ -71,13 +74,13 @@ set -o xtrace
# Attempt to build containers (to have stdout/err logs in case of failure)
export KNOT_BRANCH={{ knot_branch }}
time docker-compose up --no-start >&2
time docker-compose --no-ansi up --no-start >&2
# Start and wait for resolvers to be available
./restart-all.sh
# Run respdiff toolchain
time "{{ respdiff['srcdir'] }}/orchestrator.py" .
docker-compose stop # give lingering connections time to close
docker-compose --no-ansi stop # give lingering connections time to close
time "{{ respdiff['srcdir'] }}/msgdiff.py" -d ${JOBNAME}_report.json .
cp ${JOBNAME}_report{,.diffrepro}.json
time "{{ respdiff['srcdir'] }}/diffrepro.py" -d ${JOBNAME}_report.diffrepro.json .
......
#!/bin/bash
set -o xtrace
JOBNAME=j$1.$2
# HACK (condor): ensure files for transfer exist to avoid held jobs
touch ${JOBNAME}_kresd.docker.txt
touch ${JOBNAME}_resperf.txt
echo '0' > ${JOBNAME}_exitcode
mkdir logs
tar -czf ${JOBNAME}_logs.tar.gz -T /dev/null # empty tar by default
# HACK (condor): Create proper dir structure
mkdir -p docker-knot-resolver || :
mv Dockerfile docker-knot-resolver/ || :
# HACK (condor): set PATH, otherwise docker-compose exec will fail
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
# Check no other docker containers are running
if [ -n "$(docker ps -q)" ]; then
echo "ERROR: running docker containers detected" >&2
exit 2
fi
# Exit if any cmd fails from now on.
set -o errexit
cleanup() {
docker-compose --no-ansi rm -fsv || :
exit 0
}
trap cleanup EXIT
err_handle() {
echo '1' > ${JOBNAME}_exitcode
}
trap err_handle ERR
# Prepare ssh key for knot-resolver-security
set +o xtrace
SSH_PRIVATE_KEY=$(cat </home/respdiff/.ssh/knot_resolver_security.id_rsa || :)
export SSH_PRIVATE_KEY
set -o xtrace
# Attempt to build containers (to have stdout/err logs in case of failure)
export KNOT_BRANCH={{ knot_branch }}
time docker-compose --no-ansi up --no-start
# Run resperf test repeatedly
for i in $(seq 3); do
set +o xtrace
echo "Waiting for ports..."
# Ensure ports are empty, so we're not testing something else!
MAX_RETRIES=30 # x5s
{% for resolver in resolvers.values() -%}
retries=0
{% if resolver['port'] %}
while [ ! -z "$(ss --all --no-header '( dport = :{{ resolver["port"] }} or sport = :{{ resolver["port"] }} )')" ]; do
retries=$((retries+1))
if (( $retries > $MAX_RETRIES )); then
echo "ERROR: port {{ resolver['port'] }} not availble" >&2
exit 2
fi
sleep 5
done
{% endif %}
{% if resolver['tls_port'] %}
retries=0
while [ ! -z "$(ss --all --no-header '( dport = :{{ resolver["tls_port"] }} or sport = :{{ resolver["tls_port"] }} )')" ]; do
retries=$((retries+1))
if (( $retries > $MAX_RETRIES )); then
echo "ERROR: port {{ resolver['tls_port'] }} not availble" >&2
exit 2
fi
sleep 5
done
{% endif %}
{% endfor %}
set -o xtrace
# Execute resperf
docker-compose --no-ansi up -d
docker-compose --no-ansi exec -T resperf resperf -q 500000 -m 10000 -C 200 -c 30 -d shortlist -t 15 -r 0 -s 127.0.0.1 -p {{ resperf['port'] }} &>>${JOBNAME}_resperf.txt
docker-compose --no-ansi logs &>>${JOBNAME}_kresd.docker.txt
docker-compose --no-ansi ps | (! grep Exit)
docker-compose --no-ansi kill
done
......@@ -30,10 +30,26 @@ def condor_submit(txn, priority: int) -> int:
directory = os.getcwd()
input_files = get_all_files(directory)
if 'run_respdiff.sh' not in input_files:
if 'run_respdiff.sh' in input_files:
executable = 'run_respdiff.sh'
output_files = [
'j$(Cluster).$(Process)_report.json',
'j$(Cluster).$(Process)_report.diffrepro.json',
'j$(Cluster).$(Process)_report.txt',
'j$(Cluster).$(Process)_report.diffrepro.txt',
'j$(Cluster).$(Process)_histogram.png',
'j$(Cluster).$(Process)_logs.tar.gz']
elif 'run_resperf.sh' in input_files:
executable = 'run_resperf.sh'
output_files = [
'j$(Cluster).$(Process)_exitcode',
'j$(Cluster).$(Process)_kresd.docker.txt',
'j$(Cluster).$(Process)_resperf.txt',
'j$(Cluster).$(Process)_logs.tar.gz']
else:
raise RuntimeError(
"The provided directory doesn't look like a respdiff job. "
"{}/run_respdiff.sh is missing!".format(directory))
"The provided directory doesn't look like a respdiff/resperf job. "
"{}/run_*.sh is missing!".format(directory))
# create batch name from dir structure
commit_dir_path, test_case = os.path.split(directory)
......@@ -42,7 +58,7 @@ def condor_submit(txn, priority: int) -> int:
submit = Submit({
'priority': str(priority),
'executable': 'run_respdiff.sh',
'executable': executable,
'arguments': '$(Cluster) $(Process)',
'error': 'j$(Cluster).$(Process)_stderr.txt',
'output': 'j$(Cluster).$(Process)_stdout.txt',
......@@ -51,13 +67,7 @@ def condor_submit(txn, priority: int) -> int:
'should_transfer_files': 'YES',
'when_to_transfer_output': 'ON_EXIT',
'transfer_input_files': ', '.join(input_files),
'transfer_output_files': ', '.join([
'j$(Cluster).$(Process)_report.json',
'j$(Cluster).$(Process)_report.diffrepro.json',
'j$(Cluster).$(Process)_report.txt',
'j$(Cluster).$(Process)_report.diffrepro.txt',
'j$(Cluster).$(Process)_histogram.png',
'j$(Cluster).$(Process)_logs.tar.gz']),
'transfer_output_files': ', '.join(output_files),
})
return submit.queue(txn)
......
resolvers:
kresd_fwd_target:
type: knot-resolver
port: 55002
tls_port: &kresd_fwd_target_port 55009
listen_ipv6: true
kresd:
type: knot-resolver
port: &kresd_port 55001
listen_ipv6: true
forward:
ip: '::1'
port: *kresd_fwd_target_port
tls: true
resperf:
port: *kresd_port
resolvers:
kresd_fwd_target:
type: knot-resolver
port: &kresd_fwd_target_port 55102
listen_ipv6: true
kresd:
type: knot-resolver
port: &kresd_port 55101
forward:
ip: '::1'
port: *kresd_fwd_target_port
resperf:
port: *kresd_port
resolvers:
kresd:
type: knot-resolver
port: &kresd_port 55201
resperf:
port: *kresd_port
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment