Commit 66b2839e authored by Petr Špaček's avatar Petr Špaček

CI: add PEP8, Pylint, and comparative tests

Candidate (to-be-merged) version is compared against merge base
(git merge-base master..HEAD).

New PEP8 or Pylint defects are show stopper.
Tests results for non-modified rpl files must not change.
parent 2054fa11
Pipeline #1926 failed with stage
in 46 seconds
variables:
LC_ALL: C.UTF-8
stages:
- test
test:pep8:
image: cznic/deckard-ci
stage: test
script:
- cp ci/common.sh /tmp
- cp ci/compare-pep8.sh /tmp
- /tmp/compare-pep8.sh
tags:
- docker
- linux
- amd64
test:pylint:
image: cznic/deckard-ci
stage: test
script:
- cp ci/common.sh /tmp
- cp ci/compare-pylint.sh /tmp
- /tmp/compare-pylint.sh
artifacts:
when: on_failure
expire_in: '1 hour'
paths:
- base.log
- head.log
tags:
- docker
- linux
- amd64
test:comparative:kresd:
image: cznic/deckard-ci
stage: test
script:
- cp ci/common.sh /tmp
- cp ci/compare-tests.sh /tmp
- /tmp/compare-tests.sh $(pwd)/kresd_run.sh
artifacts:
when: always
expire_in: '1 hour'
paths:
- modified_tests
- base.log
- base.tests
- head.log
- head.tests
tags:
- docker
- linux
- amd64
FROM cznic/ubuntu:16.04
MAINTAINER Knot DNS <knot-dns@labs.nic.cz>
WORKDIR /root
CMD ["/bin/bash"]
# knot-resolver used for comparative tests
# we do not care that much about particular version
RUN apt-get update
RUN apt-get upgrade -y
RUN apt-get install -y knot-resolver pep8 pylint
set -o errexit -o nounset
HEAD="$(git log -1 --format="%H" HEAD)"
MERGEBASE="$(git merge-base origin/master "${HEAD}")"
LOGDIR="$(pwd)"
# workaround for Gitlab's missing support for absolute paths in artifacts:
# https://gitlab.com/gitlab-org/gitlab-ci-multi-runner/issues/1011
declare -a LOGS
LOGS[0]="" # avoid unbound variable error if user does not specify own logs
function collect_logs {
set +o errexit
test -n "${LOGS[*]}" && cp "--target-directory=${LOGDIR}" ${LOGS[*]}
}
trap collect_logs EXIT
#!/bin/bash
set -o nounset -o xtrace -o errexit
source "$(dirname "$(readlink -f "$0")")/common.sh"
git diff "${MERGEBASE}..${HEAD}" | pep8 --diff --show-source --max-line-length=100 && echo "OK, no PEP8 errors detected"
#!/bin/bash
set -o nounset -o xtrace -o errexit
source "$(dirname "$(readlink -f "$0")")/common.sh"
PYFILES=$(find . \
-type d -exec test -e '{}/__init__.py' \; -print -prune -o \
-name '*.py' -print -o \
-type f -exec grep -qsm1 '^#!.*\bpython' '{}' \; -print)
: check if version under test does not produce critical errors
pylint -E ${PYFILES}
: no critical errors, compare score between versions
rm -rf ~/.pylint.d
: get test results from common ancestor with master branch
git checkout --force --detach "${MERGEBASE}"
git clean -xdf
pylint ${PYFILES} &> /tmp/base.log || : old version is not clear
LOGS[0]="/tmp/base.log"
echo ==================== merge base ====================
cat /tmp/base.log
echo ==================== merge base end ====================
: get test results from version under test
git checkout --force --detach "${HEAD}"
git clean -xdf
pylint ${PYFILES} &> /tmp/head.log || : version under test is not clear
LOGS[1]="/tmp/base.log"
echo ==================== candidate version ====================
cat /tmp/head.log
echo ==================== candidate end ====================
: check if candidate version produced more messages than the base
grep '^|\(convention\|refactor\|warning\|error\).*+' /tmp/head.log \
&& echo "New pylint message detected: Use diff base.log head.log and go fix it!" \
|| echo "OK, no new pylint messages detected"
#!/bin/bash
set -o nounset -o xtrace -o errexit
source "$(dirname "$(readlink -f "$0")")/common.sh"
TESTRUNNER="$1"
: comparing results from test script "${TESTRUNNER}"
# Run specified test runner on HEAD and again on merge base for master..HEAD
# Fail if result of any test not modified between master..HEAD changed
# (i.e. any change in Deckard should not change results of non-modified tests)
function extract_test_results {
# from log $1 extract test status lines like this:
# [ FAIL ] sets/resolver/iter_badglue.rpl
# [ OK ] sets/resolver/iter_badraw.rpl
# no spaces are allowed in test names
grep -o '^\[[^]]*\] [^ ]*\.rpl' "$1" | sort --field-separator=']' --key=2 | uniq
}
function find_modified_tests {
: detect tests affected by current merge request
: store list of modified tests in ${MODIFIED_TESTS_FILE}
git diff --numstat "${MERGEBASE}..${HEAD}" | cut -f 3 | fgrep .rpl > "${MODIFIED_TESTS_FILE}" || : no modified tests detected
}
function filter_test_results {
: skip tests which are listed in ${MODIFIED_TESTS_FILE}
grep --fixed-strings --invert-match --file="${MODIFIED_TESTS_FILE}"
}
MODIFIED_TESTS_FILE="/tmp/modified_tests"
find_modified_tests
LOGS[0]="${MODIFIED_TESTS_FILE}"
: get results from all tests, including the failing ones
export MAKEFLAGS="--output-sync=target --keep-going -j$(nproc)"
: get test results from version under test
"${TESTRUNNER}" &> /tmp/head.log || :
LOGS[1]="/tmp/head.log"
extract_test_results /tmp/head.log | filter_test_results &> /tmp/head.tests || (: "no tests left, huh?" && cat /tmp/head.log)
LOGS[2]="/tmp/head.tests"
: get test results from common ancestor with master branch
git checkout --force --detach "${MERGEBASE}"
git clean -xdf
"${TESTRUNNER}" &> /tmp/base.log || :
LOGS[3]="/tmp/base.log"
extract_test_results /tmp/base.log | filter_test_results &> /tmp/base.tests || (: "no tests left, huh?" && cat /tmp/base.log)
LOGS[4]="/tmp/base.tests"
: tests which were not modified should produce the same results
diff -U0 /tmp/base.tests /tmp/head.tests && echo "OK, no differences found"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment