Commit 8ca19d77 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'blacklist' into 'master'

Document & extend blacklist

See merge request !41
parents b83ad5c2 374ce79d
Pipeline #42935 passed with stage
in 1 minute and 15 seconds
......@@ -4,7 +4,7 @@ respdiff:
target: kresd
database:
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/69e90a46226984d08e096424dd81397d/shortlist.mdb
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/f113be43534b37c60dd3628d42b2b5e6/shortlist3.mdb
dest: data.mdb
remove_after: true
......
......@@ -4,7 +4,7 @@ respdiff:
target: kresd
database:
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/69e90a46226984d08e096424dd81397d/shortlist.mdb
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/f113be43534b37c60dd3628d42b2b5e6/shortlist3.mdb
dest: data.mdb
remove_after: true
......
......@@ -4,7 +4,7 @@ respdiff:
target: kresd
database:
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/69e90a46226984d08e096424dd81397d/shortlist.mdb
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/f113be43534b37c60dd3628d42b2b5e6/shortlist3.mdb
dest: data.mdb
remove_after: true
......
......@@ -4,7 +4,7 @@ respdiff:
target: kresd
database:
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/69e90a46226984d08e096424dd81397d/shortlist.mdb
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/f113be43534b37c60dd3628d42b2b5e6/shortlist3.mdb
dest: data.mdb
remove_after: true
......
......@@ -4,7 +4,7 @@ respdiff:
target: kresd
database:
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/69e90a46226984d08e096424dd81397d/shortlist.mdb
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/f113be43534b37c60dd3628d42b2b5e6/shortlist3.mdb
dest: data.mdb
remove_after: true
......
......@@ -4,7 +4,7 @@ respdiff:
target: kresd
database:
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/69e90a46226984d08e096424dd81397d/shortlist.mdb
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/f113be43534b37c60dd3628d42b2b5e6/shortlist3.mdb
dest: data.mdb
remove_after: true
......
......@@ -4,7 +4,7 @@ respdiff:
target: kresd
database:
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/69e90a46226984d08e096424dd81397d/shortlist.mdb
url: https://gitlab.labs.nic.cz/knot/respdiff/uploads/f113be43534b37c60dd3628d42b2b5e6/shortlist3.mdb
dest: data.mdb
remove_after: true
......
......@@ -44,7 +44,7 @@ def main():
if i % 10000 == 0:
logging.info('Received {:d} answers'.format(i))
txn.put(qkey, blob)
except KeyboardInterrupt as err:
except KeyboardInterrupt:
logging.info('SIGINT received, exiting...')
sys.exit(130)
except RuntimeError as err:
......
......@@ -23,14 +23,34 @@ def read_lines(instream):
"""
Yield (line number, stripped line text, representation for logs). Skip empty lines.
"""
i = 0
i = 1
for line in instream:
if i % REPORT_CHUNKS == 0:
logging.info('Read %d lines', i)
line = line.strip()
if line:
i += 1
yield (i, line, line)
if i % REPORT_CHUNKS == 0:
logging.info('Read %d queries', i)
i += 1
def extract_wire(packet: bytes) -> bytes:
"""
Extract DNS message wire format from PCAP packet.
UDP payload is passed as it was.
TCP payload will have first two bytes removed (length prefix).
Caller must verify if return value is a valid DNS message
and decice what to do with invalid ones.
"""
frame = dpkt.ethernet.Ethernet(packet)
ip = frame.data
transport = ip.data
if isinstance(transport, dpkt.tcp.TCP):
if len(transport.data) < 2:
return transport.data
wire = transport.data[2:]
else:
wire = transport.data
return wire
def parse_pcap(pcap_file):
......@@ -38,44 +58,51 @@ def parse_pcap(pcap_file):
Filters dns query packets from pcap_file
Yield (packet number, packet as wire, representation for logs)
"""
i = 0
i = 1
pcap_file = dpkt.pcap.Reader(pcap_file)
for _, wire in pcap_file:
for _, frame in pcap_file:
if i % REPORT_CHUNKS == 0:
logging.info('Read %d frames', i)
yield (i, frame, 'frame no. {}'.format(i))
i += 1
yield (i, wire, '')
def wrk_process_line(
args: Tuple[int, str, str]
) -> Tuple[Optional[bytes], Optional[bytes]]:
) -> Tuple[Optional[int], Optional[bytes]]:
"""
Worker: parse input line, creates a packet in binary format
Skips over empty lines, raises for malformed inputs.
Skips over malformed inputs.
"""
qid, line, _ = args
qid, line, log_repr = args
try:
wire = wire_from_text(line)
msg = msg_from_text(line)
if blacklist.is_blacklisted(msg):
logging.debug('Blacklisted query "%s", skipping QID %d',
log_repr, qid)
return None, None
return qid, msg.to_wire()
except (ValueError, struct.error, dns.exception.DNSException) as ex:
logging.error('Invalid query "%s": %s (skipping query ID %d)', line, ex, qid)
logging.error('Invalid query specification "%s": %s, skipping QID %d', line, ex, qid)
return None, None
return wrk_process_wire_packet(qid, wire, line)
def wrk_process_packet(args: Tuple[int, bytes, str]):
def wrk_process_frame(args: Tuple[int, bytes, str]) -> Tuple[Optional[int], Optional[bytes]]:
"""
Worker: convert packet from pcap to binary data
"""
qid, wire, log_repr = args
wrk_process_wire_packet(qid, wire, log_repr)
qid, frame, log_repr = args
wire = extract_wire(frame)
return wrk_process_wire_packet(qid, wire, log_repr)
def wrk_process_wire_packet(
qid: int,
wire_packet: bytes,
log_repr: str
) -> Tuple[Optional[bytes], Optional[bytes]]:
) -> Tuple[Optional[int], Optional[bytes]]:
"""
Worker: Return packet's data if it's not blacklisted
......@@ -83,14 +110,17 @@ def wrk_process_wire_packet(
:arg wire_packet packet in binary data
:arg log_repr representation of packet for logs
"""
if not blacklist.is_blacklisted(wire_packet):
key = qid2key(qid)
return key, wire_packet
logging.debug('Query "%s" blacklisted (skipping query ID %d)',
log_repr if log_repr else repr(blacklist.extract_packet(wire_packet)),
qid)
return None, None
try:
msg = dns.message.from_wire(wire_packet)
except dns.exception.DNSException:
# pass invalid blobs to LMDB (for testing non-standard states)
pass
else:
if blacklist.is_blacklisted(msg):
logging.debug('Blacklisted query "%s", skipping QID %d',
log_repr, qid)
return None, None
return qid, wire_packet
def int_or_fromtext(value, fromtext):
......@@ -100,19 +130,22 @@ def int_or_fromtext(value, fromtext):
return fromtext(value)
def wire_from_text(text):
def msg_from_text(text):
"""
Convert line from <qname> <RR type> to DNS query in IN class.
Returns: DNS packet in binary form
Raises: ValueError or dns.exception.Exception on invalid input
"""
qname, qtype = text.rsplit(None, 1)
try:
qname, qtype = text.split()
except ValueError:
raise ValueError('space is only allowed as separator between qname qtype')
qname = dns.name.from_text(qname.encode('ascii'))
qtype = int_or_fromtext(qtype, dns.rdatatype.from_text)
msg = dns.message.make_query(qname, qtype, dns.rdataclass.IN,
want_dnssec=True, payload=4096)
return msg.to_wire()
return msg
def main():
......@@ -150,14 +183,15 @@ def main():
method = wrk_process_line
elif args.in_format == 'pcap':
data_stream = parse_pcap(args.pcap_file)
method = wrk_process_packet
method = wrk_process_frame
else:
logging.error('unknown in-format, use "text" or "pcap"')
sys.exit(1)
for key, wire in workers.imap(method, data_stream, chunksize=1000):
if key is not None:
for qid, wire in workers.imap(method, data_stream, chunksize=1000):
if qid is not None:
key = qid2key(qid)
txn.put(key, wire)
except KeyboardInterrupt as err:
except KeyboardInterrupt:
logging.info('SIGINT received, exiting...')
sys.exit(130)
except RuntimeError as err:
......
import dpkt
import dns
from dns.message import Message, from_wire
from dns.message import Message
# dotnxdomain.net and dashnxdomain.net are used by APNIC for ephemeral
# single-query tests so there is no point in asking these repeatedly
_BLACKLIST_SUBDOMAINS = [dns.name.from_text(name) for name in
['dotnxdomain.net.', 'dashnxdomain.net.']]
def extract_packet(packet: bytes) -> Message:
"""
Extract packet from bytes. Return dns.Message
"""
frame = dpkt.ethernet.Ethernet(packet)
ip = frame.data
transport = ip.data
if transport.data == b'':
return True
if isinstance(transport, dpkt.tcp.TCP):
wire = transport.data[2:]
else:
wire = transport.data
dnsmsg = from_wire(wire)
return dnsmsg
def is_blacklisted(packet: bytes) -> bool:
def is_blacklisted(dnsmsg: Message) -> bool:
"""
Detect if given packet is blacklisted or not.
"""
try:
dnsmsg = extract_packet(packet)
flags = dns.flags.to_text(dnsmsg.flags).split()
if 'QR' in flags: # not a query
return True
dnspacket = dnsmsg.question[0]
if dnspacket.rdtype == dns.rdatatype.ANY:
if len(dnsmsg.question) != 1:
# weird but valid packet (maybe DNS Cookies)
return False
question = dnsmsg.question[0]
# there is not standard describing common behavior for ANY/RRSIG query
if question.rdtype in {dns.rdatatype.ANY, dns.rdatatype.RRSIG}:
return True
return False
return any(question.name.is_subdomain(name)
for name in _BLACKLIST_SUBDOMAINS)
except Exception:
# weird stuff, it's better to test resolver with this as well!
return False
......@@ -25,7 +25,7 @@ def restart_resolver(script_path: str) -> None:
except subprocess.CalledProcessError as exc:
logging.warning('Resolver restart failed (exit code %d): %s',
exc.returncode, script_path)
except PermissionError as exc:
except PermissionError:
logging.warning('Resolver restart failed (permission error): %s',
script_path)
......
import binascii
import pytest
from qprep import wrk_process_frame, wrk_process_wire_packet
@pytest.mark.parametrize('wire', [
b'',
b'x',
b'xx',
])
def test_wire_input_invalid(wire):
assert wrk_process_wire_packet(1, wire, 'invalid') == (1, wire)
assert wrk_process_wire_packet(1, wire, 'invalid') == (1, wire)
@pytest.mark.parametrize('wire_hex', [
# www.audioweb.cz A
'ed21010000010000000000010377777708617564696f77656202637a00000100010000291000000080000000',
])
def test_wire_input_valid(wire_hex):
wire_in = binascii.unhexlify(wire_hex)
qid, wire_out = wrk_process_wire_packet(1, wire_in, 'qid 1')
assert wire_in == wire_out
assert qid == 1
@pytest.mark.parametrize('wire_hex', [
# test.dotnxdomain.net. A
('ce970120000100000000000104746573740b646f746e78646f6d61696e036e657400000'
'10001000029100000000000000c000a00084a69fef0f174d87e'),
# 0es-u2af5c077-c56-s1492621913-i00000000.eue.dotnxdomain.net A
('d72f01000001000000000001273065732d7532616635633037372d6335362d733134393'
'23632313931332d693030303030303030036575650b646f746e78646f6d61696e036e65'
'7400000100010000291000000080000000'),
])
def test_pcap_input_blacklist(wire_hex):
wire = binascii.unhexlify(wire_hex)
assert wrk_process_wire_packet(1, wire, 'qid 1') == (None, None)
@pytest.mark.parametrize('frame_hex, wire_hex', [
# UPD nic.cz A
('deadbeefcafecafebeefbeef08004500004bf9d000004011940d0202020201010101b533003500375520',
'b90001200001000000000001036e696302637a0000010001000029100000000000000c000a00081491f8'
'93b0c90b2f'),
# TCP nic.cz A
('deadbeefcafebeefbeefcafe080045000059e2f2400040066ae80202020201010101ace7003557b51707'
'47583400501800e5568c0000002f', '49e501200001000000000001036e696302637a00000100010000'
'29100000000000000c000a0008a1db546e1d6fa39f'),
])
def test_wrk_process_frame(frame_hex, wire_hex):
data = binascii.unhexlify(frame_hex + wire_hex)
wire = binascii.unhexlify(wire_hex)
assert wrk_process_frame((1, data, 'qid 1')) == (1, wire)
import dns.message
import dns.rdataclass
import dns.rdatatype
import dns.rrset
import pytest
from qprep import wrk_process_line
@pytest.mark.parametrize('line', [
'',
'x'*256 + ' A',
'\123x.test. 65536',
'\321.test. 1',
'test. A,AAAA',
'test. A, AAAA',
])
def test_text_input_invalid(line):
assert wrk_process_line((1, line, line)) == (None, None)
@pytest.mark.parametrize('qname, qtype', [
('x', 'A'),
('x', 1),
('blabla.test.', 'TSIG'),
])
def test_text_input_valid(qname, qtype):
line = '{} {}'.format(qname, qtype)
if isinstance(qtype, int):
rdtype = qtype
else:
rdtype = dns.rdatatype.from_text(qtype)
expected = [dns.rrset.RRset(dns.name.from_text(qname), dns.rdataclass.IN, rdtype)]
qid, wire = wrk_process_line((1, line, line))
msg = dns.message.from_wire(wire)
assert msg.question == expected
assert qid == 1
@pytest.mark.parametrize('line', [
'test. ANY',
'test. RRSIG',
'dotnxdomain.net. 28',
'something.dotnxdomain.net. A',
'something.dashnxdomain.net. AAAA',
])
def test_text_input_blacklist(line):
assert wrk_process_line((1, line, line)) == (None, None)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment