Commit 68a8fd68 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'ipaddr_norm' into 'master'

Use IPAddress objects instead of strings

See merge request !12
parents cfed1905 638ca637
Pipeline #44289 passed with stages
in 8 minutes and 36 seconds
#!/usr/bin/bash
set -o errexit
set -o errexit -o xtrace
test -f zone || wget -O zone https://www.internic.net/domain/in-addr.arpa
zone2pickle.py zone in-addr.arpa
......
......@@ -7,8 +7,9 @@ import pickle
from typing import Dict, Set
import dns.name
from evalzone import AnIPAddress
def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
def load_nsname2ipset() -> Dict[dns.name.Name, Set[AnIPAddress]]:
"""raises FileNotFoundError"""
logging.info('loading NS name -> IP address mapping')
with open('nsname2ipset.pickle', 'rb') as nsname2ipset_pickle:
......@@ -18,14 +19,14 @@ def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
len(ip_all), len(nsname2ipset))
return nsname2ipset
def load_domain2ipset() -> Dict[dns.name.Name, Set[str]]:
def load_domain2ipset() -> Dict[dns.name.Name, Set[AnIPAddress]]:
"""raises FileNotFoundError"""
logging.info('loading domain-IP sets')
with open('domain2ipset.pickle', 'rb') as domain2ipset_pickle:
domain2ipset = pickle.load(domain2ipset_pickle)
return domain2ipset
def save_nsname2ipset(nsname2ipset: Dict[dns.name.Name, Set[str]]) -> None:
def save_nsname2ipset(nsname2ipset: Dict[dns.name.Name, Set[AnIPAddress]]) -> None:
'''TODO: skip ip_cnt computation if not in verbose mode?'''
if nsname2ipset:
ip_cnt = len(set.union(*nsname2ipset.values()))
......
......@@ -16,6 +16,8 @@ from typing import Counter, Deque, Dict, Iterable, Set, Tuple
import dns.message
import dns.query
from evalzone import AnIPAddress
class IP_state(enum.Enum):
timeout = 0
notauth = 1 # refused etc.
......@@ -23,17 +25,17 @@ class IP_state(enum.Enum):
class NetStats():
def __init__(self):
self.netstats = {} # type: Dict[str, Counter[str]]
self.netstats = {} # type: Dict[AnIPAddress, Counter[str]]
def __len__(self):
return len(self.netstats)
def is_ip_dead(self, ip: str) -> bool:
def is_ip_dead(self, ip: AnIPAddress) -> bool:
if not ip in self.netstats:
self.netstats[ip] = collections.Counter()
return self.netstats[ip]['timeouts_in_row'] >= 10
def record_ip(self, ip: str, state: IP_state) -> None:
def record_ip(self, ip: AnIPAddress, state: IP_state) -> None:
if state == IP_state.timeout:
self.netstats[ip]['timeouts'] += 1
self.netstats[ip]['timeouts_in_row'] += 1
......@@ -52,7 +54,7 @@ def count_candidates(iterable):
i += 1
return i
def check_availability(args: Tuple[int, dns.name.Name, str]) -> Tuple[int, dns.name.Name, str, IP_state]:
def check_availability(args: Tuple[int, dns.name.Name, AnIPAddress]) -> Tuple[int, dns.name.Name, AnIPAddress, IP_state]:
attempt, domain, ip = args
"""
check if server replies to domain. NS query
......@@ -60,7 +62,7 @@ def check_availability(args: Tuple[int, dns.name.Name, str]) -> Tuple[int, dns.n
query = dns.message.make_query(domain, dns.rdatatype.NS)
state = IP_state.notauth
try:
answer = dns.query.udp(query, ip, timeout=2)
answer = dns.query.udp(query, str(ip), timeout=2)
if answer.rcode() == dns.rcode.NOERROR:
state = IP_state.ok
except dns.exception.Timeout:
......@@ -72,11 +74,11 @@ def check_availability(args: Tuple[int, dns.name.Name, str]) -> Tuple[int, dns.n
return (attempt, domain, ip, state)
def gen_candidates(domain2nsset: Dict[dns.name.Name, Set[dns.name.Name]],
nsname2ipset: Dict[dns.name.Name, Set[str]],
nsname2ipset: Dict[dns.name.Name, Set[AnIPAddress]],
netstats: NetStats,
retry_queue: Deque[Tuple[int, dns.name.Name, str]],
domain2ipset: Dict[dns.name.Name, Set[str]]
) -> Iterable[Tuple[int, dns.name.Name, str]]:
retry_queue: Deque[Tuple[int, dns.name.Name, AnIPAddress]],
domain2ipset: Dict[dns.name.Name, Set[AnIPAddress]]
) -> Iterable[Tuple[int, dns.name.Name, AnIPAddress]]:
for domain, nsset in domain2nsset.items():
if len(retry_queue) > 100: # prevent deque from unlimited growth
logging.info('retry queue full, processing queued retries now')
......@@ -96,9 +98,9 @@ def gen_candidates(domain2nsset: Dict[dns.name.Name, Set[dns.name.Name]],
# gen_candidates generator might pre-generate results and terminate
# before retries are even generated
def retry_candidates(retry_queue: Deque[Tuple[int, dns.name.Name, str]],
def retry_candidates(retry_queue: Deque[Tuple[int, dns.name.Name, AnIPAddress]],
netstats: NetStats
) -> Iterable[Tuple[int, dns.name.Name, str]]:
) -> Iterable[Tuple[int, dns.name.Name, AnIPAddress]]:
while retry_queue:
attempt, domain, ip = retry_queue.popleft()
if netstats.is_ip_dead(ip):
......
#!/usr/bin/python3
import collections
import ipaddress
import logging
import pickle
import re
import sys
from typing import Counter, Dict, List, Tuple
from evalzone import EDNSResult
from evalzone import EDNSResult, AnIPAddress
# zsstmesto.cz. @89.187.140.136 (01.dns.services.dmdox.com.): dns=ok edns=ok edns1=noerror,badversion,soa edns@512=ok ednsopt=ok edns1opt=noerror,badversion,soa do=ok ednsflags=ok optlist=ok,nsid signed=ok ednstcp=ok
# seznam.cz. @2a02:598:4444::4 (ams.seznam.cz.): dns=ok edns=ok,nsid edns1=noerror,badversion,soa,nsid edns@512=ok ednsopt=ok,nsid edns1opt=noerror,badversion,soa,nsid do=ok ednsflags=ok,nsid optlist=ok,nsid signed=ok ednstcp=ok
def parse_nsip_line(line: str) -> Tuple[str, Dict[str, List[str]]]:
def parse_nsip_line(line: str) -> Tuple[AnIPAddress, Dict[str, List[str]]]:
"""parse one line from ednscomp log"""
matches = re.match('^[^ ]*\\. @(?P<ip>[^ ]+) \\([^)]+\\): (?P<results>dns=.*)$', line)
if not matches:
......@@ -25,7 +26,7 @@ def parse_nsip_line(line: str) -> Tuple[str, Dict[str, List[str]]]:
for test in tests_list}
except IndexError:
raise ValueError('skipping nonsense test results "{}"'.format(line))
return matches.group("ip"), tests_results
return ipaddress.ip_address(matches.group("ip")), tests_results
def eval_edns_strict(edns0_results: Dict[str, List[str]]) -> EDNSResult:
"""
......@@ -60,11 +61,11 @@ def eval_edns(tests_results: Dict[str, List[str]], timeout_evaluator) -> EDNSRes
else: # impact of timeouts depend is different before and after the flag day
return timeout_evaluator(edns0_results)
def collect_server_stats(eval_edns_func, edns_infns: str) -> Dict[str, Counter[EDNSResult]]:
def collect_server_stats(eval_edns_func, edns_infns: str) -> Dict[AnIPAddress, Counter[EDNSResult]]:
"""
Combine results from all files with ednscomp output and summarize stats
"""
server_stats = {} # type: Dict[str, Counter[EDNSResult]]
server_stats = {} # type: Dict[AnIPAddress, Counter[EDNSResult]]
i = 1
for infilename in edns_infns:
logging.info('processed file no. {}, file name "{}"'.format(i, infilename))
......
......@@ -5,12 +5,17 @@ produce EDNS stats summary from pickled statistical data
"""
from enum import IntEnum
import collections
import ipaddress
import logging
import pickle
from typing import Counter, Dict, List, Iterable, Iterator, NamedTuple, Set, Union
import dns.name
AnIPAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address]
class EDNSResult(IntEnum):
"""
EDNS evaluation result for single zone
......
......@@ -9,8 +9,9 @@ from typing import Dict, Set
import dns.name
import dataapi
from evalzone import AnIPAddress
def gen_ip_to_nsname(nsname2ipset: Dict[dns.name.Name, Set[str]]) -> Dict[str, dns.name.Name]:
def gen_ip_to_nsname(nsname2ipset: Dict[dns.name.Name, Set[AnIPAddress]]) -> Dict[AnIPAddress, dns.name.Name]:
"""
Generate reverse mappping IP -> NS name.
"""
......
#!/usr/bin/python3
import ipaddress
import logging
import multiprocessing
import pickle
......@@ -9,6 +10,9 @@ import dns.name
import dns.rdatatype
import dns.resolver
from evalzone import AnIPAddress
def yield_ns_name(nsnames, mapping):
"""
returns: stream of NS names
......@@ -17,7 +21,7 @@ def yield_ns_name(nsnames, mapping):
if nsname not in mapping: # no ips at all
yield nsname
def resolve(qname: dns.name.Name, qtype) -> Set[str]:
def resolve(qname: dns.name.Name, qtype) -> Set[AnIPAddress]:
logging.debug('resolving %s %s', qname, qtype)
try:
answer = dns.resolver.query(qname, qtype)
......@@ -27,10 +31,10 @@ def resolve(qname: dns.name.Name, qtype) -> Set[str]:
#logging.exception(ex)
raise
else:
return set(ip.address for ip in answer)
return set(ipaddress.ip_address(ip.address) for ip in answer)
def get_ips(nsname: dns.name.Name) -> Tuple[dns.name.Name, Set[str]]:
ips = set() # type: Set[str]
def get_ips(nsname: dns.name.Name) -> Tuple[dns.name.Name, Set[AnIPAddress]]:
ips = set() # type: Set[AnIPAddress]
try:
ips = resolve(nsname, dns.rdatatype.A)
ips = ips.union(resolve(nsname, dns.rdatatype.AAAA))
......@@ -44,8 +48,8 @@ def load_nsnames() -> Set[dns.name.Name]:
nslist = pickle.load(nslist_file)
logging.info('loaded %s NS names', len(nslist))
return nslist
def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
def load_nsname2ipset() -> Dict[dns.name.Name, Set[AnIPAddress]]:
logging.info('loading previous mapping')
try:
with open('nsname2ipset.pickle', 'rb') as nsname2ipset_file:
......@@ -55,9 +59,12 @@ def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
return mapping
def update_mapping(nsnames: Set[dns.name.Name],
mapping: Dict[dns.name.Name, Set[str]]) -> None:
dns.resolver.reset_default_resolver()
dns.resolver.default_resolver.lifetime = 5 # seconds
mapping: Dict[dns.name.Name, Set[AnIPAddress]]) -> None:
# workaround for mypy 0.650 which thinks that call to
# reset_default_resolver() does not guarantee existence
# of a default_resolver instance
dns.resolver.default_resolver = dns.resolver.Resolver()
dns.resolver.default_resolver.lifetime = 5
#dns.resolver.default_resolver.nameservers = ['193.29.206.206']
with multiprocessing.Pool(processes=128) as p:
......@@ -71,7 +78,7 @@ def update_mapping(nsnames: Set[dns.name.Name],
if ipset:
mapping[nsname] = ipset
def save(mapping: Dict[dns.name.Name, Set[str]]) -> None:
def save(mapping: Dict[dns.name.Name, Set[AnIPAddress]]) -> None:
logging.info('writting %d results', len(mapping))
pickle.dump(mapping, open('nsname2ipset.pickle', 'wb'))
......
......@@ -8,14 +8,14 @@ import argparse
import logging
import pickle
import sys
from typing import Dict, Set
from typing import Dict, Optional, Set
import dns.name
from evalzone import EDNSResult
def print_domain(mode: str, result: EDNSResult, domain: dns.name.Name,
nsset: Set[dns.name.Name], reason) \
nsset: Optional[Set[dns.name.Name]], reason) \
-> None:
if not nsset:
print(mode, result, domain, ';', reason)
......@@ -24,7 +24,7 @@ def print_domain(mode: str, result: EDNSResult, domain: dns.name.Name,
print(mode, result, domain, nsname, ';', reason)
def new_domains(permissive, strict,
args, domain2ns: Dict[dns.name.Name, dns.name.Name]) -> None:
args, domain2ns: Dict[dns.name.Name, Set[dns.name.Name]]) -> None:
logging.info('computing domains with NS which are going to stop working')
edns_broken_domains = set(strict[EDNSResult.dead].keys()) \
- set(permissive[EDNSResult.dead].keys())
......
......@@ -3,6 +3,7 @@
Tranform DNS zone file into pickled Python objects.
"""
import ipaddress
import logging
import pickle
import sys
......@@ -11,8 +12,9 @@ from typing import Dict, List, Iterable, Set, Tuple
import dns.zone
import dataapi
from evalzone import AnIPAddress
def domain2nsset(zoneobj: dns.zone.Zone) -> Dict[dns.name.Name, Set[str]]:
def domain2nsset(zoneobj: dns.zone.Zone) -> Dict[dns.name.Name, Set[dns.name.Name]]:
'''
generator!
TODO: optimize, currently it requires whole zone in single object'''
......@@ -20,7 +22,7 @@ def domain2nsset(zoneobj: dns.zone.Zone) -> Dict[dns.name.Name, Set[str]]:
for domain, node in zoneobj.items()
if node.get_rdataset(dns.rdataclass.IN, dns.rdatatype.NS)}
def uniq_nslist(nssets: Iterable[Set[str]]) -> Set[dns.name.Name]:
def uniq_nslist(nssets: Iterable[Set[dns.name.Name]]) -> Set[dns.name.Name]:
'''TODO: optimize, it could be done together with domain2nsset transformation'''
uniq_ns = set() # type: Set[dns.name.Name]
for nsset in nssets:
......@@ -28,15 +30,15 @@ def uniq_nslist(nssets: Iterable[Set[str]]) -> Set[dns.name.Name]:
return uniq_ns
def glue_ns2ipset(nslist: Set[dns.name.Name], zoneobj: dns.zone.Zone) \
-> Dict[dns.name.Name, Set[str]]:
-> Dict[dns.name.Name, Set[AnIPAddress]]:
'''
NS names without glue addresses will not show up in output
'''
ns2ipset = {} # type: Dict[dns.name.Name, Set[str]]
ns2ipset = {} # type: Dict[dns.name.Name, Set[AnIPAddress]]
for nsname in nslist:
if nsname in ns2ipset: # optimization, do not redo it
continue
ipset = set() # type: Set[str]
ipset = set() # type: Set[AnIPAddress]
try:
node = zoneobj[nsname]
except KeyError: # glue name not in zone
......@@ -44,7 +46,8 @@ def glue_ns2ipset(nslist: Set[dns.name.Name], zoneobj: dns.zone.Zone) \
for rdtype in (dns.rdatatype.A, dns.rdatatype.AAAA):
rrset = node.get_rdataset(dns.rdataclass.IN, rdtype)
if rrset:
ipset = ipset.union(set(ip.address for ip in rrset))
ipset = ipset.union(set(ipaddress.ip_address(ip.address)
for ip in rrset))
if ipset: # do not store glueless names
ns2ipset[nsname] = ipset
return ns2ipset
......@@ -52,7 +55,7 @@ def glue_ns2ipset(nslist: Set[dns.name.Name], zoneobj: dns.zone.Zone) \
def convert(zone_fn: str, zone_origin: dns.name.Name) -> Tuple[ \
Dict[dns.name.Name, Set[dns.name.Name]], \
Set[dns.name.Name], \
Dict[dns.name.Name, Set[str]]]:
Dict[dns.name.Name, Set[AnIPAddress]]]:
'''
convert text zone into set of pickle files with preprocessed metadata
'''
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment