introduce IP address normalization

It turns out that genreport and dnspython use different string formats
for IPv6 addresses and this is causing problems in certain special
cases.

Thanks to Sven Van Dyck <svenvd@dnsbelgium.be> for reporting this!

ci/test.sh

 #!/usr/bin/bash
-set -o errexit
+set -o errexit -o xtrace
 
 test -f zone || wget -O zone https://www.internic.net/domain/in-addr.arpa
 zone2pickle.py zone in-addr.arpa

dataapi.py

@@ -7,8 +7,9 @@ import pickle
 from typing import Dict, Set
 
 import dns.name
+from evalzone import AnIPAddress
 
-def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
+def load_nsname2ipset() -> Dict[dns.name.Name, Set[AnIPAddress]]:
     """raises FileNotFoundError"""
     logging.info('loading NS name -> IP address mapping')
     with open('nsname2ipset.pickle', 'rb') as nsname2ipset_pickle:
@@ -18,14 +19,14 @@ def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
                  len(ip_all), len(nsname2ipset))
     return nsname2ipset
 
-def load_domain2ipset() -> Dict[dns.name.Name, Set[str]]:
+def load_domain2ipset() -> Dict[dns.name.Name, Set[AnIPAddress]]:
     """raises FileNotFoundError"""
     logging.info('loading domain-IP sets')
     with open('domain2ipset.pickle', 'rb') as domain2ipset_pickle:
         domain2ipset = pickle.load(domain2ipset_pickle)
     return domain2ipset
 
-def save_nsname2ipset(nsname2ipset: Dict[dns.name.Name, Set[str]]) -> None:
+def save_nsname2ipset(nsname2ipset: Dict[dns.name.Name, Set[AnIPAddress]]) -> None:
     '''TODO: skip ip_cnt computation if not in verbose mode?'''
     if nsname2ipset:
         ip_cnt = len(set.union(*nsname2ipset.values()))

domain2ipset.py

@@ -16,6 +16,8 @@ from typing import Counter, Deque, Dict, Iterable, Set, Tuple
 import dns.message
 import dns.query
 
+from evalzone import AnIPAddress
+
 class IP_state(enum.Enum):
     timeout = 0
     notauth = 1  # refused etc.
@@ -23,17 +25,17 @@ class IP_state(enum.Enum):
 
 class NetStats():
     def __init__(self):
-        self.netstats = {}  # type: Dict[str, Counter[str]]
+        self.netstats = {}  # type: Dict[AnIPAddress, Counter[str]]
 
     def __len__(self):
         return len(self.netstats)
 
-    def is_ip_dead(self, ip: str) -> bool:
+    def is_ip_dead(self, ip: AnIPAddress) -> bool:
         if not ip in self.netstats:
             self.netstats[ip] = collections.Counter()
         return self.netstats[ip]['timeouts_in_row'] >= 10
 
-    def record_ip(self, ip: str, state: IP_state) -> None:
+    def record_ip(self, ip: AnIPAddress, state: IP_state) -> None:
         if state == IP_state.timeout:
             self.netstats[ip]['timeouts'] += 1
             self.netstats[ip]['timeouts_in_row'] += 1
@@ -52,7 +54,7 @@ def count_candidates(iterable):
         i += 1
     return i
 
-def check_availability(args: Tuple[int, dns.name.Name, str]) -> Tuple[int, dns.name.Name, str, IP_state]:
+def check_availability(args: Tuple[int, dns.name.Name, AnIPAddress]) -> Tuple[int, dns.name.Name, AnIPAddress, IP_state]:
     attempt, domain, ip = args
     """
     check if server replies to domain. NS query
@@ -60,7 +62,7 @@ def check_availability(args: Tuple[int, dns.name.Name, str]) -> Tuple[int, dns.n
     query = dns.message.make_query(domain, dns.rdatatype.NS)
     state = IP_state.notauth
     try:
-        answer = dns.query.udp(query, ip, timeout=2)
+        answer = dns.query.udp(query, str(ip), timeout=2)
         if answer.rcode() == dns.rcode.NOERROR:
             state = IP_state.ok
     except dns.exception.Timeout:
@@ -72,11 +74,11 @@ def check_availability(args: Tuple[int, dns.name.Name, str]) -> Tuple[int, dns.n
         return (attempt, domain, ip, state)
 
 def gen_candidates(domain2nsset: Dict[dns.name.Name, Set[dns.name.Name]],
-                   nsname2ipset: Dict[dns.name.Name, Set[str]],
+                   nsname2ipset: Dict[dns.name.Name, Set[AnIPAddress]],
                    netstats: NetStats,
-                   retry_queue: Deque[Tuple[int, dns.name.Name, str]],
-                   domain2ipset: Dict[dns.name.Name, Set[str]]
-                  ) -> Iterable[Tuple[int, dns.name.Name, str]]:
+                   retry_queue: Deque[Tuple[int, dns.name.Name, AnIPAddress]],
+                   domain2ipset: Dict[dns.name.Name, Set[AnIPAddress]]
+                  ) -> Iterable[Tuple[int, dns.name.Name, AnIPAddress]]:
     for domain, nsset in domain2nsset.items():
         if len(retry_queue) > 100:  # prevent deque from unlimited growth
             logging.info('retry queue full, processing queued retries now')
@@ -96,9 +98,9 @@ def gen_candidates(domain2nsset: Dict[dns.name.Name, Set[dns.name.Name]],
     # gen_candidates generator might pre-generate results and terminate
     # before retries are even generated
 
-def retry_candidates(retry_queue: Deque[Tuple[int, dns.name.Name, str]],
+def retry_candidates(retry_queue: Deque[Tuple[int, dns.name.Name, AnIPAddress]],
                      netstats: NetStats
-                    ) -> Iterable[Tuple[int, dns.name.Name, str]]:
+                    ) -> Iterable[Tuple[int, dns.name.Name, AnIPAddress]]:
     while retry_queue:
         attempt, domain, ip = retry_queue.popleft()
         if netstats.is_ip_dead(ip):

ednscomp2pickle.py

 #!/usr/bin/python3
 
 import collections
+import ipaddress
 import logging
 import pickle
 import re
 import sys
 from typing import Counter, Dict, List, Tuple
 
-from evalzone import EDNSResult
+from evalzone import EDNSResult, AnIPAddress
 
 # zsstmesto.cz. @89.187.140.136 (01.dns.services.dmdox.com.): dns=ok edns=ok edns1=noerror,badversion,soa edns@512=ok ednsopt=ok edns1opt=noerror,badversion,soa do=ok ednsflags=ok optlist=ok,nsid signed=ok ednstcp=ok
 # seznam.cz. @2a02:598:4444::4 (ams.seznam.cz.): dns=ok edns=ok,nsid edns1=noerror,badversion,soa,nsid edns@512=ok ednsopt=ok,nsid edns1opt=noerror,badversion,soa,nsid do=ok ednsflags=ok,nsid optlist=ok,nsid signed=ok ednstcp=ok
 
-def parse_nsip_line(line: str) -> Tuple[str, Dict[str, List[str]]]:
+def parse_nsip_line(line: str) -> Tuple[AnIPAddress, Dict[str, List[str]]]:
     """parse one line from ednscomp log"""
     matches = re.match('^[^ ]*\\. @(?P<ip>[^ ]+) \\([^)]+\\): (?P<results>dns=.*)$', line)
     if not matches:
@@ -25,7 +26,7 @@ def parse_nsip_line(line: str) -> Tuple[str, Dict[str, List[str]]]:
                          for test in tests_list}
     except IndexError:
         raise ValueError('skipping nonsense test results "{}"'.format(line))
-    return matches.group("ip"), tests_results
+    return ipaddress.ip_address(matches.group("ip")), tests_results
 
 def eval_edns_strict(edns0_results: Dict[str, List[str]]) -> EDNSResult:
     """
@@ -60,11 +61,11 @@ def eval_edns(tests_results: Dict[str, List[str]], timeout_evaluator) -> EDNSRes
     else:  # impact of timeouts depend is different before and after the flag day
         return timeout_evaluator(edns0_results)
 
-def collect_server_stats(eval_edns_func, edns_infns: str) -> Dict[str, Counter[EDNSResult]]:
+def collect_server_stats(eval_edns_func, edns_infns: str) -> Dict[AnIPAddress, Counter[EDNSResult]]:
     """
     Combine results from all files with ednscomp output and summarize stats
     """
-    server_stats = {}  # type: Dict[str, Counter[EDNSResult]]
+    server_stats = {}  # type: Dict[AnIPAddress, Counter[EDNSResult]]
     i = 1
     for infilename in edns_infns:
         logging.info('processed file no. {}, file name "{}"'.format(i, infilename))

evalzone.py

@@ -5,12 +5,17 @@ produce EDNS stats summary from pickled statistical data
 """
 from enum import IntEnum
 import collections
+import ipaddress
 import logging
 import pickle
 from typing import Counter, Dict, List, Iterable, Iterator, NamedTuple, Set, Union
 
 import dns.name
 
+
+AnIPAddress = Union[ipaddress.IPv4Address, ipaddress.IPv6Address]
+
+
 class EDNSResult(IntEnum):
     """
     EDNS evaluation result for single zone

genednscomp.py

@@ -9,8 +9,9 @@ from typing import Dict, Set
 import dns.name
 
 import dataapi
+from evalzone import AnIPAddress
 
-def gen_ip_to_nsname(nsname2ipset: Dict[dns.name.Name, Set[str]]) -> Dict[str, dns.name.Name]:
+def gen_ip_to_nsname(nsname2ipset: Dict[dns.name.Name, Set[AnIPAddress]]) -> Dict[AnIPAddress, dns.name.Name]:
     """
     Generate reverse mappping IP -> NS name.
     """

nsname2ipset.py

 #!/usr/bin/python3
 
+import ipaddress
 import logging
 import multiprocessing
 import pickle
@@ -9,6 +10,9 @@ import dns.name
 import dns.rdatatype
 import dns.resolver
 
+from evalzone import AnIPAddress
+
+
 def yield_ns_name(nsnames, mapping):
     """
     returns: stream of NS names
@@ -17,7 +21,7 @@ def yield_ns_name(nsnames, mapping):
         if nsname not in mapping:  # no ips at all
             yield nsname
 
-def resolve(qname: dns.name.Name, qtype) -> Set[str]:
+def resolve(qname: dns.name.Name, qtype) -> Set[AnIPAddress]:
     logging.debug('resolving %s %s', qname, qtype)
     try:
         answer = dns.resolver.query(qname, qtype)
@@ -27,10 +31,10 @@ def resolve(qname: dns.name.Name, qtype) -> Set[str]:
         #logging.exception(ex)
         raise
     else:
-        return set(ip.address for ip in answer)
+        return set(ipaddress.ip_address(ip.address) for ip in answer)
 
-def get_ips(nsname: dns.name.Name) -> Tuple[dns.name.Name, Set[str]]:
-    ips = set()  # type: Set[str]
+def get_ips(nsname: dns.name.Name) -> Tuple[dns.name.Name, Set[AnIPAddress]]:
+    ips = set()  # type: Set[AnIPAddress]
     try:
         ips = resolve(nsname, dns.rdatatype.A)
         ips = ips.union(resolve(nsname, dns.rdatatype.AAAA))
@@ -44,8 +48,8 @@ def load_nsnames() -> Set[dns.name.Name]:
         nslist = pickle.load(nslist_file)
     logging.info('loaded %s NS names', len(nslist))
     return nslist
-    
-def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
+
+def load_nsname2ipset() -> Dict[dns.name.Name, Set[AnIPAddress]]:
     logging.info('loading previous mapping')
     try:
         with open('nsname2ipset.pickle', 'rb') as nsname2ipset_file:
@@ -55,7 +59,7 @@ def load_nsname2ipset() -> Dict[dns.name.Name, Set[str]]:
     return mapping
 
 def update_mapping(nsnames: Set[dns.name.Name],
-         mapping: Dict[dns.name.Name, Set[str]]) -> None:
+         mapping: Dict[dns.name.Name, Set[AnIPAddress]]) -> None:
     # workaround for mypy 0.650 which thinks that call to
     # reset_default_resolver() does not guarantee existence
     # of a default_resolver instance
@@ -74,7 +78,7 @@ def update_mapping(nsnames: Set[dns.name.Name],
             if ipset:
                 mapping[nsname] = ipset
 
-def save(mapping: Dict[dns.name.Name, Set[str]]) -> None:
+def save(mapping: Dict[dns.name.Name, Set[AnIPAddress]]) -> None:
     logging.info('writting %d results', len(mapping))
     pickle.dump(mapping, open('nsname2ipset.pickle', 'wb'))
 

zone2pickle.py

@@ -3,6 +3,7 @@
 Tranform DNS zone file into pickled Python objects.
 """
 
+import ipaddress
 import logging
 import pickle
 import sys
@@ -11,6 +12,7 @@ from typing import Dict, List, Iterable, Set, Tuple
 import dns.zone
 
 import dataapi
+from evalzone import AnIPAddress
 
 def domain2nsset(zoneobj: dns.zone.Zone) -> Dict[dns.name.Name, Set[dns.name.Name]]:
     '''
@@ -28,15 +30,15 @@ def uniq_nslist(nssets: Iterable[Set[dns.name.Name]]) -> Set[dns.name.Name]:
     return uniq_ns
 
 def glue_ns2ipset(nslist: Set[dns.name.Name], zoneobj: dns.zone.Zone) \
-            -> Dict[dns.name.Name, Set[str]]:
+            -> Dict[dns.name.Name, Set[AnIPAddress]]:
     '''
     NS names without glue addresses will not show up in output
     '''
-    ns2ipset = {}  # type: Dict[dns.name.Name, Set[str]]
+    ns2ipset = {}  # type: Dict[dns.name.Name, Set[AnIPAddress]]
     for nsname in nslist:
         if nsname in ns2ipset:  # optimization, do not redo it
             continue
-        ipset = set()  # type: Set[str]
+        ipset = set()  # type: Set[AnIPAddress]
         try:
             node = zoneobj[nsname]
         except KeyError:  # glue name not in zone
@@ -44,7 +46,8 @@ def glue_ns2ipset(nslist: Set[dns.name.Name], zoneobj: dns.zone.Zone) \
         for rdtype in (dns.rdatatype.A, dns.rdatatype.AAAA):
             rrset = node.get_rdataset(dns.rdataclass.IN, rdtype)
             if rrset:
-                ipset = ipset.union(set(ip.address for ip in rrset))
+                ipset = ipset.union(set(ipaddress.ip_address(ip.address)
+                                    for ip in rrset))
         if ipset:  # do not store glueless names
             ns2ipset[nsname] = ipset
     return ns2ipset
@@ -52,7 +55,7 @@ def glue_ns2ipset(nslist: Set[dns.name.Name], zoneobj: dns.zone.Zone) \
 def convert(zone_fn: str, zone_origin: dns.name.Name) -> Tuple[ \
             Dict[dns.name.Name, Set[dns.name.Name]], \
             Set[dns.name.Name], \
-            Dict[dns.name.Name, Set[str]]]:
+            Dict[dns.name.Name, Set[AnIPAddress]]]:
     '''
     convert text zone into set of pickle files with preprocessed metadata
     '''