Merge branch 'bugfixes' into 'master'

Bugfixes

See merge request !7

allinone.py

 #!/usr/bin/python3
 import glob
 import logging
+import multiprocessing
 import sys
 
 import dns.name
@@ -14,6 +15,7 @@ import testedns
 import zone2pickle
 
 def main():
+    multiprocessing.set_start_method('forkserver')
     logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
 
     try:

domain2ipset.py

@@ -9,7 +9,7 @@ script sends queries to IP addresses from glue.
 import collections
 import enum
 import logging
-from multiprocessing import Pool
+import multiprocessing
 import pickle
 from typing import Counter, Deque, Dict, Iterable, Set, Tuple
 
@@ -158,7 +158,7 @@ def update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset):
     #candidates = gen_candidates(domain2nsset, nsname2ipset, netstats, retry_queue, domain2ipset)
     #logging.info('queue contains %s queries to be checked', count_candidates(candidates))
     candidates = gen_candidates(domain2nsset, nsname2ipset, netstats, retry_queue, domain2ipset)
-    with Pool(processes = 30) as pool:
+    with multiprocessing.Pool(processes = 30) as pool:
         for attempt, domain, ip, state in pool.imap_unordered(check_availability, candidates):
             process_reply(attempt, domain, ip, state, netstats, retry_queue, domain2ipset)
             if len(domain2ipset) % 1000 == 0 and len(domain2ipset.get(domain, [])) == 1:
@@ -173,6 +173,7 @@ def update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset):
 
 if __name__ == '__main__':
     logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    multiprocessing.set_start_method('forkserver')
     domain2nsset, nsname2ipset, netstats, domain2ipset = load()
     try:
         update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset)

ednscomp2pickle.py

@@ -14,7 +14,7 @@ from evalzone import EDNSResult
 
 def parse_nsip_line(line: str) -> Tuple[str, Dict[str, List[str]]]:
     """parse one line from ednscomp log"""
-    matches = re.match('^[^ ]+\\. @(?P<ip>[^ ]+) \\([^)]+\\): (?P<results>dns=.*)$', line)
+    matches = re.match('^[^ ]*\\. @(?P<ip>[^ ]+) \\([^)]+\\): (?P<results>dns=.*)$', line)
     if not matches:
         raise ValueError('line "{}" does not have expected format, skipping'.format(line))
 

genednscomp.py

@@ -10,26 +10,28 @@ import dns.name
 
 import dataapi
 
-def get_ns_name_from_ip(ipaddr: str, nsname2ipset: Dict[dns.name.Name, Set[str]]):
+def gen_ip_to_nsname(nsname2ipset: Dict[dns.name.Name, Set[str]]) -> Dict[str, dns.name.Name]:
     """
-    It should never happen that ipaddr is not present in nsname2ipset mapping.
-    If it happens, it is most likely caused by mixing data files from
-    two versions of DNS zone. Start from scratch!
+    Generate reverse mappping IP -> NS name.
     """
-    for name, ipset in nsname2ipset.items():
-        if ipaddr in ipset:
-            return name
-    raise ValueError('IP addr {} not found in nsname2ipset'.format(ipaddr))
+    ip2nsname = {}
+    for nsname, ipset in nsname2ipset.items():
+        ip2nsname.update({ip: nsname for ip in ipset})
+    return ip2nsname
 
 def generate(nsname2ipset, domain2ipset):
     """Output format is: zone nsname NSipAddress"""
     ip_done = set()
+    ip2nsname = gen_ip_to_nsname(nsname2ipset)
 
     for domain, ipset in domain2ipset.items():
         for ipaddr in ipset:
             if ipaddr in ip_done:
                 continue
-            nsname = get_ns_name_from_ip(ipaddr, nsname2ipset)
+            # It should never happen that ipaddr is not present in nsname2ipset mapping.
+            # If it happens, it is most likely caused by mixing data files from
+            # two versions of DNS zone. Start from scratch!
+            nsname = ip2nsname[ipaddr]
             yield '{} {} {}\n'.format(domain, nsname, ipaddr)
             ip_done.add(ipaddr)
 

nsname2ipset.py

 #!/usr/bin/python3
 
 import logging
-from multiprocessing.pool import Pool
+import multiprocessing
 import pickle
 from typing import Dict, Set, Tuple
 
@@ -60,7 +60,7 @@ def update_mapping(nsnames: Set[dns.name.Name],
     dns.resolver.default_resolver.lifetime = 5  # seconds
     #dns.resolver.default_resolver.nameservers = ['193.29.206.206']
 
-    with Pool(processes=128) as p:
+    with multiprocessing.Pool(processes=128) as p:
         i = 0
         logging.info('starting DNS query machinery')
         for nsname, ipset in p.imap_unordered(get_ips, yield_ns_name(nsnames, mapping), chunksize=10):
@@ -77,6 +77,7 @@ def save(mapping: Dict[dns.name.Name, Set[str]]) -> None:
 
 if __name__ == "__main__":
     logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
+    multiprocessing.set_start_method('forkserver')
     nsnames = load_nsnames()
     mapping = load_nsname2ipset()
     try: