separate (a bit) EDNS tests from generic preprocessing

Idea behind this is to re-use preprocesing for TCP testing etc.

allinone.py

 #!/usr/bin/python3
+import argparse
 import glob
 import logging
 import multiprocessing
@@ -7,7 +8,7 @@ import sys
 import dns.name
 
 import domain2ipset
-import evalzone
+import ednsevalzone
 import ednscomp2pickle
 import genednscomp
 import nsname2ipset
@@ -18,14 +19,13 @@ def main():
     multiprocessing.set_start_method('forkserver')
     logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
 
-    try:
-        zone_fn = sys.argv[1]
-        zone_origin = dns.name.from_text(sys.argv[2])
-    except IndexError:
-        logging.critical('Usage: %s zone_file zone_origin', sys.argv[0])
-        sys.exit(1)
+    argparser = argparse.ArgumentParser(description='test delegations in given zone file')
+    argparser.add_argument('scan_type', choices=['edns2019'], help='criteria to use for scanner')
+    argparser.add_argument('zone_file', type=open, help='zone file in RFC 1035 format')
+    argparser.add_argument('zone_origin', type=dns.name.from_text, help='zone name, SOA RR must be present')
+    args = argparser.parse_args()
 
-    domain_nsset, nsnames, nsname_ipsets = zone2pickle.convert(zone_fn, zone_origin)
+    domain_nsset, nsnames, nsname_ipsets = zone2pickle.convert(args.zone_file, args.zone_origin)
     zone2pickle.save(domain_nsset, nsnames, nsname_ipsets)
 
     logging.info('resolving NS names to IP addresses')
@@ -84,30 +84,31 @@ def main():
                          remaining/total * 100)
     domain2ipset.save(domain_nsset, netstats, domain_ipset)
 
-    logging.info('generating input data for genreport tool')
-    with open('ednscomp.input', 'w') as ednscomp_input:
-        ednscomp_input.writelines(genednscomp.generate(nsname_ipsets, domain_ipset))
+    if args.scan_type == 'edns2019':
+        logging.info('generating input data for genreport tool')
+        with open('ednscomp.input', 'w') as ednscomp_input:
+            ednscomp_input.writelines(genednscomp.generate(nsname_ipsets, domain_ipset))
 
-    logging.info('executing EDNS tests')
-    testedns.repeat_genreport(10)
-    ednscompresults = glob.glob('ednscompresult-*')
-    if not ednscompresults:
-        logging.critical('error: no ednscompresult-* files from previous step found, exiting')
-        sys.exit(2)
+        logging.info('executing EDNS tests')
+        testedns.repeat_genreport(10)
+        ednscompresults = glob.glob('ednscompresult-*')
+        if not ednscompresults:
+            logging.critical('error: no ednscompresult-* files from previous step found, exiting')
+            sys.exit(2)
 
-    logging.info('processing genreport output in EDNS strict mode')
-    nsstats_strict = ednscomp2pickle.collect_server_stats(ednscomp2pickle.eval_edns_strict, ednscompresults)
-    ednscomp2pickle.save(nsstats_strict, 'strict')
+        logging.info('processing genreport output in EDNS strict mode')
+        nsstats_strict = ednscomp2pickle.collect_server_stats(ednscomp2pickle.eval_edns_strict, ednscompresults)
+        ednscomp2pickle.save(nsstats_strict, 'strict')
 
-    logging.info('processing genreport output in EDNS permissive mode')
-    nsstats_permissive = ednscomp2pickle.collect_server_stats(ednscomp2pickle.eval_edns_permissive, ednscompresults)
-    ednscomp2pickle.save(nsstats_permissive, 'permissive')
+        logging.info('processing genreport output in EDNS permissive mode')
+        nsstats_permissive = ednscomp2pickle.collect_server_stats(ednscomp2pickle.eval_edns_permissive, ednscompresults)
+        ednscomp2pickle.save(nsstats_permissive, 'permissive')
 
-    summary, results_strict, results_permissive = evalzone.evaluate(nsstats_strict, nsstats_permissive, domain_nsset, nsname_ipsets, domain_ipset)
-    evalzone.save_pickle(results_strict, 'strict')
-    evalzone.save_pickle(results_permissive, 'permissive')
-    evalzone.save_summary(summary)
-    print(summary.text)
+        summary, results_strict, results_permissive = ednsevalzone.evaluate(nsstats_strict, nsstats_permissive, domain_nsset, nsname_ipsets, domain_ipset)
+        ednsevalzone.save_pickle(results_strict, 'strict')
+        ednsevalzone.save_pickle(results_permissive, 'permissive')
+        ednsevalzone.save_summary(summary)
+        print(summary.text)
 
 if __name__ == "__main__":
     testedns.check_env()

ci/test.sh

@@ -8,7 +8,7 @@ domain2ipset.py
 genednscomp.py > ednscomp.input
 testedns.py 2
 ednscomp2pickle.py ednscompresult-*
-evalzone.py
+ednsevalzone.py
 printresults.py new
 printresults.py new --ns
 printresults.py all permissive dead --ns

dataapi.py

@@ -7,7 +7,7 @@ import pickle
 from typing import Dict, Set
 
 import dns.name
-from evalzone import AnIPAddress
+from ednsevalzone import AnIPAddress
 
 def load_nsname2ipset() -> Dict[dns.name.Name, Set[AnIPAddress]]:
     """raises FileNotFoundError"""

doc/usage.rst

@@ -146,7 +146,7 @@ PATH=$PATH:<path to genreport tool> testedns.py
 ednscomp2pickle.py ednscompresult-*
 
 # process EDNS stats for given zone
-evalzone.py
+ednsevalzone.py
 # output includes statistical results for whole zone file
 
 # print list of domains which are going to break in 2019

domain2ipset.py

@@ -18,7 +18,7 @@ from typing import Counter, Deque, Dict, Iterable, Set, Tuple
 import dns.message
 import dns.query
 
-from evalzone import AnIPAddress
+from ednsevalzone import AnIPAddress
 
 class IP_state(enum.Enum):
     timeout = 0

ednscomp2pickle.py

@@ -8,7 +8,7 @@ import re
 import sys
 from typing import Counter, Dict, List, Tuple
 
-from evalzone import EDNSResult, AnIPAddress
+from ednsevalzone import EDNSResult, AnIPAddress
 
 # zsstmesto.cz. @89.187.140.136 (01.dns.services.dmdox.com.): dns=ok edns=ok edns1=noerror,badversion,soa edns@512=ok ednsopt=ok edns1opt=noerror,badversion,soa do=ok ednsflags=ok optlist=ok,nsid signed=ok ednstcp=ok
 # seznam.cz. @2a02:598:4444::4 (ams.seznam.cz.): dns=ok edns=ok,nsid edns1=noerror,badversion,soa,nsid edns@512=ok ednsopt=ok,nsid edns1opt=noerror,badversion,soa,nsid do=ok ednsflags=ok,nsid optlist=ok,nsid signed=ok ednstcp=ok

genednscomp.py

@@ -9,7 +9,7 @@ from typing import Dict, Set
 import dns.name
 
 import dataapi
-from evalzone import AnIPAddress
+from ednsevalzone import AnIPAddress
 
 def gen_ip_to_nsname(nsname2ipset: Dict[dns.name.Name, Set[AnIPAddress]]) -> Dict[AnIPAddress, dns.name.Name]:
     """

nsname2ipset.py

@@ -10,7 +10,7 @@ import dns.name
 import dns.rdatatype
 import dns.resolver
 
-from evalzone import AnIPAddress
+from ednsevalzone import AnIPAddress
 
 
 def yield_ns_name(nsnames, mapping):

printresults.py

@@ -12,7 +12,7 @@ from typing import Dict, Optional, Set
 
 import dns.name
 
-from evalzone import EDNSResult
+from ednsevalzone import EDNSResult
 
 def print_domain(mode: str, result: EDNSResult, domain: dns.name.Name,
     nsset: Optional[Set[dns.name.Name]], reason) \

zone2pickle.py

@@ -3,16 +3,17 @@
 Tranform DNS zone file into pickled Python objects.
 """
 
+import argparse
 import ipaddress
 import logging
 import pickle
 import sys
-from typing import Dict, List, Iterable, Set, Tuple
+from typing import Dict, List, Iterable, Set, TextIO, Tuple
 
 import dns.zone
 
 import dataapi
-from evalzone import AnIPAddress
+from ednsevalzone import AnIPAddress
 
 def domain2nsset(zoneobj: dns.zone.Zone) -> Dict[dns.name.Name, Set[dns.name.Name]]:
     '''
@@ -52,7 +53,7 @@ def glue_ns2ipset(nslist: Set[dns.name.Name], zoneobj: dns.zone.Zone) \
             ns2ipset[nsname] = ipset
     return ns2ipset
 
-def convert(zone_fn: str, zone_origin: dns.name.Name) -> Tuple[ \
+def convert(zone_file: TextIO, zone_origin: dns.name.Name) -> Tuple[ \
             Dict[dns.name.Name, Set[dns.name.Name]], \
             Set[dns.name.Name], \
             Dict[dns.name.Name, Set[AnIPAddress]]]:
@@ -60,8 +61,7 @@ def convert(zone_fn: str, zone_origin: dns.name.Name) -> Tuple[ \
     convert text zone into set of pickle files with preprocessed metadata
     '''
     logging.info('loading zone file')
-    with open(zone_fn) as zone_file:
-        zone = dns.zone.from_file(zone_file, origin=zone_origin, relativize=False)
+    zone = dns.zone.from_file(zone_file, origin=zone_origin, relativize=False)
 
     logging.info('determining list of unique NS names')
     nslist = uniq_nslist(domain2nsset(zone).values())
@@ -94,13 +94,12 @@ def save(domain_nsset, nslist, nsname2ipset):
 def main():
     '''stand-alone operation if allinone module is not used'''
     logging.basicConfig(level=logging.DEBUG, format='%(asctime)s %(message)s')
-    try:
-        zone_fn = sys.argv[1]
-        zone_origin = dns.name.from_text(sys.argv[2])
-    except IndexError:
-        logging.critical('Usage: %s zone_file zone_origin', sys.argv[0])
-        sys.exit(1)
-    data = convert(zone_fn, zone_origin)
+    argparser = argparse.ArgumentParser(description='parse text zone file into binary format')
+    argparser.add_argument('zone_fn', type=open, help='zone file in RFC 1035 format')
+    argparser.add_argument('zone_origin', type=dns.name.from_text, help='zone name, SOA RR must be present')
+    args = argparser.parse_args()
+
+    data = convert(args.zone_fn, args.zone_origin)
     save(*data)
 
 if __name__ == "__main__":