Commit 412f1567 authored by Petr Špaček's avatar Petr Špaček

Merge branch 'bugfixes' into 'master'

Bugfixes

See merge request !7
parents 1b3eda0f 83a5e612
Pipeline #41255 passed with stages
in 3 minutes and 37 seconds
#!/usr/bin/python3
import glob
import logging
import multiprocessing
import sys
import dns.name
......@@ -14,6 +15,7 @@ import testedns
import zone2pickle
def main():
multiprocessing.set_start_method('forkserver')
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
try:
......
......@@ -9,7 +9,7 @@ script sends queries to IP addresses from glue.
import collections
import enum
import logging
from multiprocessing import Pool
import multiprocessing
import pickle
from typing import Counter, Deque, Dict, Iterable, Set, Tuple
......@@ -158,7 +158,7 @@ def update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset):
#candidates = gen_candidates(domain2nsset, nsname2ipset, netstats, retry_queue, domain2ipset)
#logging.info('queue contains %s queries to be checked', count_candidates(candidates))
candidates = gen_candidates(domain2nsset, nsname2ipset, netstats, retry_queue, domain2ipset)
with Pool(processes = 30) as pool:
with multiprocessing.Pool(processes = 30) as pool:
for attempt, domain, ip, state in pool.imap_unordered(check_availability, candidates):
process_reply(attempt, domain, ip, state, netstats, retry_queue, domain2ipset)
if len(domain2ipset) % 1000 == 0 and len(domain2ipset.get(domain, [])) == 1:
......@@ -173,6 +173,7 @@ def update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset):
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
multiprocessing.set_start_method('forkserver')
domain2nsset, nsname2ipset, netstats, domain2ipset = load()
try:
update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset)
......
......@@ -14,7 +14,7 @@ from evalzone import EDNSResult
def parse_nsip_line(line: str) -> Tuple[str, Dict[str, List[str]]]:
"""parse one line from ednscomp log"""
matches = re.match('^[^ ]+\\. @(?P<ip>[^ ]+) \\([^)]+\\): (?P<results>dns=.*)$', line)
matches = re.match('^[^ ]*\\. @(?P<ip>[^ ]+) \\([^)]+\\): (?P<results>dns=.*)$', line)
if not matches:
raise ValueError('line "{}" does not have expected format, skipping'.format(line))
......
......@@ -10,26 +10,28 @@ import dns.name
import dataapi
def get_ns_name_from_ip(ipaddr: str, nsname2ipset: Dict[dns.name.Name, Set[str]]):
def gen_ip_to_nsname(nsname2ipset: Dict[dns.name.Name, Set[str]]) -> Dict[str, dns.name.Name]:
"""
It should never happen that ipaddr is not present in nsname2ipset mapping.
If it happens, it is most likely caused by mixing data files from
two versions of DNS zone. Start from scratch!
Generate reverse mappping IP -> NS name.
"""
for name, ipset in nsname2ipset.items():
if ipaddr in ipset:
return name
raise ValueError('IP addr {} not found in nsname2ipset'.format(ipaddr))
ip2nsname = {}
for nsname, ipset in nsname2ipset.items():
ip2nsname.update({ip: nsname for ip in ipset})
return ip2nsname
def generate(nsname2ipset, domain2ipset):
"""Output format is: zone nsname NSipAddress"""
ip_done = set()
ip2nsname = gen_ip_to_nsname(nsname2ipset)
for domain, ipset in domain2ipset.items():
for ipaddr in ipset:
if ipaddr in ip_done:
continue
nsname = get_ns_name_from_ip(ipaddr, nsname2ipset)
# It should never happen that ipaddr is not present in nsname2ipset mapping.
# If it happens, it is most likely caused by mixing data files from
# two versions of DNS zone. Start from scratch!
nsname = ip2nsname[ipaddr]
yield '{} {} {}\n'.format(domain, nsname, ipaddr)
ip_done.add(ipaddr)
......
#!/usr/bin/python3
import logging
from multiprocessing.pool import Pool
import multiprocessing
import pickle
from typing import Dict, Set, Tuple
......@@ -60,7 +60,7 @@ def update_mapping(nsnames: Set[dns.name.Name],
dns.resolver.default_resolver.lifetime = 5 # seconds
#dns.resolver.default_resolver.nameservers = ['193.29.206.206']
with Pool(processes=128) as p:
with multiprocessing.Pool(processes=128) as p:
i = 0
logging.info('starting DNS query machinery')
for nsname, ipset in p.imap_unordered(get_ips, yield_ns_name(nsnames, mapping), chunksize=10):
......@@ -77,6 +77,7 @@ def save(mapping: Dict[dns.name.Name, Set[str]]) -> None:
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s')
multiprocessing.set_start_method('forkserver')
nsnames = load_nsnames()
mapping = load_nsname2ipset()
try:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment