Commit d55e3297 authored by Petr Špaček's avatar Petr Špaček

domain2ipset: do not excessively retry dead ip addresses

parent 328e8d65
......@@ -47,6 +47,14 @@ class NetStats():
self.netstats[ip]['replies'] += 1
self.netstats[ip]['timeouts_in_row'] = 0
def skip_dead_ips(self, candidates: Iterable[Tuple[int, dns.name.Name, AnIPAddress]]) \
-> Iterable[Tuple[int, dns.name.Name, AnIPAddress]]:
'''skip over candidates which target dead IP addresses'''
for args in candidates:
_, _, ip = args
if not self.is_ip_dead(ip):
yield args
def count_candidates(iterable):
"""
not for general use, depletes generator
......@@ -182,10 +190,11 @@ def update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset):
#
# It could also help as a workaround to agressive response rate limiting.
candidates = randomize_iter(
gen_candidates(domain2nsset, nsname2ipset, netstats, retry_queue, domain2ipset),
100000)
with multiprocessing.Pool(processes = 30) as pool:
candidates = netstats.skip_dead_ips(
randomize_iter(
gen_candidates(domain2nsset, nsname2ipset, netstats, retry_queue, domain2ipset),
100000))
with multiprocessing.Pool(processes = 256) as pool:
for attempt, domain, ip, state in pool.imap_unordered(check_availability, candidates):
process_reply(attempt, domain, ip, state, netstats, retry_queue, domain2ipset)
if len(domain2ipset) % 1000 == 0 and len(domain2ipset.get(domain, [])) == 1:
......@@ -195,7 +204,7 @@ def update_mapping(domain2nsset, nsname2ipset, netstats, domain2ipset):
while retry_queue: # retry_queue might be filled again in process_reply
for attempt, domain, ip, state in pool.imap_unordered(
check_availability,
retry_candidates(retry_queue, netstats)):
netstats.skip_dead_ips(retry_candidates(retry_queue, netstats))):
process_reply(attempt, domain, ip, state, netstats, retry_queue, domain2ipset)
if __name__ == '__main__':
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment