Commit 11eea2e9 authored by edvard's avatar edvard

přidal jsem PDNS

parent 0d75e703
......@@ -60,6 +60,12 @@ class _Domain(defaultdict):
r += "\n " + key + " " + str(self.urls[key])
for key in self.addresses.keys():
r += "\n " + key + " " + str(self.addresses[key])
if self.pdns:
r += "Informace z PDNS:\n"
for key in self.pdns:
r += key + " "
else:
r += "\n Žádné informace z PDNS."
return r
def __init__(self):
......@@ -67,6 +73,7 @@ class _Domain(defaultdict):
#self.urls = set()
self.urls = defaultdict(_Url)
self.addresses = defaultdict(_Address)
self.pdns = set()
def __getstate__(self):
state = self.__dict__.copy()
......@@ -125,7 +132,7 @@ class _Address(set):
def __init__(self):
self.country = None
self.city = None
self.city = None
#self.vote = None
"""
......
DEBUG:root:This message should go to the log file
INFO:root:So should this
WARNING:root:And this, too
WARNING:root:warning1
DEBUG:root:debug1
import datetime
import logging
import threading
from lib.config import Config
from lib.dbp import Export
from lib.dbp import Status
from lib.dbp import Turris
from lib.dbp import Whitelist
from lib.domains import Domains
import logging
import threading
from urllib.parse import parse_qs
from urllib.parse import urlparse
import urllib.request
class MetadataParser:
""" Obohatit vysledky vyhledavani (objekt crawl) o whois informace a informace z db"""
......@@ -26,6 +23,7 @@ class MetadataParser:
#kazda domena vyvola vlastni thread - trva nacist jeji geoIP
domains = list(crawl.keys())
domainThreadNumber = 0
while len(domains): # spusti maximalne 10 threadu doraz, jednou mi to totiz preteklo (kazda domena spusti jeste tolik threadu, kolik ma IP, ale tech byva jen par)
threads = []
count = 0
......@@ -34,7 +32,8 @@ class MetadataParser:
if count > Config.MAX_WHOIS_DOMAIN_THREADS:
break
domain = domains.pop()
t = threading.Thread(target=self.addDomain, args=(crawl[domain], domain))
domainThreadNumber += 1
t = threading.Thread(target=self.addDomain, args=(crawl[domain], domain, domainThreadNumber))
threads.append(t)
t.start()
#konsolidovat informace o domenach
......@@ -43,13 +42,12 @@ class MetadataParser:
#sesumirovat informace o domene
def addDomain(self, crawlDomain, domainEncountered):
def addDomain(self, crawlDomain, domainEncountered,domainThreadNumber):
domainNaked = Domains.url2domain(domainEncountered) #domainEncountered[domainEncountered.find("//") + 2:] # http://seznam.cz -> seznam.cz; //ajax.googleapis.com -> ajax.googleapis.com
logging.debug("domena: " + domainNaked)
if domainNaked in [self.websiteDomain, "127.0.0.1", "localhost"]: #domena samu sebe ignoruje. A kdybych nekdy zablokoval localhost, popravili by me.
logging.debug("skip itself")
else:
if Whitelist.matches(domainEncountered): #je domena ve whitelistu 2ndLD domen?
logging.debug("skip whitelisted")
else:
......@@ -60,51 +58,67 @@ class MetadataParser:
threads = []
#queueIp = queue.Queue()
#print("DOMAINNAKED {}".format(domainNaked))
threadNumber = 0
for ip_frame in Domains.get_ips_for_host(domainNaked): # (10, 1, 6, '', ('2001:888:2000:d::a2', 80, 0, 0))
threadNumber += 1
ip = ip_frame[4][0]
t = threading.Thread(target=self.addAddress, args=(crawlDomain.addresses[ip], ip, domainEncountered)) #r += self.addAddress(ip_frame)
t = threading.Thread(target=self.addAddress, args=(crawlDomain.addresses[ip], ip, domainEncountered, crawlDomain.pdns, threadNumber,domainThreadNumber)) #r += self.addAddress(ip_frame)
threads.append(t)
t.start()
#konsolidovat informace o IP v domenach
for thread in threads:
thread.join()
thread.join()
if len(threads) == 0: #domena zadne IP nema, je pozastavena
#presto chceme evil host zapsat - alespon s ip null
#vote =
self.addAddress(None, None, domainEncountered)
self.addAddress(None, None, domainEncountered,crawlDomain.pdns,0.0)
#if vote == None:vote = ""
#crawlDomain.vote = vote
#stat a mesto IPcka - nacitame threadovane, kazdy zvlast trva vterinu
def addAddress(self, crawlDomainIp, ip, remoteHost):
#vote = "n/a"
##
# Stat a mesto IPcka - nacitame threadovane, kazdy zvlast trva vterinu
# crawlDomainIp je objekt Address
# ip je klic, ktery z objektu Domain vytahne nas objekt Address.
#
def addAddress(self, crawlDomainIp, ip, remoteHost, pdns, threadNumber,domainThreadNumber):
#vote = "n/a"
#import pdb;pdb.set_trace()
#logging.debug("!!("+str(threadNumber)+","+str(domainThreadNumber)+") REMOTE " + " host: " + str(remoteHost))
with Config.lock:
#logging.debug("!("+str(threadNumber)+","+str(domainThreadNumber)+") REMOTE " + " host: " + str(remoteHost))
#updatovat domene timestamp setkani, aby byla v dalsim exportu #X Db.cur.
#Db.cur = Db.connection.cursor()
if not ((ip == None and Turris.update(timestamp=datetime.datetime.now()).where(Turris.remoteHost == remoteHost).execute()) #domena nema ip, v databazi je tedy 1x, vyhledavame dle nazvu domeny
or Turris.update(timestamp=datetime.datetime.now()).where(Turris.ip == ip).execute()): #domena ma ip, kazdy je v db zvlast, vyhledavame dle ip
if ((Turris.update(timestamp=datetime.datetime.now()).where(Turris.remoteHost == remoteHost).execute() == 0 #domena v db jeste nema jmeno (mozna tam ma IP) Xdomena nema ip, v databazi je tedy 1x, vyhledavame dle nazvu domeny
and Turris.select().where(Turris.remoteHost == remoteHost).count() == 0) # za tri hodiny jsem nezjistil proc, ale update vraci NULU. Ovsem jakmile zavolam pdb, uz vraci korektne treba osmicku. Sigr jeden. Tak sem davam tenhle select, funguje lip. Divne je, ze v nasledujicim radku u IP update vraci korektni cislo. Prehodit jsem je nezkousel.
or Turris.update(timestamp=datetime.datetime.now()).where(Turris.ip == ip).execute() == 0): #domena v db nema IP (mozna tam je zaznam s totoznym jmenem a treba jinou IP) Xdomena ma ip, kazdy je v db zvlast, vyhledavame dle ip
#if remoteHost == "www.corradorossi.it":
#print("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "SELECT " + str(Turris.select().count()))
#print("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "SELECT " + str())
#print("("+str(threadNumber)+","+str(domainThreadNumber)+") " + str(Turris.update(timestamp=datetime.datetime.now()).where(Turris.remoteHost == remoteHost).execute()))
#print("("+str(threadNumber)+","+str(domainThreadNumber)+") " + str(Turris.update(timestamp=datetime.datetime.now()).where(Turris.ip == ip).execute()))
#print("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "done")
#import pdb;pdb.set_trace()
#print("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "PRIDAVAM")
#adresu jsme jeste nepotkali, neni v tabulce turris
#pridat domenu do turris
logging.debug("INSERT " + remoteHost)
#logging.debug("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "INSERT " + remoteHost)
Turris.insert(ip=ip, port=80, url=self.websiteDomain, remoteHost=remoteHost).execute() # XXX port muze byt jiny nez 80
#Db.cur.execute("""INSERT INTO turris (ip, port, url, `evil host`) VALUES (%s,%s,%s,%s)""", (ip, 80, self.websiteDomain, remoteHost))
#Db.connection.commit()
#Db.cur.close()
logging.debug("ADDRESS " + " ip: " + str(ip)) #+ str(vote)
else:
#print("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "NEPRIDAVAM")
pass
#logging.debug("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "ADDRESS " + " ip: " + str(ip)) #+ str(vote)
if ip != None:
#crawlDomainIp.vote = vote
#kontaktovat externi geoIP sluzbu
try:
hostipApi = urllib.request.urlopen('http://api.hostip.info/get_html.php?ip=' + ip + '&position=true').read().decode("utf-8").split("\n")
#['Country: CZECH REPUBLIC (CZ)', 'City: Prague', '', 'Latitude: 50.0833', 'Longitude: 14.4333', 'IP: 109.123.209.188', '']
crawlDomainIp.country = hostipApi[0].split(":")[1]
crawlDomainIp.city = hostipApi[1].split(":")[1]
except UnicodeDecodeError: # as e
logging.debug("neumim dekodovat")
#logging.debug(e)
crawlDomainIp.country, crawlDomainIp.city = Domains.ip2countryAndCity(ip)
# kontaktovat PDNS
pdns.update(Domains.ip2pdnsDomains(ip))
return None
else: #zadna ip neni k dispozici, domena je asi propadla, hlas patri jmenu domeny
#return vote
......@@ -112,6 +126,7 @@ class MetadataParser:
##try:
##if ip == None: #domena nema ip, v databazi je tedy 1x, vyhledavame dle nazvu domeny
#Db.cur.execute("""SELECT status from turris JOIN status ON status.id = turris.status WHERE `evil host` = %s LIMIT 1""", (remoteHost,))
......
......@@ -177,7 +177,7 @@ class ScanController:
def dump():
with open(Config.configFile, 'w') as f:
json.dump(self.queueFF, f)
print("UNKBOOK")
#print("UNKBOOK")
try:
self.queueFF.pop(self.profile)
dump()
......@@ -193,7 +193,7 @@ class ScanController:
except OSError:
print("({}) System se nezotavil.".format(self.profile))
return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze.
print("UNKBOOKED")
#print("UNKBOOKED")
def queue(self):
......
import threading
import os
from glob import glob
from peewee import MySQLDatabase
class Config:
profileCount = 21 #pocet profilu vytvorenych ve firefoxu. Tyto je treba vytvorit rucne. Nazev profilu je cislo - 0,1...
......@@ -15,4 +16,13 @@ class Config:
THUMBNAIL_SIZE = 640, 640
MAX_WHOIS_DOMAIN_THREADS = 10 # spusti maximalne 10 threadu doraz, jednou mi to totiz preteklo (kazda domena spusti jeste tolik threadu, kolik ma IP, ale tech byva jen par)
MAX_BROWSER_RUN_TIME = 25 #maximalni cas, ktery muze browser bezet
MAX_BROWSER_EXPIRATION = 15 # pocet vterin, ktere muzeme max cekat, nez se browser zavre (trva, nez zapise soubory)
\ No newline at end of file
MAX_BROWSER_EXPIRATION = 15 # pocet vterin, ktere muzeme max cekat, nez se browser zavre (trva, nez zapise soubory)
def connect():
# XX resim problem peewee.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))") po 7 hodinach timeoutu
# XX kupodivu pripojeni nemuze byt v dbp DBModel.connect. Prestoze type je pak spravne (MySQLDatabase), nic udelat nejde a pokusy o select konci NoneType.
print("Connecting to DB.")
Config.myDB = MySQLDatabase("mdmaug", host='172.20.7.10', port=3306, user="root", passwd="lopuch") # XX dal jsem pryc: , threadlocals=False
Config.myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
Config.connect()
......@@ -6,7 +6,7 @@ from lib.config import Config
from urllib.parse import urlparse
import logging
import time
logging.basicConfig(level=logging.WARNING, format="%(message)s")
#logging.basicConfig(level=logging.WARNING, format="%(message)s")
from lib.domains import Domains
......@@ -14,23 +14,38 @@ from lib.domains import Domains
#myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
#myDB.connect()
# XX resim problem peewee.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))") po 7 hodinach timeoutu
Config.myDB = MySQLDatabase("mdmaug", host='172.20.7.10', port=3306, user="root", passwd="lopuch") # XX dal jsem pryc: , threadlocals=False
Config.myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
# Config.myDB.connect() # XX kupodivu toto neni potreba
print("Connecting to DB.")
#print("Start test.")
#print(Whitelist.matches("www.mozilla.org"))
#print("End test.")
class DbModel(Model):
def assureConnection():
print ("Assure start.")
try: # zkousim spustit nejaky prikaz
#print("1")
Whitelist.select().count()
#print("2")
except:# OperationalError:
#print("3")
Config.connect()
#print("4")
try: # zkousim spustit nejaky prikaz
#print("5")
Whitelist.select().count()
#print("6")
except:
print("7 - failed")
raise
print("Connection to DB assured.")
"""A base model that will use our MySQL database"""
def connect():
logging.debug("connecting db....")
#DbModel.Meta.myDb = Config.myDB
# Config.myDB.connect() # XX kupodivu toto neni potreba
#print(Whitelist.select().count())
#print("Connecting to DB.")
#print("Start test.")
#print(Whitelist.matches("www.mozilla.org"))
#print("End test.")
def disconnect():
logging.debug("... db ends.")
......@@ -40,6 +55,10 @@ class DbModel(Model):
database = Config.myDB
pass
#print("DVA")
#DbModel.connect()
#print(Config.myDB)
class Status(DbModel):
id = PrimaryKeyField()
name = CharField(5)
......@@ -82,7 +101,7 @@ class Export(DbModel):
#Db.cur.execute("""SELECT ip, port, url, `evil host`, `other details`,status from turris where status > 1 and timestamp > (select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)""")
#rows = Db.cur.fetchall()
#Db.cur.close()
q = "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and timestamp > "
q = "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and `ip` IS NOT NULL and timestamp > "
if days:
q += "NOW() - INTERVAL {} DAY ".format(int(days))
else :
......@@ -145,8 +164,11 @@ class Turris(DbModel):
logging.error("domain should have been inserted in the database, but it hasnt been")
print("vote error")
raise
ipList = list(set([o.ip for o in rows]))
count = Turris.update(status=str(status)).where(Turris.ip << ipList).execute()
ipList = list(set([o.ip for o in rows if o.ip != None]))
count = 0
if ipList:
count += Turris.update(status=str(status)).where(Turris.ip << ipList).execute()
count += Turris.update(status=str(status)).where(Turris.remoteHost == host).execute() # ovlivnit i remoteHost s IP = NULL
logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))
print("vote:" + vote + " host:" + host + " count:" + str(count))
return str(count) + " updated"
......@@ -163,8 +185,8 @@ class Whitelist(DbModel):
try:
url = Domains.url2domain(url)
s = url.split(".")
print(urlparse(url))
print(s)
#print(urlparse(url))
#print(s)
except AttributeError:
logging.error("ATTRIBUTE ERROR! Whitelist.matches({})".format(url))
return False
......@@ -175,79 +197,3 @@ class Whitelist(DbModel):
domain2nd = s
with Config.lock:
return Whitelist.select().where(Whitelist.domain == domain2nd).count() > 0
#Db.cur = Db.connection.cursor()
#Db.cur.execute("""SELECT count(id) from whitelist where domain = %s""", (domain2nd,))
#res = Db.cur.fetchone()
#Db.cur.close()
#print(res)
#print("is " + str(domain2nd) + " whitelisted?")
#if res != None and int(res[0]) > 0:
#domena je whitelistovana. Ani si nezapiseme, ze jsme ji videli do tabulky pro Turris firewall.
#DbModel.connect()
DbModel.connect() # XX po 8 hodinach odpadne
#domain2nd = "google.cz"
#print(Whitelist.select().where(Whitelist.domain == domain2nd))
#print(Whitelist.select().where(Whitelist.domain == domain2nd))
#present = Whitelist.select().where(Whitelist.domain == domain2nd).count() > 0
# when you're ready to start querying, remember to connect
#print(Status.select().where(Status.id == 1).get().name)
#import logging
#logger = logging.getLogger('peewee')
#Db.cur.execute("""SELECT count(id) from whitelist where domain = %s""", (domain2nd,))
#domain2nd = "mozilla.org"
#print(Whitelist.select().where(Whitelist.domain == domain2nd).count() > 0)
#Db.cur.execute("""UPDATE turris set timestamp = CURRENT_TIMESTAMP where `evil host` = %s""", (remoteHost,))
#remoteHost = "http://nakulpi.net"
#print(Turris.select().where(Turris.remoteHost == remoteHost).count())
#print(Turris.update(timestamp = datetime.datetime.now()).where(Turris.ip == "128.30.52.95" ).execute())
#Db.cur.execute("""SELECT status from turris JOIN status ON status.id = turris.status WHERE `evil host` = %s LIMIT 1""", (remoteHost,))
#Db.cur.execute("""SELECT status from turris JOIN status ON status.id = turris.status WHERE `ip` = %s LIMIT 1""", (ip,))
#try:
# ip = "fsd"
# print(Turris.select().join(Status, on = (Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get())
#except DoesNotExist:
# pass
#Export.insert(id = None).execute()
#INSERT INTO turris (ip, port, url, `evil host`) VALUES (%s,%s,%s,%s)""", (ip, 80, self.websiteDomain, remoteHost))
#ip = "test"
#print("Fsd")
#print(Turris.update(timestamp = datetime.datetime.now()).where(Turris.remoteHost == remoteHost ).execute())
#count = Db.cur.execute("""UPDATE turris set status = %s where `evil host` = %s""", (str(status), host,))
#host = remoteHost
#status = 3
#print(Turris.update(status = str(status)).where(Turris.remoteHost == host).execute())
#REPLACE INTO whitelist set domain = %s""", (self.websiteDomain,))
#try:
# Whitelist.insert(domain = "google.cz").execute()
#except IntegrityError:
# # jiz je vlozeno
# pass
#Db.cur.execute("""SELECT ip, port, url, `evil host`, `other details`,status from turris where status > 1 and timestamp > (select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)""")
#Turris.select().where(status > 1, timestamp > Turris.select())
#rq = RawQuery(Turris, "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and timestamp > (select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)")
#for obj in rq.execute():
# print (obj.ip)
#Turris.insert(ip = ip, port = 80, url = self.websiteDomain, remoteHost = remoteHost).execute()
#("""REPLACE INTO whitelist set domain = %s""", (self.websiteDomain,)))
#print(Db.cur.execute("""INSERT into export (`id`) VALUES (NULL)"""))
\ No newline at end of file
import re
import socket
import logging
import re
from urllib.parse import parse_qs
from urllib.parse import urlparse
import urllib.request
class Domains:
""" webove nastroje """
......@@ -35,4 +41,27 @@ class Domains:
url = re.sub('^(http://|https://|ftp://)', '', url) # odstrihnout protokol
url = re.sub('(/.*)', '', url) # nestojim o cestu, jen o domene
url = re.sub('[^a-z0-9\.]', '', url) # nechat jen pratelske znaky
return url
\ No newline at end of file
return url
def ip2pdnsDomains(ip):
try:
# XX mohl bych sortovat dle 2nd domeny. Pripadne oriznout 3rd domenu, nechat jen 2nd. Tam ale musim osetrit problemove dvojite tld - co.uk, gov.ua...
pdns = urllib.request.urlopen('http://pdns.cert.at/p/dns?qry=' + ip).read().decode("utf-8")
items = re.findall("<div class='x[BA]'>(.*)</div>", pdns)
return items
except Exception as e:
logging.debug("chyba pri kontaktu s PDNS: " + str(e))
return None
def ip2countryAndCity(ip):
try:
hostipApi = urllib.request.urlopen('http://api.hostip.info/get_html.php?ip=' + ip + '&position=true').read().decode("utf-8").split("\n")
#['Country: CZECH REPUBLIC (CZ)', 'City: Prague', '', 'Latitude: 50.0833', 'Longitude: 14.4333', 'IP: 109.123.209.188', '']
return hostipApi[0].split(":")[1], hostipApi[1].split(":")[1]
except UnicodeDecodeError: # as e
logging.debug("neumim dekodovat")
except Exception as e:
logging.debug("hostip.info down: " + str(e))
return None, None
\ No newline at end of file
......@@ -7,8 +7,7 @@ from lib.config import Config
#from lib.dbp import Db
from lib.rest import Rest
import time
from lib.dbp import Export
from lib.dbp import Export, DbModel
env = Environment()
env.loader = FileSystemLoader(".")
......@@ -51,7 +50,11 @@ class Server (SimpleHTTPRequestHandler):
with open(url, type) as f:
self.output(f.read(), contentType=mimetypes.guess_type(url))
def do_GET(self):
def do_GET(self):
#XX Config.myDB.get_conn() # vytvorit spojeni, kdyby po 9 hodinach vypadlo. (Funguje?)
#from lib.dbp import Whitelist
DbModel.assureConnection()
path = self.path.split("/")
print("processing", path[1])
if path[1] == "":
......
resetting running browsers
INSERT
DEBUG:resetting running browsers
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1', [])
DEBUG:log size:
DEBUG:189338
DEBUG:domena: www.kontakt.cz
DEBUG:skip itself
DEBUG:domena: download.cdn.mozilla.net
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1 WHERE (`t1`.`domain` = %s)', ['mozilla.net'])
DEBUG:domena: www.corradorossi.it
DEBUG:domena: download.mozilla.org
DEBUG:skip whitelisted
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1 WHERE (`t1`.`domain` = %s)', ['corradorossi.it'])
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1 WHERE (`t1`.`domain` = %s)', ['mozilla.org'])
DEBUG:skip whitelisted
DEBUG:('UPDATE `turris` SET `timestamp` = %s WHERE (`turris`.`remoteHost` = %s)', [datetime.datetime(2015, 8, 10, 18, 53, 26, 895061), 'www.corradorossi.it'])
DEBUG:('UPDATE `turris` SET `timestamp` = %s WHERE (`turris`.`ip` = %s)', [datetime.datetime(2015, 8, 10, 18, 53, 26, 934885), '2001:4b78:1001::101'])
DEBUG:ADDRESS ip: 2001:4b78:1001::101
DEBUG:('UPDATE `turris` SET `timestamp` = %s WHERE (`turris`.`remoteHost` = %s)', [datetime.datetime(2015, 8, 10, 18, 53, 26, 960330), 'www.corradorossi.it'])
DEBUG:INSERT www.corradorossi.it
DEBUG:('INSERT INTO `turris` (`date`, `ip`, `port`, `url`, `block`, `remoteHost`, `otherDetails`) VALUES (%s, %s, %s, %s, %s, %s, %s)', [0, '217.64.202.205', 80, 'www.kontakt.cz', 0, 'www.corradorossi.it', ''])
DEBUG:ADDRESS ip: 217.64.202.205
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1 WHERE (`t1`.`domain` = %s)', ['mozilla.org'])
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1 WHERE (`t1`.`domain` = %s)', ['corradorossi.it'])
DEBUG:('SELECT `t1`.`id`, `t1`.`timestamp`, `t1`.`status`, `t1`.`date`, `t1`.`ip`, `t1`.`port`, `t1`.`url`, `t1`.`block`, `t1`.`remoteHost`, `t1`.`otherDetails` FROM `turris` AS t1 INNER JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`) WHERE (`t1`.`remoteHost` = %s) LIMIT 1', ['www.corradorossi.it'])
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1 WHERE (`t1`.`domain` = %s)', ['mozilla.net'])
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1 WHERE (`t1`.`domain` = %s)', ['kontakt.cz'])
DEBUG:('SELECT `t1`.`id`, `t1`.`timestamp`, `t1`.`status`, `t1`.`date`, `t1`.`ip`, `t1`.`port`, `t1`.`url`, `t1`.`block`, `t1`.`remoteHost`, `t1`.`otherDetails` FROM `turris` AS t1 INNER JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`) WHERE (`t1`.`remoteHost` = %s) LIMIT 1', ['www.kontakt.cz'])
DEBUG:('SELECT Count(*) FROM `whitelist` AS t1', [])
DEBUG:INSERT1
WARNING:war1
DEBUG:INSERT2
WARNING:war2
DEBUG:resetting running browsers
#! /usr/bin/python3
#!/usr/local/bin/python3.4
'''
MDM-Augmented.
......@@ -54,10 +54,12 @@
'''
import logging
logging.basicConfig(level=logging.DEBUG, format="%(message)s")
#logging.basicConfig(level=logging.WARNING, format="%(levelname)s: %(message)s",filename="logger.log")
from http.server import HTTPServer
import os
import ssl
import logging
#import traceback
#import pdb
from xvfbwrapper import Xvfb
......@@ -66,16 +68,18 @@ from lib.config import Config
from lib.server import Server
#from lib.dbp import DbModel
from lib.rest import Rest
logging.basicConfig(level=logging.WARNING, format="%(message)s")
#from lib.dbp import Status, Export, Turris, Whitelist
#url = "http://stat.cz"
#print(Whitelist.matches(url))
#print(Whitelist.select().count())
#quit()
#ip = "112.78.117.153"
#print(Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().id)
#status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
#print(status)
#Turris.select().where(Turris.id == 5).count()
#from urllib.parse import parse_qs
#from urllib.parse import urlparse
#quit()
vdisplay = Xvfb()
......@@ -111,4 +115,4 @@ cur.execute("""SELECT name from turris JOIN status ON status.id = turris.status
vote = self.cur.fetchone()[0]
print(vote)
exit()
'''
\ No newline at end of file
'''
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
// ==UserScript==
// @name mdm list hrozeb - autoopen
// @namespace mdm
// @description Hromadne otevre napadené domény. Dá jim do hashe administrátorem zvolene příznaky, co s nimi dělat (poslat maily, analyzovat, telefonovat).
// @include https://mdm.labs.nic.cz/list/open*
// @include https://mdm.labs.nic.cz/list/new*
// @version 1
// @grant none
// ==/UserScript==
if (document.location.href.indexOf("https://mdm.labs.nic.cz/list/") === 0) {
$ = j;
/**
* Statisticke ukazatele
*/
Stats = function () {
openingI = 0;
openI = 0;
closedI = 0;
refresh = function () {
$("#statistics #stats-opening").text("Opening " + openingI);
$("#statistics #stats-open").text(" / Open " + openI);
$("#statistics #stats-closed").text(" / Closed " + closedI);
};
tabCreated = function () {
$("#statistics").show();//pri prvnim popupu ukazeme statistiku
openingI++;
refresh();
};
tabClosed = function () {
openingI--;
closedI++;
refresh();
};
tabOpened = function () {
openingI--;
openI++;
refresh();
};
init = function () {
//Statistika html
statistics = $("<div id=statistics />").insertBefore("h1").hide()
.append($("<span />", {id: "stats-opening"}).css({color: "blue"}))
.append($("<span />", {id: "stats-open"}).css({color: "green"}))
.append($("<span />", {id: "stats-closed"}).css({color: "red"}));
/**
* Listener informaci z otevrenych popupu.
*/
window.addEventListener("message", function (e) {
console.log(e);
//cross-site security
if (e.origin !== "https://mdm.labs.nic.cz") {
console.log("wrong origin");
return;
}
if (e.data.close !== undefined) {//tab se zavira
tabClosed();
e.source.close();
return;
}
if (e.data.open !== undefined) {//tab zustal uspesne otevren
tabOpened();
return;
}
}, false);
};
return {
init: init,
tabOpened: tabOpened,
tabClosed: tabClosed,
tabCreated: tabCreated
};
}();
Autoopen = function () {
popupList = [];
/**
* Akce odkazu - otevreni tabu, opatreneho hashFlagy.
* @returns {Boolean}
*/
linkClick = function () {
//Otevrit subtab
console.log("...click received");
popupList.push(window.open(this.href + getHashFlagsStr()));
window.focus();//subtab nema focus
Stats.tabCreated();//drzet prehled, kolik oken je otevrenych
//, '_blank', 'toolbar=yes, scrollbars=yes, resizable=yes, width=1200');//j(this).click();//window.open(j(this).attr("href"), "", "menubar=yes");//abychom otevreli do noveho okna - width=1000: protoze dalsi podokna by byla zprehazena.
return false; //misto nasledovani odkazu jsme sli do subtabu
};
//Listener hlasovani
window.addEventListener("message", function (e) {
if (e.data.voted) {//v tabu se hlasovalo
for (i in popupList) {//preposlat hlasovani ostatnim tabum
popupList[i].postMessage({"voted": e.data.voted, "domain": e.data.domain}, "https://mdm.labs.nic.cz");
}
}
});
/**
* Vraci hash na zaklade voleb administratora v checkboxech.
* @returns {String} hash
*/
getHashFlagsStr = function () {
//Informovat subtab hashem o akcich, ktere chceme provezt
hash = [];
$("#options label input").each(function () {
if ($(this).prop('checked')) {
hash.push($(this).attr("name"));
}
});
return "#" + hash.join("&");
};
//Spousteci tlacitko + limit
launchButton = j("<button>launch</button>").insertBefore(j("h1")).click(function () {//otevre jen zpravy
$(thisRef = this).text("...");
console.log("launch button click");
limit = $("#limit").val();//99;//maximalni pocet navstivenych linku - otevrenych podoken
j("#flexigrid_threats td[abbr=host] a").each(function () {
if (limit-- < 1) {
return;
}
console.log("click...");
$(this).unbind().click(linkClick).click();//musim mit jistotu, ze se flexigrid nerefreshnul, obnovuju tak svou modifikaci kliku
});
//za vterinu obnovit text na tlacitku (pri 15+ tabech oteviranych naraz muze FF chvili vypadat, ze se nekliklo)
setTimeout(function () {
$(thisRef).text("launch");
}, 5000);
}).after(
$("<label>Limit:</label>").append(
$("<input/>", {id: "limit", val: 50}).css("width", "30px").attr({alt: "Maximální počet otevřených oken"}))
);
//Otevirat linky do podoken
$("#flexigrid_threats td[abbr=host] a").unbind().click(linkClick);//soucasne nacetle linky
$(".flexigrid").click(function () {//linky, ktere se do flexigridu nactou v budoucnu
$("#flexigrid_threats td[abbr=host] a").unbind().click(linkClick);
});
//Statistiky
Stats.init();
//Disclaimer
$(launchButton).before("<h1>Remember: NoScript on, <a href='https://manager.nic.cz/summary/'>Daphne</a> logged, Firebug off</h1>");
//Vypsat moznosti
$("<div />", {id: "options"}).insertBefore(launchButton).append(
$("<label><input type='checkbox' checked='checked' name='message' />Chci poslat maily</label> <span title='Nově nahlášeným doménám pošle automaticky mail. Pokud se zpráva úspěšně odešle a pokud není zaškrtlá analýza, zavře okno (pokud ve FF about:config nastavime allow_scripts_to_close_windows).'>(?)</span><br />")).append(
$("<label><input type='checkbox' name='analyze' />Chci analyzovat</label> <span title='Domény zanalyzuje a nechá nás rozhodnout, která doména je malware.'>(?)</span><br />").focus()).append(
$("<label><input type='checkbox' name='contact' />Chci telefonovat lidem</label> <span title='U 3 dny starých zpráv zobrazí telefonní číslo. Lidem, kterým jsme právě poslali zprávu, nechceme rovnou volat telefonem. Automaticky spustí analýzu, abychom mohli lidem do telefonu říct, kde mají problém.'>(?)</span> <br />")).append(
$("<label><input type='checkbox' checked='checked' name='wellknownfree' />Nechci zahrnout wz.cz apod</label> <span title='Webzdarma.cz, chytrak.cz apod mají stále nějaké hrozby a někdy je jejich týmy řeší proaktivně. Zašrktnutím této možnosti takové weby budeme rovnou ignorovat.'>(?)</span> <br />"));
};
$(window).load(function () {
/**
* nacteni ceka na flexigrid a plugin tabs, muze trvat nekolik vterin, nez se nacte
*/
loadedCounter = 0;
initWhenLoaded = function () {
if (j("#flexigrid_threats td[abbr=host] a").length > 0) {
console.log("loaded");
Autoopen();
} else {<