Commit a640614e authored by Edvard Rejthar's avatar Edvard Rejthar

unikátní záznamy ip X port v exportu

kompletní migrace na nový stroj
parent d80dcfce
...@@ -5,7 +5,7 @@ Scans a website for a sign of a parasite hosts or commands. ...@@ -5,7 +5,7 @@ Scans a website for a sign of a parasite hosts or commands.
## Installation ## Installation
1. ```git clone git@gitlab.labs.nic.cz:csirt/mdmaug.git /tmp/mdmaug``` 1. ```git clone git@gitlab.labs.nic.cz:csirt/mdmaug.git /tmp/mdmaug```
2. edit config.py 2. edit mdmaug/lib/config.py
3. ```/tmp/mdmaug/INSTALL``` 3. ```/tmp/mdmaug/INSTALL```
### Notes ### Notes
...@@ -17,12 +17,12 @@ Scans a website for a sign of a parasite hosts or commands. ...@@ -17,12 +17,12 @@ Scans a website for a sign of a parasite hosts or commands.
## What is done to Firefox profiles? ## What is done to Firefox profiles?
We want no block nor safebrowsing warning. If you created the profiles manually, you'd use ```firefox -P```, the profiles names being: 0,1... We want no block nor safebrowsing warning. If you created the profiles manually, you'd use ```firefox -P```, the profiles names being: 0,1...
For about:config changes, see prefs.js. IE: For about:config changes, see prefs.js. IE:
* toolkit.startup.max_resumed_crashes = -1 (protoze i kdyz prohlizec nekdy killnu, nesmi me pri spusteni otravovat gui popupem) * toolkit.startup.max_resumed_crashes = -1 (protoze i kdyz prohlizec nekdy killnu, nesmi me pri spusteni otravovat gui popupem)
* network.http.accept-encoding = "" # ukladame streamy, ale neumim je rozzipovat * network.http.accept-encoding = "" # ukladame streamy, ale neumim je rozzipovat
* extensions.autoDisableScopes = "0" # moznost instalovat ze vsech umisteni * extensions.autoDisableScopes = "0" # moznost instalovat ze vsech umisteni
* browser.selfsupport.url = "" # tato moznost normalne v about:config neni, ale omezuje to nejake zbytecnou telemetrii, viz Mozilla Heartbeat * browser.selfsupport.url = "" # tato moznost normalne v about:config neni, ale omezuje to nejake zbytecnou telemetrii, viz Mozilla Heartbeat
* # nepamatovat si historii (Preferences / Privacy / Firefox will use custom settings for history / Clear history when closes / Setting / All) * # nepamatovat si historii (Preferences / Privacy / Firefox will use custom settings for history / Clear history when closes / Setting / All)
* # nejsem si jist, nakolik to funguje, zrejme dost * # nejsem si jist, nakolik to funguje, zrejme dost
* ... * ...
\ No newline at end of file
# XX shouldnt I delete this file?
su - mdmaug -c 'cd /home/mdmaug/mdmaug/ ; python3 mdmaug.py' su - mdmaug -c 'cd /home/mdmaug/mdmaug/ ; python3 mdmaug.py'
pkill python3 #pri Ctrl+C v prikazu su se uzavre jen terminal, ale ne uz python skript. Takhle to zas zabije veskery Python, ale sandboxovy-zavirovany uzivatel mdmaug stejne ma byt jen na spousteni tohohle skriptu. pkill python3 #pri Ctrl+C v prikazu su se uzavre jen terminal, ale ne uz python skript. Takhle to zas zabije veskery Python, ale sandboxovy-zavirovany uzivatel mdmaug stejne ma byt jen na spousteni tohohle skriptu.
\ No newline at end of file
...@@ -6,23 +6,23 @@ from lib.controller.scan_controller import ScanController ...@@ -6,23 +6,23 @@ from lib.controller.scan_controller import ScanController
from lib.model.dbp import Status, Export, Turris, Whitelist from lib.model.dbp import Status, Export, Turris, Whitelist
from lib.analysis.parser.traffic_log_parser import TrafficLogParser from lib.analysis.parser.traffic_log_parser import TrafficLogParser
class Api: class Api:
website = "" # http://site.cz website = "" # http://site.cz
websiteDomain = "" # site.cz websiteDomain = "" # site.cz
def __init__(self, path): def __init__(self, path):
self.path = path self.path = path
def run(self, cmd): def run(self, cmd):
""" Accept command """ """ Accept command """
if cmd == "analyze": if cmd == "analyze":
return ScanController().launch(self.path) return ScanController().launch(self.path)
if cmd == "analyze=cached": if cmd == "analyze=cached":
return ScanController().launch(self.path, cached = 1) return ScanController().launch(self.path, cached = 1)
if cmd == "analyze=weekcache": if cmd == "analyze=weekcache":
return ScanController().launch(self.path, cached = 7) return ScanController().launch(self.path, cached = 7)
if cmd == "analyze=oldcache": if cmd == "analyze=oldcache":
return ScanController().launch(self.path, cached = True) return ScanController().launch(self.path, cached = True)
elif cmd == "export=view": # XX deprecated? elif cmd == "export=view": # XX deprecated?
return Export.exportView() return Export.exportView()
elif cmd == "export=confirm": # XX deprecated? elif cmd == "export=confirm": # XX deprecated?
...@@ -49,16 +49,16 @@ class Api: ...@@ -49,16 +49,16 @@ class Api:
return self.whitelist() return self.whitelist()
elif cmd == "reset": elif cmd == "reset":
Server.reset() Server.reset()
return "reset" return "reset"
def reset(): def reset():
logging.debug("resetting running browsers") logging.debug("resetting running browsers")
with open(Config.configFile, 'w') as f: # clear the queue with open(Config.configFile, 'w') as f: # clear the queue
json.dump({}, f) json.dump({}, f)
subprocess.call(["pkill", Config.browser]) # kill frozen browsers subprocess.call(["pkill", Config.browser]) # kill frozen browsers
#prida 2ld domenu mezi whitelistovane #prida 2ld domenu mezi whitelistovane
def whitelist(self): def whitelist(self):
......
...@@ -39,7 +39,7 @@ class ScanController: ...@@ -39,7 +39,7 @@ class ScanController:
if cached: if cached:
# """ Pokud je k dispozici analyza, vratit ji """ # """ Pokud je k dispozici analyza, vratit ji """
dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/" dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/"
if os.path.isdir(dir): if os.path.isdir(dir):
snapdirs = [str(dir + subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu snapdirs = [str(dir + subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu
if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir+subdir + "/"+ScanController.CRAWL_FILE)] if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir+subdir + "/"+ScanController.CRAWL_FILE)]
...@@ -54,7 +54,7 @@ class ScanController: ...@@ -54,7 +54,7 @@ class ScanController:
logging.debug("({-1}) Cachovana analyza nenalezena") logging.debug("({-1}) Cachovana analyza nenalezena")
# provest novou analyzu # provest novou analyzu
if self.queue(): # /api/analyze/web - zaradi web do fronty if self.queue(): # /api/analyze/web - zaradi web do fronty
print ("({}) start crawl".format(self.profile)) print ("({}) start crawl".format(self.profile))
self.url = Domains.assureUrl(url) self.url = Domains.assureUrl(url)
try: try:
...@@ -104,7 +104,7 @@ class ScanController: ...@@ -104,7 +104,7 @@ class ScanController:
logging.debug("({}) time is run!".format(self.profile)) logging.debug("({}) time is run!".format(self.profile))
raise FileNotFoundError("time is run - browser expired") raise FileNotFoundError("time is run - browser expired")
time.sleep(1) time.sleep(1)
NsprLogParser(logfile, crawl) NsprLogParser(logfile, crawl)
self.unbookProfile() # uvolnit browser profil self.unbookProfile() # uvolnit browser profil
...@@ -117,7 +117,7 @@ class ScanController: ...@@ -117,7 +117,7 @@ class ScanController:
def _getCacheDirStamp(): def _getCacheDirStamp():
# pro archiv logu pouzit timestamp: #return "current" # pro archiv logu pouzit timestamp: #return "current"
return datetime.datetime.fromtimestamp(time.time()).strftime('%y%m%d%H%M%S') return datetime.datetime.fromtimestamp(time.time()).strftime('%y%m%d%H%M%S')
...@@ -136,7 +136,7 @@ class ScanController: ...@@ -136,7 +136,7 @@ class ScanController:
""" """
logDir = ScanController._assureDir(Config.LOG_DIR + str(self.profile) + "-log/") logDir = ScanController._assureDir(Config.LOG_DIR + str(self.profile) + "-log/")
cacheDir = ScanController._assureDir(Config.CACHE_DIR + Domains.domain2dir(self.url) + "/" + ScanController._getCacheDirStamp() + "/") cacheDir = ScanController._assureDir(Config.CACHE_DIR + Domains.domain2dir(self.url) + "/" + ScanController._getCacheDirStamp() + "/")
# info pro FF # info pro FF
with open(logDir + ScanController.FF_INFO_FILE,"w") as f: # v logDiru mu dame odkaz do cacheDiru with open(logDir + ScanController.FF_INFO_FILE,"w") as f: # v logDiru mu dame odkaz do cacheDiru
...@@ -145,9 +145,9 @@ class ScanController: ...@@ -145,9 +145,9 @@ class ScanController:
return logDir, cacheDir return logDir, cacheDir
def _loadProfileQueue(self): def _loadProfileQueue(self):
#load queue from config file #load queue from config file
try: try:
...@@ -167,7 +167,7 @@ class ScanController: ...@@ -167,7 +167,7 @@ class ScanController:
def unbookProfile(self): def unbookProfile(self):
def dump(): def dump():
with open(Config.configFile, 'w') as f: with open(Config.configFile, 'w') as f:
json.dump(self.queueFF, f) json.dump(self.queueFF, f)
#logging.debug("UNKBOOK") #logging.debug("UNKBOOK")
try: try:
self.queueFF.pop(self.profile) self.queueFF.pop(self.profile)
...@@ -176,7 +176,7 @@ class ScanController: ...@@ -176,7 +176,7 @@ class ScanController:
logging.debug("Unbook failed") logging.debug("Unbook failed")
logging.debug(self.queueFF) logging.debug(self.queueFF)
raise raise
except OSError: except OSError:
logging.debug("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile)) logging.debug("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile))
time.sleep(10) # XX jestli funkcionalitu zachovat, dat sem pocitadlo, at je na konzoli videt akce time.sleep(10) # XX jestli funkcionalitu zachovat, dat sem pocitadlo, at je na konzoli videt akce
try: try:
...@@ -185,25 +185,24 @@ class ScanController: ...@@ -185,25 +185,24 @@ class ScanController:
logging.debug("({}) System se nezotavil.".format(self.profile)) logging.debug("({}) System se nezotavil.".format(self.profile))
return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze. return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze.
#logging.debug("UNKBOOKED") #logging.debug("UNKBOOKED")
def queue(self): def queue(self):
""" Ze souboru queue.cache nacte, ktery profil je volny a zabookuje ho""" """ Reads from queue.cache what profile is available and books it """
self._loadProfileQueue() self._loadProfileQueue()
self.profile = -1 self.profile = -1
for i2 in range(4): #na volny slot zkusime nekolikrat pockat for _ in range(4): #na volny slot zkusime nekolikrat pockat
for i in range(Config.profileCount): #i = 10 if i ==10: for i in range(Config.profileCount): #i = 10 if i ==10:
if self.queueFF.get(str(i)) == None: if self.queueFF.get(str(i)) == None:
self.profile = i self.profile = i
self.bookProfile() self.bookProfile()
break break
if self.profile == -1: if self.profile == -1:
logging.debug("(-1) PLNO, cekame par vterin") logging.debug("(-1) FULL, let's wait few secs")
time.sleep(randint(5, 10)) #pockame par vterin time.sleep(randint(5, 10)) #pockame par vterin
else: else:
break #volny slot jsme nasli, muzeme dal break # we found a free slot, let's proceed
#logging.debug(" profile " + str(self.profile ) + " queueFF:")
logging.debug(self.queueFF) logging.debug(self.queueFF)
#povedlo se zabookovat profil FF? #povedlo se zabookovat profil FF?
......
...@@ -8,16 +8,14 @@ from lib.model.dbp import Export ...@@ -8,16 +8,14 @@ from lib.model.dbp import Export
import logging import logging
import mimetypes import mimetypes
import os import os
import time
env = Environment() env = Environment()
env.loader = FileSystemLoader(Config.DIR + "templates/") env.loader = FileSystemLoader(Config.DIR + "templates/")
class Server(SimpleHTTPRequestHandler): class Server(SimpleHTTPRequestHandler):
def favicon(self): def favicon(self):
with open('favicon.ico', 'rb') as f: with open('favicon.ico', 'rb') as f:
self.output(f.read(), "image/x-icon") self.output(f.read(), "image/x-icon")
def render_template(self, filename, ** kwargs): def render_template(self, filename, ** kwargs):
...@@ -41,9 +39,9 @@ class Server(SimpleHTTPRequestHandler): ...@@ -41,9 +39,9 @@ class Server(SimpleHTTPRequestHandler):
with open(url, type) as f: with open(url, type) as f:
self.output(f.read(), contentType=mimetypes.guess_type(url)) self.output(f.read(), contentType=mimetypes.guess_type(url))
def do_GET(self): def do_GET(self):
path = self.path.split("/") path = self.path.split("/")
logging.debug("Request: {}".format(path[1])) logging.debug("Request: {}".format(path[1]))
if path[1] == "": if path[1] == "":
return self.homepage() return self.homepage()
...@@ -56,6 +54,6 @@ class Server(SimpleHTTPRequestHandler): ...@@ -56,6 +54,6 @@ class Server(SimpleHTTPRequestHandler):
api = Api(self.path) api = Api(self.path)
# send everything up, we are in an iframe # send everything up, we are in an iframe
self.render_template("_message.html", contents=api.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/") self.render_template("_message.html", contents=api.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/")
elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni
url = self.path.split("/", 2) url = self.path.split("/", 2)
self.output(Export.exportView(days=url[2])) self.output(Export.exportView(days=url[2]))
\ No newline at end of file
...@@ -17,7 +17,7 @@ if(1): # Do not print all queries to stderr. ...@@ -17,7 +17,7 @@ if(1): # Do not print all queries to stderr.
logger.setLevel(logging.WARNING) logger.setLevel(logging.WARNING)
class DbModel(Model): class DbModel(Model):
def assureConnection(): def assureConnection():
logging.debug("Assure connection.") logging.debug("Assure connection.")
...@@ -40,7 +40,7 @@ class DbModel(Model): ...@@ -40,7 +40,7 @@ class DbModel(Model):
"""A base model that will use our MySQL database""" """A base model that will use our MySQL database"""
def connect(): def connect():
logging.debug("connecting db....") logging.debug("connecting db....")
#DbModel.Meta.myDb = Config.myDB #DbModel.Meta.myDb = Config.myDB
# Config.myDB.connect() # XX kupodivu toto neni potreba # Config.myDB.connect() # XX kupodivu toto neni potreba
#logging.debug(Whitelist.select().count()) #logging.debug(Whitelist.select().count())
...@@ -107,6 +107,8 @@ class Export(DbModel): ...@@ -107,6 +107,8 @@ class Export(DbModel):
q += "NOW() - INTERVAL {} DAY ".format(int(days)) q += "NOW() - INTERVAL {} DAY ".format(int(days))
else: else:
q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)" q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
q += " GROUP BY concat(`ip`,`port`) " # group by concat may be a performance issue
q += " ORDER BY `timestamp` DESC"
logging.debug(q) logging.debug(q)
rq = RawQuery(Turris, q).execute() rq = RawQuery(Turris, q).execute()
print (rq) print (rq)
...@@ -164,17 +166,17 @@ class Turris(DbModel): ...@@ -164,17 +166,17 @@ class Turris(DbModel):
except: except:
logging.error("domain should have been inserted in the database, but it hasnt been") logging.error("domain should have been inserted in the database, but it hasnt been")
logging.debug("vote error") logging.debug("vote error")
raise raise
ipList = list(set([o.ip for o in rows if o.ip != None])) ipList = list(set([o.ip for o in rows if o.ip != None]))
count = 0 count = 0
if ipList: if ipList:
count += Turris.update(status=str(status)).where(Turris.ip << ipList).execute() count += Turris.update(status=str(status)).where(Turris.ip << ipList).execute()
count += Turris.update(status=str(status)).where(Turris.remoteHost == host).execute() # ovlivnit i remoteHost s IP = NULL count += Turris.update(status=str(status)).where(Turris.remoteHost == host).execute() # ovlivnit i remoteHost s IP = NULL
logging.debug("vote:" + vote + " host:" + host + " count:" + str(count)) logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))
return str(count) + " updated" return str(count) + " updated"
except: except:
return "no update, didnt find ip" return "no update, didnt find ip"
class Whitelist(DbModel): class Whitelist(DbModel):
id = PrimaryKeyField() id = PrimaryKeyField()
timestamp = DateTimeField(datetime.datetime.now()) timestamp = DateTimeField(datetime.datetime.now())
......
...@@ -4,5 +4,5 @@ echo "mdmaug-launch start" >> ~/log.log ...@@ -4,5 +4,5 @@ echo "mdmaug-launch start" >> ~/log.log
pkill -f mdmaug.py pkill -f mdmaug.py
pkill -f Xvfb pkill -f Xvfb
export PYTHONPATH=$PYTHONPATH:/opt/mdmaug/mdmaug/ export PYTHONPATH=$PYTHONPATH:/opt/mdmaug/mdmaug/
cd /opt/mdmaug/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t yourtag cd /opt/mdmaug/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t mdmaugtag
whoami >> ~/log.log whoami >> ~/log.log
\ No newline at end of file
...@@ -27,7 +27,7 @@ vdisplay = Xvfb() ...@@ -27,7 +27,7 @@ vdisplay = Xvfb()
vdisplay.start() vdisplay.start()
try: try:
print('Listening at https://0.0.0.0:{}'.format(Config.APP_PORT)) print('Listening at https://0.0.0.0:{}'.format(Config.APP_PORT))
for _ in range(2): # XX Config.profileCount for _ in range(Config.profileCount):
threading.Thread(target=httpd.serve_forever).start() threading.Thread(target=httpd.serve_forever).start()
except (KeyboardInterrupt, SystemExit): except (KeyboardInterrupt, SystemExit):
vdisplay.stop() vdisplay.stop()
...@@ -39,7 +39,7 @@ How to debug mysql: ...@@ -39,7 +39,7 @@ How to debug mysql:
conn = pymysql.connect(host='localhost', user='root', passwd='lopuch', db='mdmaug', charset='utf8') conn = pymysql.connect(host='localhost', user='root', passwd='lopuch', db='mdmaug', charset='utf8')
cur = conn.cursor() cur = conn.cursor()
cur.execute("""SELECT name from turris JOIN status ON status.id = turris.status WHERE ip = %s""", (ip,)) cur.execute("""SELECT name from turris JOIN status ON status.id = turris.status WHERE ip = %s""", (ip,))
vote = self.cur.fetchone()[0] vote = self.cur.fetchone()[0]
logging.debug(vote) logging.debug(vote)
exit() exit()
......
file.reference.opt-mdmaug-installer=.
java.lib.path= java.lib.path=
platform.active=Python_3.5.1 platform.active=Python_3.5.1
python.lib.path= python.lib.path=
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment