Commit a640614e authored by Edvard Rejthar's avatar Edvard Rejthar

unikátní záznamy ip X port v exportu

kompletní migrace na nový stroj
parent d80dcfce
......@@ -5,7 +5,7 @@ Scans a website for a sign of a parasite hosts or commands.
## Installation
1. ```git clone git@gitlab.labs.nic.cz:csirt/mdmaug.git /tmp/mdmaug```
2. edit config.py
2. edit mdmaug/lib/config.py
3. ```/tmp/mdmaug/INSTALL```
### Notes
......@@ -17,12 +17,12 @@ Scans a website for a sign of a parasite hosts or commands.
## What is done to Firefox profiles?
We want no block nor safebrowsing warning. If you created the profiles manually, you'd use ```firefox -P```, the profiles names being: 0,1...
For about:config changes, see prefs.js. IE:
We want no block nor safebrowsing warning. If you created the profiles manually, you'd use ```firefox -P```, the profiles names being: 0,1...
For about:config changes, see prefs.js. IE:
* toolkit.startup.max_resumed_crashes = -1 (protoze i kdyz prohlizec nekdy killnu, nesmi me pri spusteni otravovat gui popupem)
* network.http.accept-encoding = "" # ukladame streamy, ale neumim je rozzipovat
* extensions.autoDisableScopes = "0" # moznost instalovat ze vsech umisteni
* browser.selfsupport.url = "" # tato moznost normalne v about:config neni, ale omezuje to nejake zbytecnou telemetrii, viz Mozilla Heartbeat
* # nepamatovat si historii (Preferences / Privacy / Firefox will use custom settings for history / Clear history when closes / Setting / All)
* # nejsem si jist, nakolik to funguje, zrejme dost
* ...
\ No newline at end of file
* ...
# XX shouldnt I delete this file?
su - mdmaug -c 'cd /home/mdmaug/mdmaug/ ; python3 mdmaug.py'
pkill python3 #pri Ctrl+C v prikazu su se uzavre jen terminal, ale ne uz python skript. Takhle to zas zabije veskery Python, ale sandboxovy-zavirovany uzivatel mdmaug stejne ma byt jen na spousteni tohohle skriptu.
\ No newline at end of file
......@@ -6,23 +6,23 @@ from lib.controller.scan_controller import ScanController
from lib.model.dbp import Status, Export, Turris, Whitelist
from lib.analysis.parser.traffic_log_parser import TrafficLogParser
class Api:
class Api:
website = "" # http://site.cz
websiteDomain = "" # site.cz
def __init__(self, path):
self.path = path
def run(self, cmd):
""" Accept command """
if cmd == "analyze":
return ScanController().launch(self.path)
if cmd == "analyze=cached":
return ScanController().launch(self.path, cached = 1)
if cmd == "analyze=weekcache":
if cmd == "analyze=weekcache":
return ScanController().launch(self.path, cached = 7)
if cmd == "analyze=oldcache":
return ScanController().launch(self.path, cached = True)
return ScanController().launch(self.path, cached = True)
elif cmd == "export=view": # XX deprecated?
return Export.exportView()
elif cmd == "export=confirm": # XX deprecated?
......@@ -49,16 +49,16 @@ class Api:
return self.whitelist()
elif cmd == "reset":
Server.reset()
return "reset"
return "reset"
def reset():
logging.debug("resetting running browsers")
logging.debug("resetting running browsers")
with open(Config.configFile, 'w') as f: # clear the queue
json.dump({}, f)
json.dump({}, f)
subprocess.call(["pkill", Config.browser]) # kill frozen browsers
#prida 2ld domenu mezi whitelistovane
def whitelist(self):
......
......@@ -39,7 +39,7 @@ class ScanController:
if cached:
# """ Pokud je k dispozici analyza, vratit ji """
dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/"
dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/"
if os.path.isdir(dir):
snapdirs = [str(dir + subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu
if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir+subdir + "/"+ScanController.CRAWL_FILE)]
......@@ -54,7 +54,7 @@ class ScanController:
logging.debug("({-1}) Cachovana analyza nenalezena")
# provest novou analyzu
if self.queue(): # /api/analyze/web - zaradi web do fronty
if self.queue(): # /api/analyze/web - zaradi web do fronty
print ("({}) start crawl".format(self.profile))
self.url = Domains.assureUrl(url)
try:
......@@ -104,7 +104,7 @@ class ScanController:
logging.debug("({}) time is run!".format(self.profile))
raise FileNotFoundError("time is run - browser expired")
time.sleep(1)
NsprLogParser(logfile, crawl)
self.unbookProfile() # uvolnit browser profil
......@@ -117,7 +117,7 @@ class ScanController:
def _getCacheDirStamp():
def _getCacheDirStamp():
# pro archiv logu pouzit timestamp: #return "current"
return datetime.datetime.fromtimestamp(time.time()).strftime('%y%m%d%H%M%S')
......@@ -136,7 +136,7 @@ class ScanController:
"""
logDir = ScanController._assureDir(Config.LOG_DIR + str(self.profile) + "-log/")
cacheDir = ScanController._assureDir(Config.CACHE_DIR + Domains.domain2dir(self.url) + "/" + ScanController._getCacheDirStamp() + "/")
# info pro FF
with open(logDir + ScanController.FF_INFO_FILE,"w") as f: # v logDiru mu dame odkaz do cacheDiru
......@@ -145,9 +145,9 @@ class ScanController:
return logDir, cacheDir
def _loadProfileQueue(self):
#load queue from config file
try:
......@@ -167,7 +167,7 @@ class ScanController:
def unbookProfile(self):
def dump():
with open(Config.configFile, 'w') as f:
json.dump(self.queueFF, f)
json.dump(self.queueFF, f)
#logging.debug("UNKBOOK")
try:
self.queueFF.pop(self.profile)
......@@ -176,7 +176,7 @@ class ScanController:
logging.debug("Unbook failed")
logging.debug(self.queueFF)
raise
except OSError:
except OSError:
logging.debug("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile))
time.sleep(10) # XX jestli funkcionalitu zachovat, dat sem pocitadlo, at je na konzoli videt akce
try:
......@@ -185,25 +185,24 @@ class ScanController:
logging.debug("({}) System se nezotavil.".format(self.profile))
return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze.
#logging.debug("UNKBOOKED")
def queue(self):
""" Ze souboru queue.cache nacte, ktery profil je volny a zabookuje ho"""
""" Reads from queue.cache what profile is available and books it """
self._loadProfileQueue()
self.profile = -1
for i2 in range(4): #na volny slot zkusime nekolikrat pockat
for _ in range(4): #na volny slot zkusime nekolikrat pockat
for i in range(Config.profileCount): #i = 10 if i ==10:
if self.queueFF.get(str(i)) == None:
self.profile = i
self.profile = i
self.bookProfile()
break
if self.profile == -1:
logging.debug("(-1) PLNO, cekame par vterin")
logging.debug("(-1) FULL, let's wait few secs")
time.sleep(randint(5, 10)) #pockame par vterin
else:
break #volny slot jsme nasli, muzeme dal
break # we found a free slot, let's proceed
#logging.debug(" profile " + str(self.profile ) + " queueFF:")
logging.debug(self.queueFF)
#povedlo se zabookovat profil FF?
......
......@@ -8,16 +8,14 @@ from lib.model.dbp import Export
import logging
import mimetypes
import os
import time
env = Environment()
env.loader = FileSystemLoader(Config.DIR + "templates/")
class Server(SimpleHTTPRequestHandler):
class Server(SimpleHTTPRequestHandler):
def favicon(self):
with open('favicon.ico', 'rb') as f:
with open('favicon.ico', 'rb') as f:
self.output(f.read(), "image/x-icon")
def render_template(self, filename, ** kwargs):
......@@ -41,9 +39,9 @@ class Server(SimpleHTTPRequestHandler):
with open(url, type) as f:
self.output(f.read(), contentType=mimetypes.guess_type(url))
def do_GET(self):
def do_GET(self):
path = self.path.split("/")
logging.debug("Request: {}".format(path[1]))
if path[1] == "":
return self.homepage()
......@@ -56,6 +54,6 @@ class Server(SimpleHTTPRequestHandler):
api = Api(self.path)
# send everything up, we are in an iframe
self.render_template("_message.html", contents=api.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/")
elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni
elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni
url = self.path.split("/", 2)
self.output(Export.exportView(days=url[2]))
\ No newline at end of file
......@@ -17,7 +17,7 @@ if(1): # Do not print all queries to stderr.
logger.setLevel(logging.WARNING)
class DbModel(Model):
class DbModel(Model):
def assureConnection():
logging.debug("Assure connection.")
......@@ -40,7 +40,7 @@ class DbModel(Model):
"""A base model that will use our MySQL database"""
def connect():
logging.debug("connecting db....")
logging.debug("connecting db....")
#DbModel.Meta.myDb = Config.myDB
# Config.myDB.connect() # XX kupodivu toto neni potreba
#logging.debug(Whitelist.select().count())
......@@ -107,6 +107,8 @@ class Export(DbModel):
q += "NOW() - INTERVAL {} DAY ".format(int(days))
else:
q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
q += " GROUP BY concat(`ip`,`port`) " # group by concat may be a performance issue
q += " ORDER BY `timestamp` DESC"
logging.debug(q)
rq = RawQuery(Turris, q).execute()
print (rq)
......@@ -164,17 +166,17 @@ class Turris(DbModel):
except:
logging.error("domain should have been inserted in the database, but it hasnt been")
logging.debug("vote error")
raise
ipList = list(set([o.ip for o in rows if o.ip != None]))
raise
ipList = list(set([o.ip for o in rows if o.ip != None]))
count = 0
if ipList:
count += Turris.update(status=str(status)).where(Turris.ip << ipList).execute()
count += Turris.update(status=str(status)).where(Turris.remoteHost == host).execute() # ovlivnit i remoteHost s IP = NULL
logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))
logging.debug("vote:" + vote + " host:" + host + " count:" + str(count))
return str(count) + " updated"
except:
return "no update, didnt find ip"
class Whitelist(DbModel):
id = PrimaryKeyField()
timestamp = DateTimeField(datetime.datetime.now())
......
......@@ -4,5 +4,5 @@ echo "mdmaug-launch start" >> ~/log.log
pkill -f mdmaug.py
pkill -f Xvfb
export PYTHONPATH=$PYTHONPATH:/opt/mdmaug/mdmaug/
cd /opt/mdmaug/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t yourtag
cd /opt/mdmaug/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t mdmaugtag
whoami >> ~/log.log
\ No newline at end of file
......@@ -27,7 +27,7 @@ vdisplay = Xvfb()
vdisplay.start()
try:
print('Listening at https://0.0.0.0:{}'.format(Config.APP_PORT))
for _ in range(2): # XX Config.profileCount
for _ in range(Config.profileCount):
threading.Thread(target=httpd.serve_forever).start()
except (KeyboardInterrupt, SystemExit):
vdisplay.stop()
......@@ -39,7 +39,7 @@ How to debug mysql:
conn = pymysql.connect(host='localhost', user='root', passwd='lopuch', db='mdmaug', charset='utf8')
cur = conn.cursor()
cur.execute("""SELECT name from turris JOIN status ON status.id = turris.status WHERE ip = %s""", (ip,))
vote = self.cur.fetchone()[0]
vote = self.cur.fetchone()[0]
logging.debug(vote)
exit()
......
file.reference.opt-mdmaug-installer=.
java.lib.path=
platform.active=Python_3.5.1
python.lib.path=
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment