Commit 10048ff6 authored by Edvard Rejthar's avatar Edvard Rejthar

jdu se podivat, co se zmenilo na implmentai http google safebrowsingu

parent 69a45e98
......@@ -21,7 +21,7 @@ const data = require('sdk/self').data;
// **********
//profile dir
// **********
logDir = "/tmp/mdm/";
logDir = "/home/mdmaug/.cache/mdmaug-scans/_tmp/" //"/tmp/mdm/"; temp byl maly 200 MB - a zabira cennou RAMku
profileDir = OS.Constants.Path.profileDir;
profileName = profileDir.substr(profileDir.lastIndexOf("/") + 1);
console.log("profile name: " + profileName);
......
test readme2
\ No newline at end of file
MDMaug scan a website for other parasite hosts and commands.
\ No newline at end of file
This diff is collapsed.
......@@ -26,11 +26,13 @@ class Crawl(defaultdict):
def __init__(self, state = None, logDir = None, cacheDir = None):
def __init__(self, host = None, state = None, logDir = None, cacheDir = None):
""" State muze obsahovat vystup __getstate__() (serializace YAMLem) """
self.default_factory = _Domain
self.screenfile = None # HTML output XXX
if host:
self.host = host
if logDir:
self.logDir = logDir
if cacheDir:
......@@ -106,18 +108,18 @@ class _Url(set):
#return "spyfile {} {}, vote {}".format(self.spyfile, self.city, self.vote)
def __init__(self):
self.spyfile = None # cesta k souboru se podezrelym kodem poustenym strankou
self.sourcefile = None # cesta k souboru se zdrojovym kodem
self.spyfile = [] # cesta k souboru se podezrelym kodem poustenym strankou
self.sourcefile = [] # cesta k souboru se zdrojovym kodem. Muze jich byt vice, http://seznam.cz/index.htm a https://seznam.cz/index.htm jsou oba pod domenou seznam.cz
def setSourcefile(self, sourcefile):
if self.sourcefile != None:
raise RuntimeError('sourcefile', 'uz bylo definovano') # na tohle nejsme pripraveni - asi funkce v main.js pro jmeno souboru je spatna, protoze je jina od Domains.funkce
self.sourcefile = sourcefile
def addSourcefile(self, sourcefile):
#if self.sourcefile != None:
# raise RuntimeError('sourcefile', 'uz bylo definovano ' + self.sourcefile) # na tohle nejsme pripraveni - asi funkce v main.js pro jmeno souboru je spatna, protoze je jina od Domains.funkce
self.sourcefile.append(sourcefile)
def setSpyfile(self, spyfile):
if self.spyfile != None:
raise RuntimeError('spyfile', 'uz bylo definovano') # na tohle nejsme pripraveni - asi funkce v main.js pro jmeno souboru je spatna, protoze je jina od Domains.funkce
self.spyfile = spyfile
def addSpyfile(self, spyfile):
#if self.spyfile != None:
# raise RuntimeError('spyfile', 'uz bylo definovano') # na tohle nejsme pripraveni - asi funkce v main.js pro jmeno souboru je spatna, protoze je jina od Domains.funkce
self.spyfile.append(spyfile)
class _Address(set):
""" Adresa navstivene domeny """
......@@ -139,7 +141,7 @@ class _Address(set):
Example (and yaml-serialization check):
c = Crawl()
c["seznam.cz"].urls["/aurl"].spyfile = "/soubor-spyfil"
c["seznam.cz"].urls["/aurl"].sourcefiles.add("/1.source")
c["seznam.cz"].urls["/aurl"].sourcefiles.add("/1.source") tady ma byt asi append, ne?
c["seznam.cz"].urls["/aurl"].sourcefiles.add("/2.source")
c["seznam.cz"].addresses["8.8.8.8"]
c["seznam.cz"].addresses["9.5.2.1"]
......
......@@ -36,9 +36,13 @@ class MetadataParser:
t = threading.Thread(target=self.addDomain, args=(crawl[domain], domain, domainThreadNumber))
threads.append(t)
t.start()
print("EDVARD 10")
#konsolidovat informace o domenach
for thread in threads:
thread.join()
print("EDVARD 11")
print("EDVARD 12")
print("EDVARD END METADATA")
#sesumirovat informace o domene
......@@ -67,11 +71,13 @@ class MetadataParser:
t.start()
#konsolidovat informace o IP v domenach
for thread in threads:
thread.join()
thread.join()
print("("+str(threadNumber),"EDVARD 9")
if len(threads) == 0: #domena zadne IP nema, je pozastavena
#presto chceme evil host zapsat - alespon s ip null
#vote =
self.addAddress(None, None, domainEncountered,crawlDomain.pdns,0.0)
self.addAddress(None, None, domainEncountered,crawlDomain.pdns,0,0)
print("("+str(threadNumber),"EDVARD 8")
#if vote == None:vote = ""
#crawlDomain.vote = vote
......@@ -113,15 +119,19 @@ class MetadataParser:
pass
#logging.debug("("+str(threadNumber)+","+str(domainThreadNumber)+") " + "ADDRESS " + " ip: " + str(ip)) #+ str(vote)
print("("+str(threadNumber),"EDVARD 4")
if ip != None:
#crawlDomainIp.vote = vote
#kontaktovat externi geoIP sluzbu
print("("+str(threadNumber),"EDVARD 5")
crawlDomainIp.country, crawlDomainIp.city = Domains.ip2countryAndCity(ip)
# kontaktovat PDNS
pdns.update(Domains.ip2pdnsDomains(ip))
print("("+str(threadNumber),"EDVARD 6")
return None
else: #zadna ip neni k dispozici, domena je asi propadla, hlas patri jmenu domeny
#return vote
print("("+str(threadNumber),"EDVARD 7")
return None
......
......@@ -23,7 +23,7 @@ class SpyParser:
continue
crawl[Domains.url2domain(url)].urls[Domains.url2path(url)].setSpyfile(path)
crawl[Domains.url2domain(url)].urls[Domains.url2path(url)].addSpyfile(path)
#spy = "<h3>Nebezpečné funkce</h3>"
#with open(spyfile, 'r') as f:
......
......@@ -43,8 +43,10 @@ class TrafficLogParser:
# kdyz je domena whitelistovana, preskocime ji
if (Whitelist.matches(url)):
continue
crawl[Domains.url2domain(url)].urls[Domains.url2path(url)].setSourcefile(path)
print("EDVARD")
print(Domains.url2domain(url), Domains.url2path(url), path)
crawl[Domains.url2domain(url)].urls[Domains.url2path(url)].addSourcefile(path)
def nicifyFile(sourcefile):
""" Vraci zhezceny vypis .tmp filu se zdrojovymi kodu """
......
......@@ -103,7 +103,7 @@ class ScanController:
subprocess.call([command], shell=True)
print("({}) stopped!".format(self.profile))
# shromazdit informace z analyz
crawl = Crawl(logDir = logDir, cacheDir = cacheDir)
crawl = Crawl(host = self.url, logDir = logDir, cacheDir = cacheDir)
expiration = 0
while os.path.isfile(logfile) == False: # i po zavreni FF nekdy trva, nez se soubor zapise
......
......@@ -6,10 +6,10 @@ from peewee import MySQLDatabase
class Config:
profileCount = 21 #pocet profilu vytvorenych ve firefoxu. Tyto je treba vytvorit rucne. Nazev profilu je cislo - 0,1...
browser = 'firefox' #iceweasel, firefox. Ktery prohlizec se spousti.
configFile = '/tmp/mdm/queue.cache'
configFile = '/home/mdmaug/.cache/mdmaug-scans/_tmp/queue.cache' # RAM disk byl maly: '/tmp/mdm/queue.cache'
APP_PORT = 8000
APP_DOMAIN = 'https://172.20.7.10:' + str(APP_PORT) #csirt.csirt.office.nic.cz
LOG_DIR = "/tmp/mdm/"
LOG_DIR = "/home/mdmaug/.cache/mdmaug-scans/_tmp/" # X /tmp/mdm/
CACHE_DIR = "/home/mdmaug/.cache/mdmaug-scans/"
myDB = ""
lock = threading.RLock() # doufam, ze kdyz je lock tady, ze je funknci. Closure...? XX nejak otestovat
......
......@@ -43,10 +43,13 @@ class Domains:
url = re.sub('[^a-z0-9\.]', '', url) # nechat jen pratelske znaky
return url
def getPdnsLink(ip):
return 'http://pdns.cert.at/p/dns?qry=' + ip
def ip2pdnsDomains(ip):
try:
# XX mohl bych sortovat dle 2nd domeny. Pripadne oriznout 3rd domenu, nechat jen 2nd. Tam ale musim osetrit problemove dvojite tld - co.uk, gov.ua...
pdns = urllib.request.urlopen('http://pdns.cert.at/p/dns?qry=' + ip).read().decode("utf-8")
pdns = urllib.request.urlopen(Domains.getPdnsLink(ip)).read().decode("utf-8")
items = re.findall("<div class='x[BA]'>(.*)</div>", pdns)
return items
except Exception as e:
......@@ -64,4 +67,21 @@ class Domains:
logging.debug("neumim dekodovat")
except Exception as e:
logging.debug("hostip.info down: " + str(e))
return None, None
\ No newline at end of file
return None, None
##
# Kontaktuje sluzbu safebrowsing a snazi se z jejich nekonzistentnich udaju vycist, zda kdyz na URL clovek pristoupi, objevi se cervena stranka.
# To zavisi na vicero udajich - napr. na tom, kdy si vas prohlizec updatoval seznam oproti sluzbe safebrowsing.
# Taky je mozne, ze sluzba zmenila wording. Mnoho zdaru!
#
# @param format 'bool' Vraci bool True/False/None, nebo 'attr' vraci int "1"/"0"/"" pro atribut
def isSuspicious(domain, output='bool'):
contents = urllib.request.urlopen('http://www.google.com/safebrowsing/diagnostic?site=' + domain).read().decode("utf-8")
with open("debugsf.tmp","a") as f:
f.write(contents + "\n\n")
if "Site is listed as suspicious" in contents:
return True if output == 'bool' else "1"
elif "This site is not currently listed as suspicious." in contents:
return False if output == 'bool' else "0"
else:
return None if output == 'bool' else ""
\ No newline at end of file
......@@ -63,11 +63,11 @@ class Server (SimpleHTTPRequestHandler):
self.static_file("static/" + path[1])
elif path[1] == "rest": # /rest/analyze/web
cmd = path[2]
url = path[2] if len(path) > 3 else ""
#url = path[2] if len(path) > 3 else ""
rest = Rest(self.path)
#poslat vsechna data vys, pokud jsme vlozeni v iframe (https-mdm nemuze nacitat z http stroje - mixed content)
self.render_template("templates/_message.html", contents = rest.run(cmd), cmd=cmd, url=url, destination="https://mdm.labs.nic.cz/")
self.render_template("templates/_message.html", contents = rest.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.labs.nic.cz/")
elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni
url = self.path.split("/", 2)
self.output(Export.exportView(days = url[2]))
\ No newline at end of file
......@@ -2,7 +2,7 @@
# This file may be placed to the ~ location and launched by CRON. 03 1,7,13,19 * * * ~/mdmaug-launch
echo "mdmaug-launch start" >> ~/log.log
pkill -f mdmaug.py
pkill Xvbf
pkill -f Xvfb
export PYTHONPATH=$PYTHONPATH:/home/mdmaug/mdmaug/
cd ~/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t yourtag
whoami >> ~/log.log
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
#import cgi
from lib.config import Config
from lib.analysis.parser.spy_parser import SpyParser
from lib.dbp import Turris, Whitelist
from lib.config import Config
from lib.dbp import Turris
from lib.dbp import Whitelist
from lib.domains import Domains
class CrawlView:
""" HTML vypis instance Crawl """
def outputHtml(crawl):
#print(crawl.keys())
r = ""
r = "<div class='analysis' data-host='{}' data-safebrowsing-suspicious='{}'>".format(crawl.host, Domains.isSuspicious(crawl.host, 'attr'))
if crawl.screenfile:
with open(crawl.screenfile,"r") as f:
r += "<div class'screenshot'><img class='thumbnail' src='data:image/png;base64,{}' /></div>".format(f.read())
with open(crawl.screenfile, "r") as f:
r += "<div class='screenshot'><img class='thumbnail' src='data:image/png;base64,{}' /></div>".format(f.read())
for domain in crawl.keys():
if (Whitelist.matches(domain)):
continue
r += "<div class=web vote='{}'>".format(Turris.getVote(host = domain)) # crawl[domain].vote
r += "<div class=web data-vote='{}' data-safebrowsing-suspicious='{}'>".format(Turris.getVote(host=domain), Domains.isSuspicious(domain, 'attr')) # crawl[domain].vote
r += "<span class=domain>{}</span>".format(domain)
r += "<div class=addresses>"
......@@ -31,8 +33,11 @@ class CrawlView:
r += "</div>"
if crawl[domain].pdns:
r += "<span class=pdns>"
for pdnsDomain in crawl[domain].pdns:
for i, pdnsDomain in enumerate(crawl[domain].pdns):
r += "<span>{}</span>".format(pdnsDomain)
if i == 10 and (len(crawl[domain].pdns) - i > 5): # kdyz je 15+ domen, vypise jich jen 10
r += "... ({})".format(len(crawl[domain].pdns)-i) #XX link nelze dát, protože je tolik linku, kolik IP ma navstiveny host. Lze udelat, ze odkaz povede na vypis vsech domen. XDomains.getPdnsLink(crawl[domain])
break
r += "</span>"
r += "</div>"
......@@ -42,17 +47,20 @@ class CrawlView:
urlI = crawl[domain].urls[url]
r += "<li>"
r += "<span class='value'>{}</span>".format(url)
if(urlI.spyfile):
text, shorten = SpyParser.getShort(urlI.spyfile)
r += "<div class='spyfile'>{}</div>".format(text)
if shorten: # text byl zkracen, odkaz na plnou verzi
r += "<a href='{}' class='spyfile-full'>-></a>".format(Config.APP_DOMAIN + "/rest/nicify/" + urlI.spyfile)
if(urlI.sourcefile):
#with open(uriI.sourcefile,"r") as f:
r += "<a href='{}' class='sourcefile'>-></a>".format(Config.APP_DOMAIN + "/rest/nicify/" + urlI.sourcefile)
if urlI.spyfile: # XX tento radek lze v listopadu 2015 dat pryc
for file in urlI.spyfile:
text, shorten = SpyParser.getShort(file)
r += "<div class='spyfile'>{}</div>".format(text)
if shorten: # text byl zkracen, odkaz na plnou verzi
r += "<a href='{}' class='spyfile-full'>-></a>".format(Config.APP_DOMAIN + "/rest/nicify/" + file)
if urlI.sourcefile: # XX tento radek lze v listopadu 2015 dat pryc
for file in urlI.sourcefile:
#with open(uriI.sourcefile,"r") as f:
r += "<a href='{}' class='sourcefile'>-></a>".format(Config.APP_DOMAIN + "/rest/nicify/" + file)
r += "</li>"
r += "</ul>"
r += "</div>"
r += "</div>"
return r
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment