Skip to content

M
mdmaug
Commit 43f6307b authored by Edvard Rejthar's avatar Edvard Rejthar
Browse files
Options

#29 working well 

Signed-off-by: Edvard Rejthar's avatarEdvard Rejthar <edvard.rejthar@nic.cz>
parent 59f31252
Hide whitespace changes
Inline Side-by-side
Showing with 54 additions and 66 deletions
.mozilla/firefox/0/prefs.js
View file @ 43f6307b
......@@ -87,6 +87,7 @@ user_pref("media.gmp-gmpopenh264.version", "1.6");
user_pref("media.gmp-manager.buildID", "20170201180315");
user_pref("media.gmp-manager.lastCheck", 1486578560);
user_pref("media.gmp.storage.version.observed", 1);
user_pref("network.captive-portal-service.enabled", false); // don't hit http://detectportal.firefox.com/success.txt when browser opens
user_pref("network.cookie.prefsMigrated", true);
user_pref("network.http.accept-encoding", ""); // ukladame streamy, ale neumim je rozzipovat
user_pref("network.http.accept-encoding.secure", "");
......
README.md
View file @ 43f6307b
......@@ -16,9 +16,7 @@ Scans a website for a sign of a parasite hosts or commands.
### Notes
* If you use NoScript, make sure it doesn't block the MDM-Augmented server.
* Certificate error: Make sure that the browser doesn't blockt the MDM-Augmented server if used from MDM.
* If you want other count of profiles than 21, change INSTALL + config.py + profiles.ini
* If you want other count of profiles than 21, change `./INSTALL` + `mdmaug/lib/config.py` + `.mozilla/firefox/profiles.ini`
* You may put ```03 1,7,13,19 * * * ~/mdmaug-launch``` in ```crontab -e``` of user mdmaug.
* We are using Python3.6+, Firefox 62.0
......@@ -28,6 +26,9 @@ Scans a website for a sign of a parasite hosts or commands.
### Troubleshooting
* If you use NoScript, make sure it doesn't block the MDM-Augmented server.
* Certificate error: Make sure that the browser doesn't block the MDM-Augmented server if used from MDM.
#### Debugging session
I'm launching it like this:
......
mdmaug/lib/controller/api.py
View file @ 43f6307b
......@@ -39,7 +39,7 @@ class Api:
elif "scan" in params:
if "date" not in params:
params["date"] = ""
crawl = ScanController().get_scan(escape(params["scan"]), scan=escape(params["date"]))
crawl = ScanController().get_scan(domain2dir(escape(params["scan"])), scan=escape(params["date"]))
elif "prune" in params:
return ScanController.prune()
elif "nicify" in params:
......@@ -59,6 +59,8 @@ class Api:
return CrawlView.output_json(crawl)
else:
return CrawlView.output_html(crawl)
else:
return "Unspecified error :("
@staticmethod
def aggregate(params):
......@@ -74,11 +76,7 @@ class Api:
limit = request.args.get('limit', default=params["paging"], type=int)
offset = request.args.get('offset', default=0, type=int)
vote_filter = params.get("filter", None)
if vote_filter == "-":
vote_filter = None
domains, hosts, total = Encounter.by_origin_count(limit, offset, vote_filter)
domains, hosts, total = Encounter.by_origin_count(limit, offset, params)
for domain, host in zip(domains, hosts):
if host in crawl: # we already have a domain that connected to the host
continue
......
mdmaug/lib/controller/server.py
View file @ 43f6307b
......@@ -4,7 +4,7 @@ import re
from html import escape
from flask import Blueprint, send_from_directory, render_template, request, make_response
from wtforms import Form, Label, SelectField, IntegerField
from wtforms import Form, Label, SelectField, IntegerField, StringField
from wtforms.fields import BooleanField
from wtforms.widgets.html5 import NumberInput
......@@ -62,9 +62,11 @@ def homepage():
class AggregationOptions(Form):
order = SelectField('Order', choices=[('origins', 'origins count'), ('date', 'date seen'), ('ips', 'IP count'),
('unordered', 'unordered (slow)')])
('unordered', 'unordered (extremely slow)')])
paging = IntegerField('Paging', widget=NumberInput(max=100), default=100)
filter = SelectField('Filter', choices=[("-", "-")]+[(v, "vote: " + k) for k, v in Status.enum.items()])
filter = SelectField('Filter', choices=[("-", "-")] + [(v, "vote: " + k) for k, v in Status.enum.items()])
host_regex = StringField('Host regex', render_kw={"placeholder": ".org$"})
host_regex_not = StringField('Host regex not', render_kw={"placeholder": ".(cz|com|org|net)$"})
# if request.method == 'POST':
# name = request.form['name']
......
mdmaug/lib/model/dbp.py
View file @ 43f6307b
......@@ -206,7 +206,7 @@ class Encounter(DbModel):
cls.relateds[host] = [i for i in origins.split(",") if i != ignore_host]
@classmethod
def by_origin_count(cls, limit=None, offset=None, vote_filter=None):
def by_origin_count(cls, limit=None, offset=None, params={}):
""" Returns set of origins ordered by the count of potentially evil host connecting to them.
XX DB should have scan_timestamp :( which differs from creation+vote updated timestamp; then we could filter
by date from/to aggregation
......@@ -214,31 +214,27 @@ class Encounter(DbModel):
Returns domain that seen the tracked host and the host.
"""
vv = 0
for i in Encounter.select(Encounter.origin,
Encounter.host,
fn.Group_Concat(fn.Distinct(Encounter.origin)).alias("gr")) \
.group_by(Encounter.host) \
.order_by(fn.Count("*").desc()) \
.limit(int(limit)) \
.offset(int(offset)):
# ZDEEEEEEEEE predelej to dolu a dej pak v GUI moznost regex filteru na host
#print(i.ccc, i.gr)
vv += 1
# if i.origin == "vino-magnum.cz":
# import ipdb; ipdb.set_trace()
# if i.host == "ls.hit.gemius.pl":
# import ipdb;
# ipdb.set_trace()
break
query = Encounter.select(Encounter.origin,
Encounter.host,
fn.Group_Concat(fn.Distinct(Encounter.origin))) \
.group_by(Encounter.host) \
.order_by(fn.Count("*").desc())
vote_filter = params.get("filter", None)
if vote_filter and vote_filter != "-":
query = query.where(Encounter.status == int(vote_filter))
query = Encounter.select(Encounter.origin,
Encounter.host,
fn.Group_Concat(fn.Distinct(Encounter.origin)).alias("gr")) \
.group_by(Encounter.host) \
.order_by(fn.Count("*").desc())
host_regex = params.get("host_regex", None)
if host_regex:
query = query.where(Encounter.host.regexp(host_regex))
host_regex_not = params.get("host_regex_not", None)
if host_regex_not:
query = query.where(~Encounter.host.regexp(host_regex_not))
# cz|com|org|net
print(query.sql())
total = query.count()
if limit:
......@@ -246,26 +242,8 @@ class Encounter(DbModel):
if offset:
query = query.offset(int(offset))
print(total, "hej")
#import ipdb; ipdb.set_trace()
if 0:
total = Config.db.execute_sql("SELECT count(distinct host) FROM `encounter`"
"").fetchone()[0]
# WHERE host not regexp '\.(cz|com|org|net)'
q = "SELECT origin, host, group_concat(distinct origin) FROM `encounter`"
if vote_filter:
q += f" WHERE status = {int(vote_filter)}"
# q += "WHERE host not regexp '\.(cz|com|org|net)'" # XXX double where
q += " group by host ORDER BY count(*) DESC"
if limit:
q += f" LIMIT {int(limit)}"
if offset:
q += f" OFFSET {int(offset)}"
domains = []
hosts = []
# for origin, host, origins in Config.db.execute_sql(q):
for origin, host, origins in query.tuples().iterator():
cls.relateds[host] = origins.split(",")
domains.append(origin)
......
mdmaug/lib/parser/metadata_parser.py
View file @ 43f6307b
......@@ -91,7 +91,7 @@ class MetadataParser:
# this IP is not yet associated with domain
insert = True
elif not Encounter.select().where(Encounter.host == host, Encounter.origin == self.origin_domain).count():
# this encountered host is not yet associated with scanned domain ()
# this encountered host is not yet associated with scanned domain
insert = True
else:
insert = True
......
mdmaug/static/mdmaug-analysis.js
View file @ 43f6307b
......@@ -12,7 +12,7 @@ Messages:
const LOCAL_DESTINATION = new URL(location.href);
const VOTING_SELECTOR = ".analysis > form > [data-group] > .web > .voting > input[type=radio]";
const RELATED_SELECTOR = ".analysis > form > [data-group] > .web > .addresses > .related > span";
const RELATED_SELECTOR = ".analysis > form > [data-group] > .web > .addresses > .related > a";
const SCAN_SELECTOR = ".analysis > form > .scans > span";
//const APP_HOST = must be defined before // "https://217.31.202.41:8000";
var $analysis_panel;
......@@ -531,6 +531,7 @@ $(function () {
} else { // query for scan results
launch_request("/scan=" + url, url);
}
return false;
})
// click on another scan to go there
......
mdmaug/static/style.css
View file @ 43f6307b
......@@ -101,7 +101,9 @@ input[type=number] {
margin-left: 5px;
cursor: pointer;
}
#analysis-result-panel .addresses .related span {
#analysis-result-panel .addresses .related span,
#analysis-result-panel .addresses .related a {
color: gray;
padding-left: 2px;
}
#analysis-result-panel .urls {
......
mdmaug/static/style.css.map
View file @ 43f6307b
{"version":3,"sources":["style.less"],"names":[],"mappings":"AAAA;EACE,uBAAA;EACA,YAAA;EACA,WAAA;;AAGF;EACE,YAAA;;AAGF;EACE,iBAAA;;AAGF;EACE,WAAA;;AAGF;EACE,cAAA;;AAGF;EACE,UAAA;;AAGF,KAAK;EACH,WAAA;;AAIF;EACE,aAAA;EACA,WAAA;EACA,mBAAA;;AAGF;EACE,aAAA;;AAGF,aAAc;EACZ,YAAA;;AAGF;EACE,YAAA;;AAGF,eAAgB,aAAY,WAAY;EACtC,yBAAA;EACA,YAAA;;AAGF;EACE,yBAAA;EACA,eAAA;EACA,gBAAA;;AAHF,YAKE;EACE,iBAAA;;AAGF,YAAC;EACC,YAAA;;AADF,YAAC,UAEC;EACE,YAAA;;AAHJ,YAAC,UAKC;EACE,aAAA;;AAfN,YAkBE;EACE,cAAA;EACA,eAAA;EACA,WAAA;EACA,kBAAA;;AAKJ,sBACE;EACE,eAAA;EACA,gBAAA;;AACA,sBAHF,MAGG,IAAI;EACH,0BAAA;;AALN,sBAQE;EACE,WAAA;;;AAKJ,sBAIE,OACE;EACE,uBAAA;EACA,WAAA;EACA,YAAA;EACA,eAAA;;AATN,sBAYE;EACE,YAAA;EACA,iBAAA;;AAdJ,sBAgBE,kBAAkB,WAAW;EAC3B,YAAA;;AAjBJ,sBAmBE;EACE,cAAA;;AApBJ,sBAmBE,WAEE;EACE,WAAA;EACA,gBAAA;;AAvBN,sBAmBE,WAME;EACE,iBAAA;;AA1BN,sBAmBE,WASE;EACE,WAAA;EACA,gBAAA;EACA,eAAA;;AA/BN,sBAmBE,WASE,SAIE;EACE,iBAAA;;AAjCR,sBAsCE;EACE,gBAAA;;AAvCJ,sBAsCE,MAEE;EAEE,YAAA;EACA,SAAA;;AACA,sBANJ,MAEE,GAIG;EACC,WAAA;;AA7CR,sBAiDE;EACE,iBAAA;;AAlDJ,sBAoDE;EACE,kBAAA;;AAGE,sBAJJ,KAGG,aACE;EACC,kBAAA;EACA,YAAA;EACA,UAAA;EACA,UAAA;EACA,sBAAA;EACA,SAAS,EAAT;;AAGJ,sBAbF,KAaG,iBACC;AADF,sBAbF,KAaG,iBACQ;EACL,aAAA;;AAGJ,sBAlBF,KAkBG;EACC,sBAAA;;AADF,sBAlBF,KAkBG,iBAEC,MACA;EACE,YAAA;;AAIJ,sBA1BF,KA0BG;EACC,sBAAA;;AAMN;EACE,WAAA;;AAGF;EACE,qBAAA;EACA,WAAA;EACA,uBAAA;EACA,YAAA;;AAGF,MAAO;EACL,aAAA;;AAGF;EACE,uBAAA;EACA,YAAA;EACA,aAAA;EACA,YAAA;EACA,eAAA;EACA,YAAA;EACA,YAAA;EACA,YAAA;EACA,cAAA;EACA,gBAAA;EAEA,QAAA;EACA,eAAA;EACA,UAAA;;AACA,WAAC;EACC,YAAA;;AAhBJ,WAmBE;EACE,YAAA;;;;;;;AAUJ,EAAE;EACA,gBAAA;;AAGF;EACE,cAAA;EACA,gBAAA;EACA,oBAAA;EACA,SAAA;;AAGF,kBAAmB;EACjB,gBAAA;EACA,UAAA;EACA,SAAA;;AAGF,kBAAmB;EACjB,gBAAA;EACA,UAAA;EACA,SAAA;EACA,WAAA;;AAGF,IAAI,QAAQ;EACV,eAAA;;AAGF,QAAS;EACP,UAAA;;;;;AAMF,cAAe;EACb,eAAA;;AAGF,cAAe;AAAG,cAAe;EAC/B,WAAA;EACA,WAAA;EACA,eAAA;EACA,iBAAA;EACA,mBAAA;EACA,kBAAA;EACA,sBAAA;EACA,iBAAA;EACA,eAAA;EACA,cAAA;EACA,0CAAA;EACA,mBAAA;;EACA,YAAY,mDAAZ;;EACA,YAAY,gDAAgD,yBAAyB,0BAArF;;EACA,YAAY,sDAAZ;;EACA,YAAY,iDAAZ;;EACA,YAAY,kDAAZ;;EACA,YAAY,8CAAZ;;;AAGF,cAAe,EAAC;AAAQ,cAAe,GAAE,IAAI,WAAW,IAAI,SAAU,KAAI;EACxE,qBAAA;EACA,mBAAA;;EACA,YAAY,mDAAZ;;EACA,YAAY,gDAAgD,yBAAyB,0BAArF;;EACA,YAAY,sDAAZ;;EACA,YAAY,iDAAZ;;EACA,YAAY,kDAAZ;;EACA,YAAY,8CAAZ;;;AAGF,cAAe,GAAE,YAAa;AAAG,cAAe,GAAE,YAAa;EAC7D,2BAAA;EACA,0BAAA;;AAGF,cAAe,GAAE,WAAY;AAAG,cAAe,GAAE,WAAY;EAC3D,0BAAA;;AAGF,cAAe;EACb,mBAAA;;EACA,YAAY,mDAAZ;;EACA,YAAY,gDAAgD,yBAAyB,0BAArF;;EACA,YAAY,sDAAZ;;EACA,YAAY,iDAAZ;;EACA,YAAY,kDAAZ;;EACA,YAAY,8CAAZ;;EACA,eAAA;;AAGF,cAAe;EACb,mBAAA;EACA,eAAA;EACA,eAAA","file":"style.css"}
\ No newline at end of file
{"version":3,"sources":["style.less"],"names":[],"mappings":"AAAA;EACE,uBAAA;EACA,YAAA;EACA,WAAA;;AAGF;EACE,YAAA;;AAGF;EACE,iBAAA;;AAGF;EACE,WAAA;;AAGF;EACE,cAAA;;AAGF;EACE,UAAA;;AAGF,KAAK;EACH,WAAA;;AAIF;EACE,aAAA;EACA,WAAA;EACA,mBAAA;;AAGF;EACE,aAAA;;AAGF,aAAc;EACZ,YAAA;;AAGF;EACE,YAAA;;AAGF,eAAgB,aAAY,WAAY;EACtC,yBAAA;EACA,YAAA;;AAGF;EACE,yBAAA;EACA,eAAA;EACA,gBAAA;;AAHF,YAKE;EACE,iBAAA;;AAGF,YAAC;EACC,YAAA;;AADF,YAAC,UAEC;EACE,YAAA;;AAHJ,YAAC,UAKC;EACE,aAAA;;AAfN,YAkBE;EACE,cAAA;EACA,eAAA;EACA,WAAA;EACA,kBAAA;;AAKJ,sBACE;EACE,eAAA;EACA,gBAAA;;AACA,sBAHF,MAGG,IAAI;EACH,0BAAA;;AALN,sBAQE;EACE,WAAA;;;AAKJ,sBAIE,OACE;EACE,uBAAA;EACA,WAAA;EACA,YAAA;EACA,eAAA;;AATN,sBAYE;EACE,YAAA;EACA,iBAAA;;AAdJ,sBAgBE,kBAAkB,WAAW;EAC3B,YAAA;;AAjBJ,sBAmBE;EACE,cAAA;;AApBJ,sBAmBE,WAEE;EACE,WAAA;EACA,gBAAA;;AAvBN,sBAmBE,WAME;EACE,iBAAA;;AA1BN,sBAmBE,WASE;EACE,WAAA;EACA,gBAAA;EACA,eAAA;;AA/BN,sBAmBE,WASE,SAIE;AAhCN,sBAmBE,WASE,SAIQ;EACJ,WAAA;EACA,iBAAA;;AAlCR,sBAuCE;EACE,gBAAA;;AAxCJ,sBAuCE,MAEE;EAEE,YAAA;EACA,SAAA;;AACA,sBANJ,MAEE,GAIG;EACC,WAAA;;AA9CR,sBAkDE;EACE,iBAAA;;AAnDJ,sBAqDE;EACE,kBAAA;;AAGE,sBAJJ,KAGG,aACE;EACC,kBAAA;EACA,YAAA;EACA,UAAA;EACA,UAAA;EACA,sBAAA;EACA,SAAS,EAAT;;AAGJ,sBAbF,KAaG,iBACC;AADF,sBAbF,KAaG,iBACQ;EACL,aAAA;;AAGJ,sBAlBF,KAkBG;EACC,sBAAA;;AADF,sBAlBF,KAkBG,iBAEC,MACA;EACE,YAAA;;AAIJ,sBA1BF,KA0BG;EACC,sBAAA;;AAMN;EACE,WAAA;;AAGF;EACE,qBAAA;EACA,WAAA;EACA,uBAAA;EACA,YAAA;;AAGF,MAAO;EACL,aAAA;;AAGF;EACE,uBAAA;EACA,YAAA;EACA,aAAA;EACA,YAAA;EACA,eAAA;EACA,YAAA;EACA,YAAA;EACA,YAAA;EACA,cAAA;EACA,gBAAA;EAEA,QAAA;EACA,eAAA;EACA,UAAA;;AACA,WAAC;EACC,YAAA;;AAhBJ,WAmBE;EACE,YAAA;;;;;;;AAUJ,EAAE;EACA,gBAAA;;AAGF;EACE,cAAA;EACA,gBAAA;EACA,oBAAA;EACA,SAAA;;AAGF,kBAAmB;EACjB,gBAAA;EACA,UAAA;EACA,SAAA;;AAGF,kBAAmB;EACjB,gBAAA;EACA,UAAA;EACA,SAAA;EACA,WAAA;;AAGF,IAAI,QAAQ;EACV,eAAA;;AAGF,QAAS;EACP,UAAA;;;;;AAMF,cAAe;EACb,eAAA;;AAGF,cAAe;AAAG,cAAe;EAC/B,WAAA;EACA,WAAA;EACA,eAAA;EACA,iBAAA;EACA,mBAAA;EACA,kBAAA;EACA,sBAAA;EACA,iBAAA;EACA,eAAA;EACA,cAAA;EACA,0CAAA;EACA,mBAAA;;EACA,YAAY,mDAAZ;;EACA,YAAY,gDAAgD,yBAAyB,0BAArF;;EACA,YAAY,sDAAZ;;EACA,YAAY,iDAAZ;;EACA,YAAY,kDAAZ;;EACA,YAAY,8CAAZ;;;AAGF,cAAe,EAAC;AAAQ,cAAe,GAAE,IAAI,WAAW,IAAI,SAAU,KAAI;EACxE,qBAAA;EACA,mBAAA;;EACA,YAAY,mDAAZ;;EACA,YAAY,gDAAgD,yBAAyB,0BAArF;;EACA,YAAY,sDAAZ;;EACA,YAAY,iDAAZ;;EACA,YAAY,kDAAZ;;EACA,YAAY,8CAAZ;;;AAGF,cAAe,GAAE,YAAa;AAAG,cAAe,GAAE,YAAa;EAC7D,2BAAA;EACA,0BAAA;;AAGF,cAAe,GAAE,WAAY;AAAG,cAAe,GAAE,WAAY;EAC3D,0BAAA;;AAGF,cAAe;EACb,mBAAA;;EACA,YAAY,mDAAZ;;EACA,YAAY,gDAAgD,yBAAyB,0BAArF;;EACA,YAAY,sDAAZ;;EACA,YAAY,iDAAZ;;EACA,YAAY,kDAAZ;;EACA,YAAY,8CAAZ;;EACA,eAAA;;AAGF,cAAe;EACb,mBAAA;EACA,eAAA;EACA,eAAA","file":"style.css"}
\ No newline at end of file
mdmaug/static/style.less
View file @ 43f6307b
......@@ -125,7 +125,8 @@ input[type=number] {
color: gray;
margin-left: 5px;
cursor: pointer;
span {
span, a {
color: gray;
padding-left: 2px;
}
......
mdmaug/templates/crawl_view.py
View file @ 43f6307b
......@@ -3,7 +3,7 @@ from collections import defaultdict
from flask import render_template
from ..lib.controller.scan_controller import ScanController
from ..lib.domains import is_suspicious, url2domain
from ..lib.domains import is_suspicious, url2domain, domain2dir
from ..lib.model.dbp import Encounter
from ..lib.model.dbp import Whitelist
......@@ -23,7 +23,7 @@ class CrawlView:
"sfb": is_suspicious(crawl.host, 'attr'),
"domains": defaultdict(dict),
"title": crawl.title or crawl.host,
"scans": ScanController().get_domain_scans(url2domain(crawl.host)) if crawl.host else None,
"scans": ScanController().get_domain_scans(domain2dir(url2domain(crawl.host))) if crawl.host else None,
}
if hasattr(crawl, "paging"):
output["paging"] = ",".join(map(str,crawl.paging))
......
mdmaug/templates/output_html.html
View file @ 43f6307b
......@@ -6,7 +6,7 @@
{%- endif %}
{%- endmacro %}
{% macro shortenable(l, class, max) -%}
{% macro shortenable(l, class, max, href_prepend="") -%}
{# if list is longer than 15, print out first 10 and tell that there is another X hidden #}
{% if l|length > max -%}
{% set i = max*2/3|int %}
......@@ -15,7 +15,11 @@
{% endif %}
<span data-length='{{ l|length }}' class={{ class }}>
{% for el in l -%}
<span>{{ el }}</span>
{% if href_prepend -%}
<a href="{{ href_prepend }}{{ el }}">{{ el }}</a>
{% else %}
<span>{{ el }}</span>
{%- endif %}
{%- endfor %}
{% if text -%}
... ({{ text|int }})
......@@ -27,9 +31,9 @@
<div class='col-sm-8 analysis' data-host='{{ output["host"] }}' data-sfb='{{ output["sfb"] }}'>
<form>
<h2>{{ output["title"] }}</h2>
{% if output["paging"] %}
<div class="pagination" data-pages="{{ output["paging"] }}"></div>
{% endif %}
{% if output["paging"] %}
<div class="pagination" data-pages="{{ output["paging"] }}"></div>
{% endif %}
{{ safebrowsing(output["host"], output["sfb"]) }}
{% if output["scans"] -%}
......@@ -66,7 +70,7 @@
{% endif %}
</span>
{% endfor %}
{{ shortenable(domain["related"], "related", 15) }}
{{ shortenable(domain["related"], "related", 15, "api=html/scan=") }}
{# # XX PDNS link nelze dát, protože je tolik linku, kolik IP ma navstiveny host. Lze udelat, ze odkaz povede na vypis vsech domen. XDomains.get_pdns_link(crawl[domain]) #}
{{ shortenable(domain["pdns"], "pdns", 15) }}
</div>
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment