Commit 3f0a81b4 authored by Edvard Rejthar's avatar Edvard Rejthar

UI scans and cache working

parent f3914864
import json
import logging
import subprocess
from urllib.parse import urlparse
from peewee import IntegrityError
from ...templates.crawl_view import CrawlView
from ..domains import Domains
from .scan_controller import ScanController
from ..config import Config
from ..model.dbp import Turris, Whitelist
from ..parser.traffic_log_parser import TrafficLogParser
from ...templates.crawl_view import CrawlView
class Api:
......@@ -20,20 +22,31 @@ class Api:
:type path: dict from URL request. /api/analyze=cache/http://example.com → {"api": True, "analyze": cache, "page": "http://example.com"}
"""
crawl = None
if "analyze" in request:
crawl = ScanController().launch(request["page"], {"cached": 1, "weekcache":7, "oldcache": True, True: None}[request["analyze"]])
if request["api"] == "json":
return CrawlView.output_json(crawl)
map_ = {"fresh": None, "cached": 1, "weekcache": 7, "oldcache": True, True: None}
if request["analyze"] in map_:
days = map_[request["analyze"]]
else:
return CrawlView.output_html(crawl)
days = int(request["analyze"])
crawl = ScanController().launch(request["page"], days)
elif "decide" in request: # XX deprecated?
return self.get_undecided()
elif "nicify" in request:
return TrafficLogParser.getStylesheet() + TrafficLogParser.nicifyFile(request["page"])
elif "vote" in request: # /api/vote/block/example.org/10.0.0.1
elif "vote" in request: # /api/vote/block/example.org/10.0.0.1
logging.debug("vote cmd")
return Turris.vote(request["vote"], request["page"])
elif "whitelist" in request: # XXX not implemented yet
elif "scans" in request:
if "url" in request: # /api/scans/url/http://example.com
domain = Domains.domain2dir(request["page"])
if not domain:
return "Wrong domain"
return ScanController().get_domain_snapdirs(domain, full_dirs=False)
else:
return "Not implemented"
elif "whitelist" in request: # XXX not implemented yet
"""url = path.split("/", 3)
if len(url) > 3:
self._setWebsite(url[2]) # osetrit self.website, ze je URL, a nikoli shell
......@@ -47,6 +60,14 @@ class Api:
elif "reset" in request:
self.reset()
return "reset"
else:
return "Unknown method."
if crawl:
if request["api"] == "json":
return CrawlView.output_json(crawl)
else:
return CrawlView.output_html(crawl)
@staticmethod
def reset():
......
......@@ -28,30 +28,31 @@ class ScanController:
# cacheDir = None
# logDir = None
def get_domain_snapdirs(self, domain, full_dirs=True):
dir = Config.CACHE_DIR + domain + "/"
if os.path.isdir(dir):
return [str(dir + subdir) if full_dirs else str(subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu
if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir + subdir + "/" + ScanController.CRAWL_FILE)]
##
# @param cached Pokud chceme zobrazit cachovanou verzi analyzy, dejme True. Pokud dame int, je to maximalni stari (ve dnech). Kdyz se nenalezne, zanazyzuje se znovu.
def launch(self, url, cached=None):
if cached:
# """ Pokud je k dispozici analyza, vratit ji """
dir = Config.CACHE_DIR + Domains.domain2dir(url) + "/"
if os.path.isdir(dir):
snapdirs = [str(dir + subdir) for subdir in os.listdir(dir) # adresare vsech moznych snapshotu
if os.path.isdir(str(dir + subdir)) and os.path.isfile(dir + subdir + "/" + ScanController.CRAWL_FILE)]
if snapdirs:
cache_dir = max(snapdirs, key=os.path.getmtime) + "/" # nejnovejsi dir analyzy
if type(cached) != int or os.path.getmtime(
cache_dir) > time.time() - 3600 * 24 * cached: # maximalni stari analyzy
try:
logging.debug(f"Returning a previous crawl from: {cache_dir + ScanController.CRAWL_FILE}")
crawl = Crawl.load_from_file(cache_dir + ScanController.CRAWL_FILE)
return crawl
except ValueError:
pass
logging.debug("({-1}) Cachovana analyza nenalezena")
# provest novou analyzu
if self.queue(): # /api/analyze/web - zaradi web do fronty
snapdirs = self.get_domain_snapdirs(Domains.domain2dir(url))
if snapdirs:
# get the most recent snapdir and check if it's not too old
cache_dir = max(snapdirs, key=os.path.getmtime) + "/"
if cached is True or os.path.getmtime(cache_dir) > time.time() - (3600 * 24 * cached):
try:
logging.debug(f"Returning a previous crawl from: {cache_dir + ScanController.CRAWL_FILE}")
crawl = Crawl.load_from_file(cache_dir + ScanController.CRAWL_FILE)
return crawl
except ValueError:
pass
logging.debug("({-1}) Convenient cached analysis not found")
# perform fresh analysis
if self.queue(): # /api/analyze/web - queue current analysis
print("({}) start crawl".format(self.profile))
self.url = Domains.assureUrl(url)
try:
......
import json
import logging
import mimetypes
import os
......@@ -42,7 +43,7 @@ class Server(SimpleHTTPRequestHandler):
bytearray([7, 8, 9, 10, 12, 13, 27]) + bytearray(
range(0x20, 0x100)))) else 'r'
with open(url, type_) as f:
self.output(f.read(), content_type=mimetypes.guess_type(url))
self.output(f.read(), content_type=mimetypes.guess_type(url)[0])
def do_GET(self):
"""
......@@ -90,9 +91,16 @@ class Server(SimpleHTTPRequestHandler):
if "api" in request: # /api/analyze/web
output = Api().run(request)
if request["api"] == "json":
if type(output) is dict:
output = json.dumps(output)
elif type(output) in [str, list]:
output = json.dumps(output)
if "destination" in request:
# send everything up, we are in an iframe
self.render_template("_message.html", contents=output, cmd=request, url=self.path,
self.render_template("_message.html", contents=output, url=self.path,
destination=f"https://{request['destination']}/")
else:
self.output(output)
......
......@@ -35,7 +35,12 @@ class Domains:
'(((((http|https|ftp)://)?[\w\-_]+(?:(?:\.[\w\-_]+)+)))([\w\-\.,@?^=%&:/~\+#]*[\w\-\@?^=%&/~\+#])?)',
url).group(0)
def domain2dir(url): # friendly nazev adresare z domeny v url
def domain2dir(url) -> str:
"""
Friendly dir name from a domain contained in an url
XXmay be replaced by urlparse(url).netloc?
"""
url = url.lower()
url = re.sub('^(http://|https://|ftp://)', '', url) # odstrihnout protokol
url = re.sub('(/.*)', '', url) # nestojim o cestu, jen o domene
......
auxiliary.org-netbeans-modules-css-prep.less_2e_compiler_2e_options=
auxiliary.org-netbeans-modules-css-prep.less_2e_configured=true
auxiliary.org-netbeans-modules-css-prep.less_2e_enabled=true
auxiliary.org-netbeans-modules-css-prep.less_2e_mappings=/static:/static
auxiliary.org-netbeans-modules-css-prep.sass_2e_compiler_2e_options=
auxiliary.org-netbeans-modules-css-prep.sass_2e_enabled=false
auxiliary.org-netbeans-modules-css-prep.sass_2e_mappings=/scss:/css
file.reference.mdmaug-installer-mdmaug=.
files.encoding=UTF-8
site.root.folder=${file.reference.mdmaug-installer-mdmaug}
source.folder=${file.reference.mdmaug-installer-mdmaug}
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://www.netbeans.org/ns/project/1">
<type>org.netbeans.modules.web.clientproject</type>
<configuration>
<data xmlns="http://www.netbeans.org/ns/clientside-project/1">
<name>mdmaug</name>
</data>
</configuration>
</project>
/**
* Cache radio buttons responsivity
*/
$("input[name=cache]").change(function () {
let $days_input = $("input[name=days]");
if ($(this).val() === "days") {
$days_input = $("input[name=days]");
$days_input.show().siblings("span").hide();
setTimeout(() => {
$days_input.select();
}, 100);
} else {
$days_input.hide().siblings("span").show();
}
});
/**
* Initialization
*/
$("input[type=url]").val(new URL(window.location.href).searchParams.get("url")); // fetch the default URL from the address bar
/**
* list scans button
$("#list-scans").click(() => {
let urls = get_urls();
if (urls) {
let url = urls[0];
launch_request("/scans/url/{0}".format(url), "json");
}
return false;
});*/
/**
* Send request, analyze button
*/
var $requests = $("#requests-panel");
$("form").submit(() => {
let cache = $(".active > input[name=cache]").val();
if (cache === "days") {
cache = $(".active > input[type=number]").val();
}
let method = "html";
for (let url of get_urls()) {
let rest_url = "/analyze={0}/{1}".format(cache, url);
launch_request(rest_url, method, url);
}
//.map(); // let's create a closure to preserve counter etc
return false;
});
/**
* Toggle single and multiple URLs
*/
$("#request-url-toggler").change(function () {
if ($(this).prop("checked")) {
$("#request-url").hide();
let $el = $("#request-urls").show();
if (!$el.val().trim()) {
$el.val($("#request-url").val());
}
} else {
let $el = $("#request-url").show();
$("#request-urls").hide();
if (!$el.val().trim()) {
$el.val($("#request-urls").val().split("\n")[0]);
}
}
});
/**
* Launches a new request to the api
* @param {type} header Ex: Scanned domain
* @param {string} method html or json
* @param {type} rest_url Part of the URI of the API.
* @returns {undefined}
*/
function launch_request(rest_url, method = "json", header = "Result", callback=null) {
let counter = 0;
//let uri = "/api={0}/analyze={1}/{2}".format(method, cache, url);
let uri = "/api={0}{1}".format(method, rest_url);
let $el = $(".template >", $requests).clone();
$(".uri", $el).text(uri);
$("> .placeholder", $requests).remove();
$("> b", $requests).after($el);
// elapsed time counter
let increase_elapsed = setInterval(() => {
$(".timer", $el).html("{0} s".format(++counter));
if (counter > 10) {
bootstrap_change_bg($el, "bg-warning");
$el.addClass();
}
}, 1000);
// start the ajax
$.ajax({
dataType: method,
url: uri
}).done((data) => {
bootstrap_change_bg($el, "bg-success").find(".timer").prop("disabled", true).prepend("✓ ");
let s;
if (callback) {
s = callback(data);
} else {
if (method === "json") {
s = "<pre>{0}</pre>".format(syntaxHighlight(JSON.stringify(data, undefined, 4)));
} else {
s = data;
}
$("<div/>", {"class": "analysis", "html": "<h3>{0}</h3>".format(header) + s}).appendTo($("#analysis-result-panel"));
}
}).fail(() => {
bootstrap_change_bg($el, "bg-danger").find(".timer").prepend("× ");
}).always(() => {
clearInterval(increase_elapsed);
});
}
/**
* Prints out a JSON in a nice manner.
* @param {JSON} json
* @returns {HTML}
*/
function syntaxHighlight(json) {
json = json.replace(/&/g, '&amp;').replace(/</g, '&lt;').replace(/>/g, '&gt;');
return json.replace(/("(\\u[a-zA-Z0-9]{4}|\\[^u]|[^\\"])*"(\s*:)?|\b(true|false|null)\b|-?\d+(?:\.\d*)?(?:[eE][+\-]?\d+)?)/g, function (match) {
var cls = 'number';
if (/^"/.test(match)) {
if (/:$/.test(match)) {
cls = 'key';
} else {
cls = 'string';
}
} else if (/true|false/.test(match)) {
cls = 'boolean';
} else if (/null/.test(match)) {
cls = 'null';
}
return '<span class="' + cls + '">' + match + '</span>';
});
}
// String formatting function usage "string {0}".format("1") => "string 1"
if (!String.prototype.format) {
String.prototype.format = function () {
var args = arguments;
return this.replace(/{(\d+)}/g, function (match, number) {
return typeof args[number] !== 'undefined'
? args[number]
: match
;
});
};
}
/**
* Removes all the bg- classes.
* @param {type} $el
* @param {type} new_class Optionally append new class.
* @returns {$el} for chaining
*/
function bootstrap_change_bg($el, new_class = false) {
$el.removeClass(function (index, className) {
return (className.match(/\bbg-\S+/g) || []).join(' ');
});
if (new_class) {
$el.addClass(new_class);
}
return $el;
}
/**
* get list of urls that we want to analyze
*/
function get_urls() {
let urls = $("#request-url-toggler").prop("checked") ? $("#request-urls").val() : $("#request-url").val();
return urls.split("\n");
}
\ No newline at end of file
......@@ -108,7 +108,8 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
* @returns {undefined}
*/
safebrowsingIcon = function (webEl, url) {
a = $(webEl).attr("data-safebrowsing-suspicious");
let a = $(webEl).attr("data-safebrowsing-suspicious");
let str;
if (a === "1") {// je suspicious
str = "!";
} else if (a === "0") {// neni suspicious
......@@ -123,10 +124,6 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
window.addEventListener("message", function (e) {
//prevzit vysledky | e.data ve formatu: <div class=web><div class=domain>jedinecna domena</div></div>
console.log("event", e);
if (e.data["analyze=cached"]) {
e.data["analyze"] = e.data["analyze=cached"];
}
if (e.data.analyze) {//jedna se o zpravu pro me
// dat vedet uzivateli
$("#analysis-info").text("✓");
......@@ -135,12 +132,12 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
//ulozit do sessionStorage (pri prostem refreshi nedojde k narocne reanalyze)
if (e.data.url !== "refresh") {//neukladame do storage, protoze tohle je load ze storage, nikoli fresh analyza
analysisHost = $(".analysis", $analysisTemp).attr("data-host");
let analysisHost = $(".analysis", $analysisTemp).attr("data-host");
console.log("HOST", analysisHost);
if (typeof analysisHost == "undefined") {//mdmaug skoncil chybou, zkusime host dosta z URL iframu
// "/api/analyze=cached/http://www.klein-automotive.cz/aktuality/page/2/" -> http://...
if (typeof analysisHost === "undefined") {//mdmaug skoncil chybou, zkusime host dostat z URL iframu
// "/destination=mdm.nic.cz/api/analyze=cached/http://www.klein-automotive.cz/aktuality/page/2/" -> http://...
var path = e.data.url.split("/");
for (var i = 0; i < 3; i++) {
for (var i = 0; i < 4; i++) {
path.shift();
}
analysisHost = path.join("/");
......@@ -150,17 +147,18 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
}
$(".analysis", $analysisTemp).each(function () {// probehla jedina analyza, ale radsi to prozenu eachem
// add the safebrowsing icon to the main page header
$("body h1:eq(0)").append(safebrowsingIcon(this, $(this).attr("data-host")));
//help text
// help text
$("#advice").remove();
$advice = $("<p/>", {id: "advice", "text": "Když je to ! nebo wp a stránka není sundaná a ještě nemáme korespondenci, dát poslat."}).insertAfter($("body h1:eq(0)"));
$(".web", this).each(function () {//primerguje weby nalezene v nove analyze do stare analyzy.
host = $.trim($(".domain", this).text());
let host = $.trim($(".domain", this).text());
safebrowsingIcon(this, host);
contained = false;
let contained = false;
$(".web > .domain", $analysis).each(function () {
if ($(this).text() == host) {
console.log($(this).text());
......@@ -177,7 +175,7 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
} else {
//voting, zda je web malicious
voting = $("<span class='voting'/>").insertBefore($(".domain", this));
let voting = $("<span class='voting'/>").insertBefore($(".domain", this));
$.each(["block", "log", "n/a", "allow"], function (i, val) {//vlozit 4 hlasovaci tlacitka
$("<input type=radio name='" + host + "' value='" + val + "'/>").appendTo(voting).change(function () {
console.log("VOTING CHANGE");
......@@ -186,7 +184,9 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
});
//localStorage -> default voting value
if ((vote = $(this).attr("data-vote")) == "") {//vote pro web je v jeho atributu, protoze k domene webu nepatri zadne ip
if ((vote = $(".address:eq(0)", this).attr("data-vote")) == "") {//vote pro web se bere z votu pro jeho prvni ip XXTakze kdyz se budou voty pro IP z nejakeho duvodu lisit, admin na to neprijde a vote premaze.
if ((vote = $(".address:eq(0)", this).attr("data-vote")) == "") {
//vote pro web se bere z votu pro jeho prvni ip
//XXTakze kdyz se budou voty pro IP z nejakeho duvodu lisit, admin na to neprijde a vote premaze.
vote = "n/a"; //localStorage.getItem("domain_" + host)
}
}
......@@ -198,7 +198,6 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
} else {
$(analysis).prepend(this);
}*/
}
});
......@@ -210,7 +209,7 @@ if (document.location.href.indexOf("https://mdm.nic.cz/detail/") === 0) {
//refreshOutput();
//rovnou zaciname hlasovat, analyza kompletni
$(".voting:eq(0) input:eq(0)",sections.$undecided).focus();//focusujeme pouze z bloku undecided; nema smysl focusovat allowed web, jeste nekomu ujede ruka
$(".voting:eq(0) input:eq(0)", sections.$undecided).focus();//focusujeme pouze z bloku undecided; nema smysl focusovat allowed web, jeste nekomu ujede ruka
} else if (e.data.voted) {//ve vedlejsim tabu se hlasovalo, zmen hodnotu i zde
changeVote($("[type=radio][value='" + e.data.voted + "']", $(".web .domain:contains('" + e.data.domain + "'):eq(0)").siblings(".voting")).prop("checked", true), false);//hlasovat
}
......
#page0 {
margin:20px;
}
#page1 {
background:#888;
margin:0;
padding:1%;
}
#container {
overflow:hidden;
}
#menu {
background-color:#eed;
float:left;
margin:0 0 -100000px;
padding:1% 1% 100000px;
width:20%;
}
#menu ul {
border:none;
list-style:none;
margin:0 auto;
padding:0;
width:100%;
}
#menu ul li {
background-color:#ccb;
margin:2px 0 0;
padding:5px;
text-align:left;
}
#menu ul li.indent {
margin-left:25px;
}
#menu ul li a {
display:block;
font-weight:700;
width:auto;
}
#menu ul li:hover {
background-color:#edb;
}
#menu > div {
background-color:#ccb;
display:block;
padding:5px;
}
#menu img {
border:0;
}
#content {
background-color:#9bc;
float:left;
margin:0 0 0 1%;
min-height:650px;
overflow:auto;
padding:1%;
width:75%;
}
#content h1 {
margin-top:0;
}
.roundedcorners {
-khtml-border-radius:5px;
-moz-border-radius:5px;
-webkit-border-radius:5px;
border-radius:5px;
}
.fail {
background-color:#fdd;
color:#800;
display:block;
text-align:center;
}
.info {
background-color:#dfd;
color:#800;
display:block;
text-align:center;
}
.loading {
text-decoration:blink;
}
label.upper {
display:block;
margin-top:.33em;
}
#ticket {
margin-top:5px;
padding:5px;
}
#ticket>div {
background-color:#def;
border:2px solid #888;
margin-top:5px;
padding:10px;
}
#ticket>div.odd {
background-color:#cde;
}
#ticket>div>div.caption {
float:left;
}
#ticket>div>div.reply {
margin:0 0 0 auto;
position:relative;
text-align:left;
width:80px;
}
#ticket>div>div.reply img {
left:-25px;
position:absolute;
top:-4px;
}
#ticket>div>div.clear {
clear:both;
}
#ticket span {
float:left;
width:80px;
}
#ticket hr {
color:#888;
}
#ticket div.download {
background-color:#ade;
border:2px solid #888;
float:left;
margin-right:5px;
padding:10px;
width:200px;
}
body {
font-family:Verdana,Geneva,Arial,Helvetica,sans-serif;
font-size:0.6875em;
}
#content table.zebra {
border-collapse:collapse;
border-color:#ade;
border-style:solid;
border-width:0 2px 2px;
margin:10px 0;
width:100%;
}
#content table.zebra tbody tr:nth-child(even) {
background-color:#ade;
}
#content table.zebra th {
background-color:#ade;
font-weight:700;
}
#content table.zebra td,th {
padding:2px;
text-align:center;
}
#content table.zebra td img {
height:16px;
width:16px;
}
#content table.zebra td a:hover {
color:#a60;
}
/*
* Wrapping inside <pre> tags
* http://www.longren.org/wrapping-text-inside-pre-tags/
*/
pre {
font-size:1.25em;
white-space:pre-wrap; /* css-3 */
white-space:-moz-pre-wrap !important; /* Mozilla, since 1999 */
white-space:-pre-wrap; /* Opera 4-6 */
white-space:-o-pre-wrap; /* Opera 7 */
word-wrap:break-word; /* Internet Explorer 5.5+ */
width:99%;
}
#lastupdated-filter .form-fields {
margin:0 0 20px;
padding:0;
}
#lastupdated-filter .form-fields > li {
float:left;
list-style-type:none;
margin:0 20px 0 0;
padding:0;
}
#lastupdated-filter .form-fields li label {
margin-right:7px;
}
#lastupdated-filter .form-fields input[type="text"] {
width:100px;
}
#lastupdated-filter .form-fields img {
display:none;
margin-left:5px;
margin-top:-2px;
vertical-align:middle;
}
.js #lastupdated-filter .form-fields img {
cursor:pointer;
display:inline;
}
#reply-filter>fieldset>label {
float:left;
width:15em;
}