Commit f0ec24e8 authored by Edvard Rejthar's avatar Edvard Rejthar

#25 dir structure, "rest" -> "api"

parent 3a4dbd60
...@@ -30,6 +30,7 @@ user_pref("browser.newtabpage.storageVersion", 1); ...@@ -30,6 +30,7 @@ user_pref("browser.newtabpage.storageVersion", 1);
user_pref("browser.pagethumbnails.storage_version", 3); user_pref("browser.pagethumbnails.storage_version", 3);
user_pref("browser.places.smartBookmarksVersion", 8); user_pref("browser.places.smartBookmarksVersion", 8);
user_pref("browser.preferences.advanced.selectedTabIndex", 2); user_pref("browser.preferences.advanced.selectedTabIndex", 2);
user_pref("browser.privatebrowsing.autostart", true);
user_pref("browser.safebrowsing.provider.google.lastupdatetime", "1486578575144"); user_pref("browser.safebrowsing.provider.google.lastupdatetime", "1486578575144");
user_pref("browser.safebrowsing.provider.google.nextupdatetime", "1486580358144"); user_pref("browser.safebrowsing.provider.google.nextupdatetime", "1486580358144");
user_pref("browser.safebrowsing.provider.mozilla.lastupdatetime", "1486578502288"); user_pref("browser.safebrowsing.provider.mozilla.lastupdatetime", "1486578502288");
...@@ -92,8 +93,11 @@ user_pref("pdfjs.previousHandler.alwaysAskBeforeHandling", true); ...@@ -92,8 +93,11 @@ user_pref("pdfjs.previousHandler.alwaysAskBeforeHandling", true);
user_pref("pdfjs.previousHandler.preferredAction", 4); user_pref("pdfjs.previousHandler.preferredAction", 4);
user_pref("places.history.expiration.transient_current_max_pages", 122334); user_pref("places.history.expiration.transient_current_max_pages", 122334);
user_pref("plugin.disable_full_page_plugin_for_types", "application/pdf"); user_pref("plugin.disable_full_page_plugin_for_types", "application/pdf");
user_pref("privacy.clearOnShutdown.offlineApps", true); user_pref("privacy.clearOnShutdown.*", true);
user_pref("privacy.clearOnShutdown.openWindows", true); user_pref("privacy.clearOnShutdown.openWindows", true);
user_pref("privacy.clearOnShutdown.cookies", true);
user_pref("privacy.clearOnShutdown.sessions", true);
user_pref("privacy.clearOnShutdown.siteSettings", true);
user_pref("privacy.clearOnShutdown.passwords", true); user_pref("privacy.clearOnShutdown.passwords", true);
user_pref("privacy.clearOnShutdown.siteSettings", true); user_pref("privacy.clearOnShutdown.siteSettings", true);
user_pref("privacy.sanitize.didShutdownSanitize", true); user_pref("privacy.sanitize.didShutdownSanitize", true);
......
...@@ -5,7 +5,6 @@ ...@@ -5,7 +5,6 @@
DESTINATION=/opt/mdmaug DESTINATION=/opt/mdmaug
PROFILE_COUNT=21 PROFILE_COUNT=21
# sometimes, there is no repository with pip on the machine, trying to add some sources # sometimes, there is no repository with pip on the machine, trying to add some sources
apt install software-properties-common apt install software-properties-common
add-apt-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc) main universe restricted multiverse" add-apt-repository "deb http://archive.ubuntu.com/ubuntu $(lsb_release -sc) main universe restricted multiverse"
......
__author__ = "edvard"
__date__ = "$Apr 27, 2015 5:40:11 AM$"
\ No newline at end of file
DEBUG:root:This message should go to the log file
INFO:root:So should this
WARNING:root:And this, too
WARNING:root:warning1
DEBUG:root:debug1
import threading import threading
import os import os
import logging
from glob import glob from glob import glob
from peewee import MySQLDatabase from peewee import MySQLDatabase
...@@ -10,7 +11,8 @@ class Config: ...@@ -10,7 +11,8 @@ class Config:
APP_PORT = 8000 APP_PORT = 8000
APP_DOMAIN = 'https://217.31.202.41:' + str(APP_PORT) #csirt.csirt.office.nic.cz APP_DOMAIN = 'https://217.31.202.41:' + str(APP_PORT) #csirt.csirt.office.nic.cz
LOG_DIR = "/opt/mdmaug/.cache/mdmaug-scans/_tmp/" # X /tmp/mdm/ LOG_DIR = "/opt/mdmaug/.cache/mdmaug-scans/_tmp/" # X /tmp/mdm/
CACHE_DIR = "/opt/mdmaug/.cache/mdmaug-scans/" CACHE_DIR = "/opt/mdmaug/.cache/mdmaug-scans/"
DIR = os.path.dirname(os.path.realpath(__file__)) + "/../"
myDB = "" myDB = ""
lock = threading.RLock() # doufam, ze kdyz je lock tady, ze je funknci. Closure...? XX nejak otestovat lock = threading.RLock() # doufam, ze kdyz je lock tady, ze je funknci. Closure...? XX nejak otestovat
THUMBNAIL_SIZE = 640, 640 THUMBNAIL_SIZE = 640, 640
...@@ -21,7 +23,7 @@ class Config: ...@@ -21,7 +23,7 @@ class Config:
def connect(): def connect():
# XX resim problem peewee.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))") po 7 hodinach timeoutu # XX resim problem peewee.OperationalError: (2006, "MySQL server has gone away (BrokenPipeError(32, 'Broken pipe'))") po 7 hodinach timeoutu
# XX kupodivu pripojeni nemuze byt v dbp DBModel.connect. Prestoze type je pak spravne (MySQLDatabase), nic udelat nejde a pokusy o select konci NoneType. # XX kupodivu pripojeni nemuze byt v dbp DBModel.connect. Prestoze type je pak spravne (MySQLDatabase), nic udelat nejde a pokusy o select konci NoneType.
print("Connecting to DB.") logging.debug("Connecting to DB.")
Config.myDB = MySQLDatabase("mdmaug", host='localhost', port=3306, user="mdmaug", passwd="fidFDSs676") # XX dal jsem pryc: , threadlocals=False Config.myDB = MySQLDatabase("mdmaug", host='localhost', port=3306, user="mdmaug", passwd="fidFDSs676") # XX dal jsem pryc: , threadlocals=False
Config.myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'}) Config.myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
......
#import os
#from pprint import pprint
#import queue
#import re
#import threading
#import dill
#import pickle
#import pdb
import json import json
import subprocess import subprocess
import logging import logging
logging.basicConfig(level=logging.WARNING, format="%(message)s")
from lib.config import Config from lib.config import Config
from lib.analysis.scan_controller import ScanController from lib.controller.scan_controller import ScanController
from lib.dbp import Status, Export, Turris, Whitelist from lib.model.dbp import Status, Export, Turris, Whitelist
from lib.analysis.parser.traffic_log_parser import TrafficLogParser from lib.analysis.parser.traffic_log_parser import TrafficLogParser
class Rest: class Api:
allow_reuse_address = True # XX tohle nevim, co dela
website = "" # http://site.cz website = "" # http://site.cz
websiteDomain = "" # site.cz websiteDomain = "" # site.cz
#spojit s mysql
def __init__(self, path): def __init__(self, path):
self.path = path self.path = path
##
# Spusti prikaz
def run(self, cmd): def run(self, cmd):
if cmd == "analyze": #Xqueue se do db pripojuje sama (je mozne, ze vsechny FF jsou obsazene, pak se db zbytecne vytezuje) """ Accept command """
if cmd == "analyze":
return ScanController().launch(self.path) return ScanController().launch(self.path)
if cmd == "analyze=cached": if cmd == "analyze=cached":
return ScanController().launch(self.path, cached = 1) return ScanController().launch(self.path, cached = 1)
...@@ -37,45 +23,39 @@ class Rest: ...@@ -37,45 +23,39 @@ class Rest:
return ScanController().launch(self.path, cached = 7) return ScanController().launch(self.path, cached = 7)
if cmd == "analyze=oldcache": if cmd == "analyze=oldcache":
return ScanController().launch(self.path, cached = True) return ScanController().launch(self.path, cached = True)
elif cmd == "export=view": # XX asi deprecated elif cmd == "export=view": # XX deprecated?
return Export.exportView() return Export.exportView()
elif cmd == "export=confirm": # XX asi deprecated elif cmd == "export=confirm": # XX deprecated?
return Export.exportConfirm() return Export.exportConfirm()
elif cmd == "decide": # XX asi deprecated elif cmd == "decide": # XX deprecated?
return self.getUndecided() return self.getUndecided()
elif cmd == "nicify": elif cmd == "nicify":
url = self.path.split("/", 3) url = self.path.split("/", 3)
return TrafficLogParser.getStylesheet() + TrafficLogParser.nicifyFile(url[3]) return TrafficLogParser.getStylesheet() + TrafficLogParser.nicifyFile(url[3])
elif cmd == "vote": # /rest/vote/block/example.org/10.0.0.1 elif cmd == "vote": # /api/vote/block/example.org/10.0.0.1
logging.debug("vote cmd") logging.debug("vote cmd")
url = self.path.split("/", 4) url = self.path.split("/", 4)
logging.debug(url[3]) logging.debug(url[3])
return Turris.vote(url[3], url[4]) return Turris.vote(url[3], url[4])
elif cmd == "whitelist": elif cmd == "whitelist": # XXX not implemented yet
url = self.path.split("/", 3) url = self.path.split("/", 3)
if len(url) > 3: if len(url) > 3:
self._setWebsite(url[2]) #osetrit self.website, ze je URL, a nikoli shell self._setWebsite(url[2]) # osetrit self.website, ze je URL, a nikoli shell
print("XXX nejsem si jist, zda url je spravne na url[2]") # XXX logging.debug("XXX nejsem si jist, zda url je spravne na url[2]") # XXX
print(url) # XXX logging.debug(url) # XXX
quit() # XXX quit() # XXX
logging.debug(self.website) logging.debug(self.website)
logging.debug(self.websiteDomain) logging.debug(self.websiteDomain)
return self.whitelist() return self.whitelist()
elif cmd == "reset": elif cmd == "reset":
Server.reset() Server.reset()
return "reset" return "reset"
def reset(): def reset():
logging.debug("resetting running browsers") logging.debug("resetting running browsers")
#smazat frontu with open(Config.configFile, 'w') as f: # clear the queue
with open(Config.configFile, 'w') as f: json.dump({}, f)
json.dump({}, f) subprocess.call(["pkill", Config.browser]) # kill frozen browsers
#zabit zamrzle prohlizece
subprocess.call(["pkill", Config.browser])
...@@ -84,7 +64,7 @@ class Rest: ...@@ -84,7 +64,7 @@ class Rest:
def whitelist(self): def whitelist(self):
logging.debug("whitelistuju") logging.debug("whitelistuju")
#Db.cur = Db.connection.cursor() #Db.cur = Db.connection.cursor()
#self._print(Db.cur.execute("""REPLACE INTO whitelist set domain = %s""", (self.websiteDomain, ))) #self._logging.debug(Db.cur.execute("""REPLACE INTO whitelist set domain = %s""", (self.websiteDomain, )))
#Db.connection.commit() #Db.connection.commit()
#Db.cur.close() #Db.cur.close()
try:Whitelist.insert(domain=self.websiteDomain).execute() try:Whitelist.insert(domain=self.websiteDomain).execute()
...@@ -94,7 +74,4 @@ class Rest: ...@@ -94,7 +74,4 @@ class Rest:
def getUndecided(self): def getUndecided(self):
logging.debug("XXX jeste jsem neudelal - ma vylezt tabulka vsech nerozhodlych domen od posledniho exportu") logging.debug("XXX jeste jsem neudelal - ma vylezt tabulka vsech nerozhodlych domen od posledniho exportu")
pass pass
\ No newline at end of file
# Sprava dat scannovani
from pprint import pprint
import json import json
import datetime import datetime
import time import time
import os import os
from glob import glob
from random import randint
import subprocess import subprocess
import traceback import traceback
import logging import logging
logging.basicConfig(level=logging.WARNING, format="%(message)s") from glob import glob
from random import randint
from lib.config import Config from lib.config import Config
from lib.domains import Domains from lib.domains import Domains
from lib.analysis.crawl import Crawl from lib.model.crawl import Crawl
from templates.crawl_view import CrawlView from templates.crawl_view import CrawlView
from lib.analysis.parser.traffic_log_parser import TrafficLogParser from lib.parser.traffic_log_parser import TrafficLogParser
from lib.analysis.parser.nspr_log_parser import NsprLogParser from lib.parser.nspr_log_parser import NsprLogParser
from lib.analysis.parser.metadata_parser import MetadataParser from lib.parser.metadata_parser import MetadataParser
from lib.analysis.parser.screenshot_parser import ScreenshotParser from lib.parser.screenshot_parser import ScreenshotParser
from lib.analysis.parser.spy_parser import SpyParser from lib.parser.spy_parser import SpyParser
import pdb
class ScanController: class ScanController:
...@@ -53,25 +47,25 @@ class ScanController: ...@@ -53,25 +47,25 @@ class ScanController:
cacheDir = max(snapdirs, key = os.path.getmtime)+ "/" # nejnovejsi dir analyzy cacheDir = max(snapdirs, key = os.path.getmtime)+ "/" # nejnovejsi dir analyzy
if type(cached) != int or os.path.getmtime(cacheDir) > time.time()-3600*24*cached: # maximalni stari analyzy if type(cached) != int or os.path.getmtime(cacheDir) > time.time()-3600*24*cached: # maximalni stari analyzy
try: try:
print("returning") logging.debug("returning")
return CrawlView.outputHtml(Crawl.loadFromFile(cacheDir + ScanController.CRAWL_FILE)) #"crawlSave.tmp" return CrawlView.outputHtml(Crawl.loadFromFile(cacheDir + ScanController.CRAWL_FILE)) #"crawlSave.tmp"
except ValueError: except ValueError:
pass pass
print("({-1}) Cachovana analyza nenalezena") logging.debug("({-1}) Cachovana analyza nenalezena")
# provest novou analyzu # provest novou analyzu
if self.queue(): # /rest/analyze/web - zaradi web do fronty if self.queue(): # /api/analyze/web - zaradi web do fronty
print ("({}) start crawl".format(self.profile)) print ("({}) start crawl".format(self.profile))
self.url = Domains.assureUrl(url) self.url = Domains.assureUrl(url)
try: try:
crawl = self.analyze() crawl = self.analyze()
except Exception as e: except Exception as e:
print("PROFILE EXCEPTION {}".format(self.profile)) logging.debug("PROFILE EXCEPTION {}".format(self.profile))
print(traceback.format_exc()) logging.debug(traceback.format_exc())
# XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000; # XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000;
return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e)) return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e))
print("SAVE") logging.debug("SAVE")
crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani
return CrawlView.outputHtml(crawl) return CrawlView.outputHtml(crawl)
...@@ -81,11 +75,6 @@ class ScanController: ...@@ -81,11 +75,6 @@ class ScanController:
return "<div id='analysis-results'>{}</div>".format(result) return "<div id='analysis-results'>{}</div>".format(result)
def _print(self,str):
#logging.debug("({}) {}".format(self.profile ,str))
#print("({}) {}".format(self.profile ,str))
pass
def analyze(self): def analyze(self):
# spustit firefox pod profilem # spustit firefox pod profilem
print ("({}) browser launch".format(self.profile)) print ("({}) browser launch".format(self.profile))
...@@ -96,23 +85,23 @@ class ScanController: ...@@ -96,23 +85,23 @@ class ScanController:
#max_time = 3 # XXX #max_time = 3 # XXX
#,nsSocketTransport:5,nsStreamPump:5,nsHostResolver:5 #,nsSocketTransport:5,nsStreamPump:5,nsHostResolver:5
print("({}) FF -P {} -no-remote {}".format(self.profile,self.profile,self.url)) logging.debug("({}) FF -P {} -no-remote {}".format(self.profile,self.profile,self.url))
command = "export NSPR_LOG_MODULES=timestamp,nsHttp:5 ; export NSPR_LOG_FILE={} ; {} -P {} -no-remote '{}'".format(logfile, Config.browser, self.profile , "http://localhost/redirect/" + self.url) # http://localhost/redirect/ gets stripped by the extension command = "export NSPR_LOG_MODULES=timestamp,nsHttp:5 ; export NSPR_LOG_FILE={} ; {} -P {} -no-remote '{}'".format(logfile, Config.browser, self.profile , "http://localhost/redirect/" + self.url) # http://localhost/redirect/ gets stripped by the extension
# terminate Config.browser if hes not able to (everything has to be in single command because there is no heritance of $! amongst subprocesses) # terminate Config.browser if hes not able to (everything has to be in single command because there is no heritance of $! amongst subprocesses)
command += " & echo $!;ii=0; while [ -n \"`ps -p $! | grep {}`\" ];do echo \"({}) running\" ;ii=$((ii+1)); if [ $ii -gt {} ]; then echo '({}) kill';kill $!; break;fi; sleep 1; done".format(Config.browser, self.profile , Config.MAX_BROWSER_RUN_TIME,self.profile) #(pokud bezi proces $! (posledni backgroudovany process), spi 1 s) command += " & echo $!;ii=0; while [ -n \"`ps -p $! | grep {}`\" ];do echo \"({}) running\" ;ii=$((ii+1)); if [ $ii -gt {} ]; then echo '({}) kill';kill $!; break;fi; sleep 1; done".format(Config.browser, self.profile , Config.MAX_BROWSER_RUN_TIME,self.profile) #(pokud bezi proces $! (posledni backgroudovany process), spi 1 s)
#> /dev/null #> /dev/null
print(command) logging.debug(command)
subprocess.call([command], shell=True) subprocess.call([command], shell=True)
print("({}) stopped!".format(self.profile)) logging.debug("({}) stopped!".format(self.profile))
# shromazdit informace z analyz # shromazdit informace z analyz
crawl = Crawl(host = self.url, logDir = logDir, cacheDir = cacheDir) crawl = Crawl(host = self.url, logDir = logDir, cacheDir = cacheDir)
expiration = 0 expiration = 0
while os.path.isfile(logfile) == False: # i po zavreni FF nekdy trva, nez se soubor zapise while os.path.isfile(logfile) == False: # i po zavreni FF nekdy trva, nez se soubor zapise
expiration += 1 expiration += 1
print("({}) waiting to close...".format(self.profile)) logging.debug("({}) waiting to close...".format(self.profile))
if expiration > Config.MAX_BROWSER_EXPIRATION: if expiration > Config.MAX_BROWSER_EXPIRATION:
print("({}) time is run!".format(self.profile)) logging.debug("({}) time is run!".format(self.profile))
raise FileNotFoundError("time is run - browser expired") raise FileNotFoundError("time is run - browser expired")
time.sleep(1) time.sleep(1)
...@@ -179,23 +168,23 @@ class ScanController: ...@@ -179,23 +168,23 @@ class ScanController:
def dump(): def dump():
with open(Config.configFile, 'w') as f: with open(Config.configFile, 'w') as f:
json.dump(self.queueFF, f) json.dump(self.queueFF, f)
#print("UNKBOOK") #logging.debug("UNKBOOK")
try: try:
self.queueFF.pop(self.profile) self.queueFF.pop(self.profile)
dump() dump()
except KeyError: except KeyError:
print("Unbook failed") logging.debug("Unbook failed")
print(self.queueFF) logging.debug(self.queueFF)
raise raise
except OSError: except OSError:
print("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile)) logging.debug("({}) OS Error - interferuje s pustenym FF, ktere zere prilis pameti. Zkusime pockat.".format(self.profile))
time.sleep(10) # XX jestli funkcionalitu zachovat, dat sem pocitadlo, at je na konzoli videt akce time.sleep(10) # XX jestli funkcionalitu zachovat, dat sem pocitadlo, at je na konzoli videt akce
try: try:
dump() dump()
except OSError: except OSError:
print("({}) System se nezotavil.".format(self.profile)) logging.debug("({}) System se nezotavil.".format(self.profile))
return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze. return "Memory may be exhausted. See mdmaug-server/scan_controller.py for details." # FF sezral vsechnu pamet asi. Stranka je problematicka. UrlQuery podle me taky selze.
#print("UNKBOOKED") #logging.debug("UNKBOOKED")
def queue(self): def queue(self):
...@@ -209,13 +198,13 @@ class ScanController: ...@@ -209,13 +198,13 @@ class ScanController:
self.bookProfile() self.bookProfile()
break break
if self.profile == -1: if self.profile == -1:
print("(-1) PLNO, cekame par vterin") logging.debug("(-1) PLNO, cekame par vterin")
time.sleep(randint(5, 10)) #pockame par vterin time.sleep(randint(5, 10)) #pockame par vterin
else: else:
break #volny slot jsme nasli, muzeme dal break #volny slot jsme nasli, muzeme dal
#logging.debug(" profile " + str(self.profile ) + " queueFF:") #logging.debug(" profile " + str(self.profile ) + " queueFF:")
print(self.queueFF) logging.debug(self.queueFF)
#povedlo se zabookovat profil FF? #povedlo se zabookovat profil FF?
return self.profile > -1 return self.profile > -1
\ No newline at end of file
from http.server import SimpleHTTPRequestHandler from http.server import SimpleHTTPRequestHandler
from jinja2 import Environment from jinja2 import Environment
from jinja2 import FileSystemLoader from jinja2 import FileSystemLoader
from lib.config import Config
from lib.controller.api import Api
from lib.model.dbp import DbModel
from lib.model.dbp import Export
import logging
import mimetypes import mimetypes
import os import os
from lib.config import Config
#from lib.dbp import Db
from lib.rest import Rest
import time import time
from lib.dbp import Export, DbModel
env = Environment() env = Environment()
env.loader = FileSystemLoader(".") env.loader = FileSystemLoader(Config.DIR + "templates/")
class Server (SimpleHTTPRequestHandler): class Server(SimpleHTTPRequestHandler):
def favicon(self): def favicon(self):
with open('favicon.ico', 'rb') as f: with open('favicon.ico', 'rb') as f:
self.output(f.read(), "image/x-icon") self.output(f.read(), "image/x-icon")
def render_template(self, filename, ** kwargs): def render_template(self, filename, ** kwargs):
#pars = parse_qs(urlparse(self.path).query)
#if 'view' in pars:
#if pars["view"] == "message": # jsme nacteni pres iframe
self.output(env.get_template(filename).render(kwargs)) self.output(env.get_template(filename).render(kwargs))
#with open(filename,'r') as f:
#self.output(Environment().from_string(f.read()).render(kwargs))
def output(self, contents, contentType="text/html"): def output(self, contents, contentType="text/html"):
#http response self.send_response(200)
self.send_response(200) self.send_header("Content-type", contentType)
self.send_header("Content-type", contentType) self.end_headers()
self.end_headers() try:
try: self.wfile.write(contents)
self.wfile.write(contents) except:
except: self.wfile.write(contents.encode("UTF-8"))
self.wfile.write(contents.encode("UTF-8"))
def homepage(self): def homepage(self):
self.render_template("templates/homepage.html") self.render_template("homepage.html")
#self.output(render_template('_layout.html'))
def static_file(self, url): def static_file(self, url):
is_binary_string = lambda bytes: bool(bytes.translate(None, bytearray([7, 8, 9, 10, 12, 13, 27]) + bytearray(range(0x20, 0x100)))) is_binary_string = lambda bytes: bool(bytes.translate(None, bytearray([7, 8, 9, 10, 12, 13, 27]) + bytearray(range(0x20, 0x100))))
...@@ -48,24 +41,21 @@ class Server (SimpleHTTPRequestHandler): ...@@ -48,24 +41,21 @@ class Server (SimpleHTTPRequestHandler):
with open(url, type) as f: with open(url, type) as f:
self.output(f.read(), contentType=mimetypes.guess_type(url)) self.output(f.read(), contentType=mimetypes.guess_type(url))
def do_GET(self): def do_GET(self):
#XX Config.myDB.get_conn() # vytvorit spojeni, kdyby po 9 hodinach vypadlo. (Funguje?)
#from lib.dbp import Whitelist
DbModel.assureConnection()
path = self.path.split("/") path = self.path.split("/")
print("processing", path[1])
logging.debug("Request: {}".format(path[1]))
if path[1] == "": if path[1] == "":
self.homepage() return self.homepage()
elif os.path.isfile("static/" + path[1]): #faviconka, nebo jiny existujici soubor elif os.path.isfile(Config.DIR + "static/" + path[1]): #faviconka, nebo jiny existujici soubor
self.static_file("static/" + path[1]) return self.static_file(Config.DIR + "static/" + path[1])
elif path[1] == "rest": # /rest/analyze/web
cmd = path[2]
#url = path[2] if len(path) > 3 else ""
rest = Rest(self.path) DbModel.assureConnection()
#poslat vsechna data vys, pokud jsme vlozeni v iframe (https-mdm nemuze nacitat z http stroje - mixed content) if path[1] == "api": # /api/analyze/web
self.render_template("templates/_message.html", contents = rest.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/") cmd = path[2]
api = Api(self.path)
# send everything up, we are in an iframe
self.render_template("_message.html", contents=api.run(cmd), cmd=cmd, url=self.path, destination="https://mdm.nic.cz/")
elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni elif path[1] == "export": # /export/{days} - csv za poslednich 7 dni
url = self.path.split("/", 2) url = self.path.split("/", 2)
self.output(Export.exportView(days = url[2])) self.output(Export.exportView(days=url[2]))
\ No newline at end of file \ No newline at end of file
...@@ -21,8 +21,8 @@ class Domains: ...@@ -21,8 +21,8 @@ class Domains:
try: try:
return re.search('(([\w\-_]+(?:(?:\.[\w\-_]+)+)))', url).group(0) return re.search('(([\w\-_]+(?:(?:\.[\w\-_]+)+)))', url).group(0)
except TypeError: except TypeError:
print("Domains/url2domain type error") logging.debug("Domains/url2domain type error")
print(url) logging.debug(url)
raise #return "" raise #return ""
......
...@@ -47,7 +47,7 @@ class Crawl(defaultdict): ...@@ -47,7 +47,7 @@ class Crawl(defaultdict):
return state return state
def __setstate__(self,state): def __setstate__(self,state):
#print("fdsfsfds",self.__dict__) #logging.debug("fdsfsfds",self.__dict__)
for tup in state["keys"]: for tup in state["keys"]:
key, val = tup key, val = tup
self[key].__setstate__(val) self[key].__setstate__(val)
...@@ -151,13 +151,13 @@ c["seznam.cz"].addresses["8.8.8.8"].country = "preague" ...@@ -151,13 +151,13 @@ c["seznam.cz"].addresses["8.8.8.8"].country = "preague"
e = Crawl() e = Crawl()
e.__setstate__( c.__getstate__() ) e.__setstate__( c.__getstate__() )
#e = dill.loads(dill.dumps(c)) #e = dill.loads(dill.dumps(c))
print(str(c) == str(e)) logging.debug(str(c) == str(e))
print(c) logging.debug(c)
output = dump(c.__getstate__(), Dumper=Dumper) output = dump(c.__getstate__(), Dumper=Dumper)
e = Crawl() e = Crawl()
e.__setstate__(load(output, Loader=Loader)) e.__setstate__(load(output, Loader=Loader))
print(e) logging.debug(e)
print(str(c) == str(e)) logging.debug(str(c) == str(e))
""" """
\ No newline at end of file
...@@ -20,34 +20,34 @@ if(1): # Do not print all queries to stderr. ...@@ -20,34 +20,34 @@ if(1): # Do not print all queries to stderr.
class DbModel(Model): class DbModel(Model):
def assureConnection(): def assureConnection():
print ("Assure start.") logging.debug("Assure connection.")
try: # zkousim spustit nejaky prikaz try: # zkousim spustit nejaky prikaz
#print("1") #logging.debug("1")
Whitelist.select().count() Whitelist.select().count()
#print("2") #logging.debug("2")
except:# OperationalError: except:# OperationalError:
#print("3") #logging.debug("3")
Config.connect() Config.connect()
#print("4") #logging.debug("4")
try: # zkousim spustit nejaky prikaz try: # zkousim spustit nejaky prikaz
#print("5") #logging.debug("5")
Whitelist.select().count() Whitelist.select().count()
#print("6") #logging.debug("6")
except: except:
print("7 - failed") logging.debug("7 - failed")
raise raise
print("Connection to DB assured.") logging.debug("Connection to DB assured.")
"""A base model that will use our MySQL database""" """A base model that will use our MySQL database"""
def connect(): def connect():
logging.debug("connecting db....") logging.debug("connecting db....")
#DbModel.Meta.myDb = Config.myDB #DbModel.Meta.myDb = Config.myDB
# Config.myDB.connect() # XX kupodivu toto neni potreba # Config.myDB.connect() # XX kupodivu toto neni potreba
#print(Whitelist.select().count()) #logging.debug(Whitelist.select().count())
#print("Connecting to DB.") #logging.debug("Connecting to DB.")
#print("Start test.") #logging.debug("Start test.")
#print(Whitelist.matches("www.mozilla.org")) #logging.debug(Whitelist.matches("www.mozilla.org"))
#print("End test.") #logging.debug("End test.")
def disconnect(): def disconnect():
logging.debug("... db ends.") logging.debug("... db ends.")
...@@ -57,9 +57,9 @@ class DbModel(Model): ...@@ -57,9 +57,9 @@ class DbModel(Model):
database = Config.myDB database = Config.myDB