Commit a2c7179f authored by Edvard Rejthar's avatar Edvard Rejthar

Občas nefungovalo voting.

Zjistil jsem, že to bylo, protože se použával (INNER) JOIN místo LEFT JOINu.
Ale opravdu, opravdu jsem nepochopil, proč INNER JOIN nefungoval:


SELECT status FROM `turris` AS t1
left outer  JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 1
SELECT status FROM `turris` AS t1
outer JOIN `status` AS t2 ON (`t2`.`id` = `t1`.`status`)
WHERE (`t1`.`remoteHost` = 'api.w.org') LIMIT 1 # vraci status = 0. A to prestoze t1.status samo o sobe je rovno 1 (v tabulce status je pod id = 1 -> name = allow, místo name = n/a, které mi to házelo do prezentace.)
parent 197e6772
......@@ -101,12 +101,12 @@ originalListener: null,
//if (1){ //XXX XXX XXX
//ulozit nactenou stranku
console.error("current last", request.originalURI.spec, lastRequest);//XX debug
//console.error("current last", request.originalURI.spec, lastRequest);//XX debug
if (request.originalURI.spec != lastRequest) {//novy request, vytisknout do souboru hlavicku
lastRequest = request.originalURI.spec;
console.error("request", request.originalURI.spec);
//console.error("request", request.originalURI.spec);
header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
console.error("("+profileName+") " + header);//xx debug
//console.error("("+profileName+") " + header);//xx debug
} else {
header = ""; //preskakujeme hlavicku
}
......
......@@ -144,7 +144,7 @@ TracingListener.prototype = {
lastRequest = request.originalURI.spec;
//console.error("request", request.originalURI.spec);
header = request.originalURI.spec + " " + request.getResponseHeader("Content-Type") + "\n";
console.error("(" + profileName + ") " + header);//xx debug
//console.error("(" + profileName + ") " + header);//xx debug
} else {
header = ""; //preskakujeme hlavicku
}
......
......@@ -126,7 +126,9 @@ class MetadataParser:
print("("+str(threadNumber),"EDVARD 5")
crawlDomainIp.country, crawlDomainIp.city = Domains.ip2countryAndCity(ip)
# kontaktovat PDNS
pdns.update(Domains.ip2pdnsDomains(ip))
items = Domains.ip2pdnsDomains(ip)
if items:
pdns.update(items)
print("("+str(threadNumber),"EDVARD 6")
return None
else: #zadna ip neni k dispozici, domena je asi propadla, hlas patri jmenu domeny
......
......@@ -70,7 +70,8 @@ class ScanController:
print(traceback.format_exc())
# XX Pokud je potiz, ze JS zabiji FF, mozno experimentovat s ulimit -Sv 500000;
return ("PROFILE EXCEPTION ({}) {} See logs, i.e. mdmaug/nohup.out. ".format(self.profile, e))
print("SAVE")
crawl.saveToFile(crawl.cacheDir + ScanController.CRAWL_FILE) # ulozit vysledky hledani
return CrawlView.outputHtml(crawl)
......
from peewee import *
from peewee import RawQuery
import datetime
import logging
from lib.config import Config
from urllib.parse import urlparse
from lib.domains import Domains
import logging
from peewee import *
from peewee import RawQuery
import time
#logging.basicConfig(level=logging.WARNING, format="%(message)s")
from lib.domains import Domains
from urllib.parse import urlparse
#myDB = MySQLDatabase("mdmaug", host='172.20.7.10', port=3306, user="root", passwd="lopuch", threadlocals=False)
#myDB.register_fields({'primary_key': 'BIGINT AUTOINCREMENT'})
#myDB.connect()
if(0): # Do not print all queries to stderr.
logger = logging.getLogger('peewee')
logger.setLevel(logging.WARNING)
class DbModel(Model):
def assureConnection():
......@@ -83,15 +85,14 @@ class Status(DbModel):
if status == 3:
return "block"
class Export(DbModel):
id = IntegerField(primary_key = True)
id = IntegerField(primary_key=True)
timestamp = DateTimeField(datetime.datetime.now())
##
# Exportuje csv
# Kdyz je urcen parametr days, bere export za tolik dni. Jinak bere export od posledniho exportu v tabulce Export. (Od tebe doby, co naposled byla volana funkce exportConfirm.)
def exportView(days = None):
def exportView(days=None):
#nacte vsechna ip, ktera loggujeme (status=2) ci blokujeme (status=3), jejichz timestamp je vetsi nez posledni export
xstr = lambda s:'' if s is None else str(s)
delimiter = ","
......@@ -104,7 +105,7 @@ class Export(DbModel):
q = "SELECT ip, port, url, `remoteHost`, `otherDetails`,status from turris where status > 1 and `ip` IS NOT NULL and timestamp > "
if days:
q += "NOW() - INTERVAL {} DAY ".format(int(days))
else :
else:
q += "(select case when MAX(timestamp IS NULL)=0 THEN max(timestamp) ELSE 0 END from export)"
print(q)
rq = RawQuery(Turris, q).execute()
......@@ -134,21 +135,21 @@ class Turris(DbModel):
id = PrimaryKeyField()
timestamp = DateTimeField(datetime.datetime.now())
status = IntegerField()
date = IntegerField(default = 0)
date = IntegerField(default=0)
ip = CharField(45, default=None)
port = IntegerField(default = 0)
port = IntegerField(default=0)
url = CharField(255, default="")
block = IntegerField(default = 0)
block = IntegerField(default=0)
remoteHost = CharField(255, default="")
otherDetails = CharField(255, default="")
def getVote(host = None, ip = None):
def getVote(host=None, ip=None):
status = None
try:
if host:
status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status
status = Turris.select().join(Status,JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.remoteHost == host).limit(1).get().status
if ip:
status = Turris.select().join(Status, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
status = Turris.select().join(Status, JOIN.LEFT_OUTER, on=(Status.id == Turris.status)).where(Turris.ip == ip).limit(1).get().status
except:
status = 0
return Status.int2word(status)
......
This diff is collapsed.
......@@ -16,7 +16,7 @@ class CrawlView:
r += "<div class='screenshot'><img class='thumbnail' src='data:image/png;base64,{}' /></div>".format(f.read())
for domain in crawl.keys():
if (Whitelist.matches(domain)):
continue
continue
r += "<div class=web data-vote='{}' data-safebrowsing-suspicious='{}'>".format(Turris.getVote(host=domain), Domains.isSuspicious(domain, 'attr')) # crawl[domain].vote
r += "<span class=domain>{}</span>".format(domain)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment