api.py 4.71 KB
Newer Older
1 2
import json
import logging
3
import subprocess
4 5
from builtins import FileNotFoundError
from glob import escape
6 7 8

from peewee import IntegrityError

9
from mdmaug.lib.model.crawl import Crawl
10 11
from .scan_controller import ScanController
from ..config import Config
12
from ..model.dbp import Encounter, Whitelist
13
from ..parser.traffic_log_parser import TrafficLogParser
14
from ...templates.crawl_view import CrawlView
15

Edvard Rejthar's avatar
Edvard Rejthar committed
16
logger = logging.getLogger("mdmaug")
17

18

19
class Api:
20

21 22
    def run(self, request):
        """ Accept command
Edvard Rejthar's avatar
Edvard Rejthar committed
23
        :type request: dict from URL request. /api/analyze=cache/http://example.com → {"api": True, "analyze": cache, "page": "http://example.com"}
24
        """
25

26 27
        crawl = None

28
        if "analyze" in request:
29 30 31
            map_ = {"fresh": None, "cached": 1, "weekcache": 7, "oldcache": True, True: None}
            if request["analyze"] in map_:
                days = map_[request["analyze"]]
32
            else:
33
                days = int(request["analyze"])
34 35
            crawl = ScanController().launch(request["page"], days, request.get("autoprune") in ["y", "1", True],
                                            request.get("creation_spree") in ["y", "1", True])
36 37
        elif "aggregate" in request:
            crawl = self.aggregate(request)
38 39 40 41 42 43
        elif "scan" in request:
            if "date" not in request:
                request["date"] = ""
            crawl = ScanController().get_scan(escape(request["scan"]), scan=escape(request["date"]))
        elif "prune" in request:
            return ScanController.prune()
44
        elif "nicify" in request:
Edvard Rejthar's avatar
Edvard Rejthar committed
45
            return TrafficLogParser.getStylesheet() + TrafficLogParser.nicify_file(request["page"])
46
        elif "vote" in request:  # /api/vote/block/example.org/10.0.0.1
47
            return Encounter.vote(request["vote"], request["page"])
48 49
        elif "reset" in request:
            self.reset()
50
            return "reset"
51
        else:
Edvard Rejthar's avatar
Edvard Rejthar committed
52
            return "Unknown API method."
53 54

        if crawl:
55 56
            if type(crawl) is str:
                return crawl  # containing an error message
Edvard Rejthar's avatar
Edvard Rejthar committed
57
            elif request["api"] == "json":
58 59 60
                return CrawlView.output_json(crawl)
            else:
                return CrawlView.output_html(crawl)
61

62 63 64 65 66 67 68
    @staticmethod
    def aggregate(request):
        date_from = int(request["from"])
        date_to = int(request["to"])
        crawl = Crawl()
        scan_count = set()
        domain_count = set()
69 70 71 72 73 74 75 76 77
        for domain, scan in ScanController.get_all_scans():
            if date_from < int(scan) < date_to:
                try:
                    scan_count.add("/".join([domain, scan]))
                    domain_count.add(domain)
                    crawl += Crawl.load_from_scan(domain, scan)
                except FileNotFoundError:
                    logger.warning("Wrong analysis stored at %s/%s", domain, scan)
                    pass
78 79 80 81 82 83

        crawl.title = f"Merged {len(scan_count)} scans from {len(domain_count)} domains"
        if not crawl:
            crawl = "No scan between these dates."
        return crawl

84
    @staticmethod
85
    def reset():
Edvard Rejthar's avatar
Edvard Rejthar committed
86 87
        logger.debug("resetting running browsers")
        with open(Config.config_file, 'w') as f:  # clear the queue
88
            json.dump({}, f)
89
        subprocess.call(["pkill", Config.browser])  # kill frozen browsers
90

91
    # prida 2ld domenu mezi whitelistovane
92
    def whitelist(self):
93
        logger.debug("whitelisting")
94
        # Db.cur = Db.connection.cursor()
95
        # self._logger.debug(Db.cur.execute("""REPLACE INTO whitelist set domain = %s""", (self.origin_domain, )))
96 97
        # Db.connection.commit()
        # Db.cur.close()
98
        return  # not yet implemented
99
        try:
100
            Whitelist.insert(domain=self.origin_domain).execute()
101
        except IntegrityError:
102
            pass  # already inserted
103

104 105
    @staticmethod
    def get_undecided():
106
        logger.debug("XXX not implemented yet - table of undecideds domain since last export")
107
        pass
108

109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127
# elif "inspect" in request:
#     # XXX → migrate to dbp
#     output = []
#     for row in Config.db.execute_sql("SELECT url from encounter where host = %s", request["inspect"]):
#         output.append(row[0])
#     return "<br>".join(output)
# elif "decide" in request:  # XX deprecated?
#     return self.get_undecided()
# elif "whitelist" in request:  # XXX not implemented yet
#     """url = path.split("/", 3)
#     if len(url) > 3:
#         self._setWebsite(url[2])  # osetrit self.website, ze je URL, a nikoli shell
#         logger.debug("XXX nejsem si jist, zda url je spravne na url[2]")  # XXX
#         logger.debug(url)  # XXX
#         quit()  # XXX
#         logger.debug(self.website)
#         logger.debug(self.origin_domain)
#     return self.whitelist()"""
#     return "Implement first if needed."