Commit db4c1e13 authored by Edvard Rejthar's avatar Edvard Rejthar

Merge branch 'master' of gitlab.labs.nic.cz:csirt/mdmaug

parents 3db7b09e fb6ebd63
{"name": "firefox_mdmaug_writer", "description": "Firefox disk writer", "path": "/opt/mdmaug/mdmaug/bin/firefox_mdmaug_writer.py", "type": "stdio", "allowed_extensions": [ "mdmaug@csirt.cz" ] } {"name": "firefox_mdmaug_writer", "description": "Firefox disk writer", "path": "/opt/mdmaug/firefox_mdmaug_writer.py", "type": "stdio", "allowed_extensions": [ "mdmaug@csirt.cz" ] }
...@@ -32,6 +32,8 @@ echo " ***** Copying files into" $DESTINATION ...@@ -32,6 +32,8 @@ echo " ***** Copying files into" $DESTINATION
mkdir $DESTINATION mkdir $DESTINATION
cp -r mdmaug $DESTINATION cp -r mdmaug $DESTINATION
cp -r .mozilla $DESTINATION cp -r .mozilla $DESTINATION
cp misc/production.ini $DESTINATION
cp misc/firefox_mdmaug_writer.py $DESTINATION
cp *.md $DESTINATION cp *.md $DESTINATION
cd $DESTINATION cd $DESTINATION
......
...@@ -10,20 +10,17 @@ Scans a website for a sign of a parasite hosts or commands. ...@@ -10,20 +10,17 @@ Scans a website for a sign of a parasite hosts or commands.
4. Perform installation: ```/tmp/mdmaug/INSTALL``` 4. Perform installation: ```/tmp/mdmaug/INSTALL```
5. Everything should be located in `/opt/mdmaug`. 5. Everything should be located in `/opt/mdmaug`.
6. For testing purposes, launch it under newly created `mdmaug` user: `su - mdmaug -c 'python3 -m mdmaug'` 6. For testing purposes, launch it under newly created `mdmaug` user: `su - mdmaug -c 'python3 -m mdmaug'`
7. Connect in the browser at: https://127.0.0.1:5000 7. Connect in the browser at: https://127.0.0.1:8000
8. Try analysing `https://127.0.0.1:5000/static/demopage.html` on local server 8. Try analysing `https://127.0.0.1:8000/static/demopage.html` on local server
9. For deployment, configure nginx properly to be used with flask 9. For deployment, configure nginx (`sudo apt install nginx`) properly to be used with flask:
* If you are using systemd you may want to copy `misc/mdmaug.service` to `/etc/systemd/system/` so that MDMaug runs after restart (or with `sudo service mdmaug start`)
* `misc/mdmaug.nginx` can be integrated to nginx `/etc/nginx/sites-available/` (& symlinked to `/etc/nginx/sites-enabled/`)
### Notes ### Notes
* If you want other count of profiles than 21, change `./INSTALL` + `mdmaug/lib/config.py` + `.mozilla/firefox/profiles.ini` * If you want other count of profiles than 21, change `./INSTALL` + `mdmaug/lib/config.py` + `.mozilla/firefox/profiles.ini`
* You may put ```03 1,7,13,19 * * * ~/mdmaug-launch``` in ```crontab -e``` of user mdmaug.
* We are using Python3.6+, Firefox 62.0 * We are using Python3.6+, Firefox 62.0
* You may launch MDMaug with environmental variable `PORT` to change the port the application is bound to
## Tips
* You may use /static/demopage.html as a testing page.
* You may launch MDMaug with environmental variable `PORT` to change the port the applicaction is bound to
### Troubleshooting ### Troubleshooting
...@@ -32,8 +29,8 @@ Scans a website for a sign of a parasite hosts or commands. ...@@ -32,8 +29,8 @@ Scans a website for a sign of a parasite hosts or commands.
#### Debugging session #### Debugging session
I'm launching it like this: * I'm launching it like this:: `su - mdmaug -c 'LC_ALL=C.UTF-8 FLASK_ENV=development FLASK_APP=mdmaug.__main__:app flask run -h 217.31.202.41 -p 8001`
`su - mdmaug -c 'export FLASK_APP=mdmaug.__main__:app && export PYTHONPATH=/opt/mdmaug/mdmaug && ./local/bin/flask run'` * or `su - mdmaug -c 'PORT=8001 python3 -m mdmaug`
#### Wanna see what Firefox is really doing? #### Wanna see what Firefox is really doing?
...@@ -42,4 +39,4 @@ I'm launching it like this: ...@@ -42,4 +39,4 @@ I'm launching it like this:
* If no Firefox window appears try * If no Firefox window appears try
* `xhost +local:mdmaug` if you're on the same machine * `xhost +local:mdmaug` if you're on the same machine
* `root@mdmaugmachine$xauth list` on remote root and `mdmaug$xauth add ...` display cookie * `root@mdmaugmachine$xauth list` on remote root and `mdmaug$xauth add ...` display cookie
* When Firefox window appear, run MDMaug with `export FIREFOX_DEBUG=1`. Now, instead of virtual display your monitor should be used. * When Firefox window appear, run MDMaug with `FIREFOX_DEBUG=1`. Now, instead of virtual display your monitor should be used.
#!/usr/bin/env python3
Works well but process requests one by one:
from http.server import HTTPServer, BaseHTTPRequestHandler
from socketserver import ThreadingMixIn
import threading
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.end_headers()
message = threading.currentThread().getName()
self.wfile.write(message)
self.wfile.write('\n')
return
class ThreadedHTTPServer(ThreadingMixIn, HTTPServer):
"""Handle requests in a separate thread."""
if __name__ == '__main__':
server = ThreadedHTTPServer(('localhost', 80), Handler)
print('Starting server, use <Ctrl-C> to stop')
server.serve_forever()
exit()
from http.server import HTTPServer, SimpleHTTPRequestHandler
class Server(SimpleHTTPRequestHandler):
def do_GET(self):
self.send_response(200)
self.end_headers()
self.wfile.write("MDMaug: dumb server used as a landing page for the browser extension. " \
"Because this needs to be loaded before any web page but needs a web page to be loaded.".encode("UTF-8"))
httpd = HTTPServer(('127.0.0.1', 80), Server)
# httpd.socket = ssl.wrap_socket(httpd.socket,
# server_side=True,
# certfile= Config.DIR + 'python.pem', # together private + cert, http://stackoverflow.com/questions/19705785/python-3-https-webserver
# ssl_version=ssl.PROTOCOL_TLSv1)
httpd.serve_forever()
Couldnt handle much requests at once
##!/usr/bin/env bash
#while true; do { echo -e "HTTP/1.1 200 OK\r\n$(date)\r\n\r\nMDMaug: dumb server used as a landing page for the browser extension. Because this needs to be loaded before any web page but needs a web page to be loaded." | nc -vl 80; } done
\ No newline at end of file
# XX shouldnt I delete this file?
su - mdmaug -c 'cd /home/mdmaug/mdmaug/ ; python3 mdmaug.py'
pkill python3 #pri Ctrl+C v prikazu su se uzavre jen terminal, ale ne uz python skript. Takhle to zas zabije veskery Python, ale sandboxovy-zavirovany uzivatel mdmaug stejne ma byt jen na spousteni tohohle skriptu.
\ No newline at end of file
...@@ -24,9 +24,9 @@ class Config: ...@@ -24,9 +24,9 @@ class Config:
profile_count = 21 # number of Firefox profiles. Its name is just a number – 0,1... profile_count = 21 # number of Firefox profiles. Its name is just a number – 0,1...
browser = 'firefox' # iceweasel, firefox. What browser gets launched. browser = 'firefox' # iceweasel, firefox. What browser gets launched.
config_file = '/opt/mdmaug/.cache/mdmaug-scans/_tmp/queue.cache' # RAM disk was too small: '/tmp/mdm/queue.cache' config_file = '/opt/mdmaug/.cache/mdmaug-scans/_tmp/queue.cache' # RAM disk was too small: '/tmp/mdm/queue.cache'
APP_PORT = 5000 APP_PORT = 8000
APP_IP = "127.0.0.1" # CHANGE HERE IF NOT RUN ON LOCALHOST APP_IP = "217.31.202.41" # "127.0.0.1" # CHANGE HERE IF NOT RUN ON LOCALHOST
APP_HOST = f'http://{APP_IP}:{APP_PORT}' # YOU MAY NEED TO CHANGE **https** APP_HOST = f'https://{APP_IP}:{APP_PORT}' # YOU MAY NEED TO CHANGE **https**
LOG_DIR = "/opt/mdmaug/.cache/mdmaug-scans/_tmp/" LOG_DIR = "/opt/mdmaug/.cache/mdmaug-scans/_tmp/"
CACHE_DIR = "/opt/mdmaug/.cache/mdmaug-scans/" CACHE_DIR = "/opt/mdmaug/.cache/mdmaug-scans/"
ALLOWED_DESTINATION = {"mdm.nic.cz": "https://mdm.nic.cz", ALLOWED_DESTINATION = {"mdm.nic.cz": "https://mdm.nic.cz",
...@@ -37,10 +37,11 @@ class Config: ...@@ -37,10 +37,11 @@ class Config:
lock = threading.RLock() lock = threading.RLock()
THUMBNAIL_SIZE = 640, 640 THUMBNAIL_SIZE = 640, 640
MAX_WHOIS_DOMAIN_THREADS = 10 # spusti maximalne 10 threadu doraz, jednou mi to totiz preteklo (kazda domena spusti jeste tolik threadu, kolik ma IP, ale tech byva jen par) MAX_WHOIS_DOMAIN_THREADS = 10 # spusti maximalne 10 threadu doraz, jednou mi to totiz preteklo (kazda domena spusti jeste tolik threadu, kolik ma IP, ale tech byva jen par)
MAX_BROWSER_RUN_TIME = 25 # maximum time for a browser to run MAX_BROWSER_RUN_TIME = 45 # maximum time for a browser to run
MAX_BROWSER_EXPIRATION = 15 # seconds that we wait before killing the browser (waiting for the files to be written) MAX_BROWSER_EXPIRATION = 15 # seconds that we wait before killing the browser (waiting for the files to be written)
EXPORT_NOTBLOCK_TLD = ".cz" # lowercase; this TLD is ignored in the export EXPORT_NOTBLOCK_TLD = ".cz" # lowercase; this TLD is ignored in the export
CRAWL_FILE = "analysis.json" CRAWL_FILE = "analysis.json"
SCREENSHOT_FILE = "screenshot.base64"
@staticmethod @staticmethod
def connect(): def connect():
......
...@@ -18,7 +18,6 @@ from ..parser.spy_parser import SpyParser ...@@ -18,7 +18,6 @@ from ..parser.spy_parser import SpyParser
logger = logging.getLogger("mdmaug") logger = logging.getLogger("mdmaug")
# from yaml import load, dump
class Crawl(defaultdict): class Crawl(defaultdict):
""" Analysis results data model""" """ Analysis results data model"""
......
...@@ -12,7 +12,7 @@ class ScreenshotParser: ...@@ -12,7 +12,7 @@ class ScreenshotParser:
def __init__(self, crawl): def __init__(self, crawl):
screenfile = crawl.cache_dir+'screenshot.base64' screenfile = crawl.cache_dir + Config.SCREENSHOT_FILE
if os.path.isfile(screenfile): if os.path.isfile(screenfile):
with open(screenfile, "r+") as f: with open(screenfile, "r+") as f:
data = (b64decode(f.read())) data = (b64decode(f.read()))
...@@ -26,4 +26,5 @@ class ScreenshotParser: ...@@ -26,4 +26,5 @@ class ScreenshotParser:
f.write(b64encode(data.getvalue()).decode("utf-8")) f.write(b64encode(data.getvalue()).decode("utf-8"))
f.truncate() f.truncate()
#"<img class='thumbnail' src='data:image/png;base64,{}' />".format(b64encode(data.getvalue()).decode("utf-8")) #"<img class='thumbnail' src='data:image/png;base64,{}' />".format(b64encode(data.getvalue()).decode("utf-8"))
crawl.screenfile = screenfile os.rename(screenfile, crawl.cache_dir + Config.SCREENSHOT_FILE)
crawl.screenfile = 1 # screenfile
...@@ -49,7 +49,9 @@ class TrafficLogParser: ...@@ -49,7 +49,9 @@ class TrafficLogParser:
continue continue
# logger.debug(Domains.url2domain(url), Domains.url2path(url), path) # logger.debug(Domains.url2domain(url), Domains.url2path(url), path)
crawl[url2domain(url)].urls[url2path(url)].sourcefiles.append(path) o = crawl[url2domain(url)].urls[url2path(url)]
if f.readline() != "": # some content has been fetched
o.sourcefiles.append(path)
@staticmethod @staticmethod
def nicify_file(sourcefile): def nicify_file(sourcefile):
......
#!/bin/bash
# This file may be launched by CRON. 03 1,7,13,19 * * * ~/mdmaug/mdmaug-launch
echo "mdmaug-launch start" >> ~/log.log
pkill -f mdmaug.py
pkill -f Xvfb
export PYTHONPATH=$PYTHONPATH:/opt/mdmaug/mdmaug/
cd /opt/mdmaug/mdmaug && ./mdmaug.py 2>&1 | /usr/bin/logger -t mdmaugtag
whoami >> ~/log.log
\ No newline at end of file
This diff is collapsed.
This diff is collapsed.
...@@ -6,6 +6,7 @@ from ..lib.controller.scan_controller import ScanController ...@@ -6,6 +6,7 @@ from ..lib.controller.scan_controller import ScanController
from ..lib.domains import is_suspicious, url2domain, domain2dir from ..lib.domains import is_suspicious, url2domain, domain2dir
from ..lib.model.dbp import Encounter from ..lib.model.dbp import Encounter
from ..lib.model.dbp import Whitelist from ..lib.model.dbp import Whitelist
from ..lib.config import Config
class CrawlView: class CrawlView:
...@@ -13,10 +14,11 @@ class CrawlView: ...@@ -13,10 +14,11 @@ class CrawlView:
@staticmethod @staticmethod
def output_json(crawl, expand_screenfile=False): def output_json(crawl, expand_screenfile=False):
# print(crawl) if crawl.screenfile:
if expand_screenfile and crawl.screenfile: crawl.screenfile = crawl.cache_dir + Config.SCREENSHOT_FILE # expand '1' to screnshot file path
with open(crawl.screenfile, "r") as f: if expand_screenfile:
crawl.screenfile = f.read() with open(crawl.screenfile, "r") as f:
crawl.screenfile = f.read()
output = {"screenfile": crawl.screenfile, output = {"screenfile": crawl.screenfile,
"host": crawl.host, "host": crawl.host,
......
...@@ -105,7 +105,6 @@ ...@@ -105,7 +105,6 @@
<script> <script>
var APP_HOST = "{{ APP_HOST }}"; // "http://localhost:5000" var APP_HOST = "{{ APP_HOST }}"; // "http://localhost:5000"
</script> </script>
<script src="static/homepage_script.js"></script>
<script src="static/mdmaug-analysis.js"></script> <script src="static/mdmaug-analysis.js"></script>
......
##
# You should look at the following URL's in order to grasp a solid understanding
# of Nginx configuration files in order to fully unleash the power of Nginx.
# https://www.nginx.com/resources/wiki/start/
# https://www.nginx.com/resources/wiki/start/topics/tutorials/config_pitfalls/
# https://wiki.debian.org/Nginx/DirectoryStructure
#
# In most cases, administrators will remove this file from sites-enabled/ and
# leave it as reference inside of sites-available where it will continue to be
# updated by the nginx packaging team.
#
# This file will automatically load configuration files provided by other
# applications, such as Drupal or Wordpress. These applications will be made
# available underneath a path with that package name, such as /drupal8.
#
# Please see /usr/share/doc/nginx-doc/examples/ for more detailed examples.
##
# Default server configuration
#
server {
listen 80 default_server;
listen [::]:80 default_server;
# SSL configuration
#
listen 8000 ssl default_server;
listen [::]:8000 ssl default_server;
ssl_certificate /opt/mdmaug/mdmaug/cert-mdmaug.pem;
ssl_certificate_key /opt/mdmaug/mdmaug/key-mdmaug.pem;
#
# Note: You should disable gzip for SSL traffic.
# See: https://bugs.debian.org/773332
#
# Read up on ssl_ciphers to ensure a secure configuration.
# See: https://bugs.debian.org/765782
#
# Self signed certs generated by the ssl-cert package
# Don't use them in a production server!
#
# include snippets/snakeoil.conf;
root /var/www/html;
# Add index.php to the list if you are using PHP
index index.html index.htm index.nginx-debian.html;
server_name _;
merge_slashes off; # we need to allow double slash in: /api=json/analyze/http://example.com
location / {
try_files $uri @mdmaug;
}
location @mdmaug {
include uwsgi_params;
uwsgi_pass unix:/tmp/mdmaug.sock;
}
}
[Unit]
Description=uWSGI server instance configured to serve MDMaug
[Service]
Environment="PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/mdmaug/mdmaug"
WorkingDirectory=/opt/mdmaug
ExecStart=/usr/local/bin/uwsgi -s /tmp/mdmaug.sock --manage-script-name --ini /opt/mdmaug/production.ini
User=mdmaug
Group=www-data
Restart=on-failure
RestartSec=10s
[Install]
WantedBy=multi-user.target
[uwsgi]
module = mdmaug.__main__
callable = app
master = true
processes = 21
socket = /tmp/mdmaug.sock
chmod-socket = 660
vacuum = true
die-on-term = true
UWSGI_SCHEME = https
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment