Refactor web_scan.py for improved readability and maintainability; add cron job script for npm log backups; update start.ps1 for consistent formatting.
This commit is contained in:
zwitschi committed
1 parent
0d1cd21468
commit
c2eba88ac8
113 files changed
+772606
-772579
No files matched your search
+171
-171
@@ -1,171 +1,171 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Flask web UI for the intrusion-detection system.
|
||||
|
||||
A localhost-only web server (``127.0.0.1:<high port>``) for running scans,
|
||||
browsing reports, and managing the MikroTik blocklist / firewall.
|
||||
|
||||
Run directly: ``python -m app.web``
|
||||
Or via the entry point: ``python app.py --web`` (see ``serve()``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from flask import Flask, jsonify, request
|
||||
|
||||
from ..utils.constants import (
|
||||
APP_REPORT_DATA_DIR,
|
||||
APP_REPORT_DIR,
|
||||
COPYPARTY_LOG_PATH,
|
||||
COPYPARTY_REPORT_DIR,
|
||||
DEFAULT_ERROR_RATIO,
|
||||
DEFAULT_MIN_DENIED,
|
||||
DEFAULT_MIN_ERRORS,
|
||||
DEFAULT_TOP,
|
||||
MIKROTIK_REPORT_DIR,
|
||||
NPM_LOG_PATH,
|
||||
NPM_REPORT_DIR,
|
||||
REPORT_DIR,
|
||||
WEB_HOST,
|
||||
WEB_PORT,
|
||||
)
|
||||
from ..firewall import MikrotikFirewall
|
||||
from ..scan import ScanParams
|
||||
from . import web_scan
|
||||
|
||||
#: report sources exposed by the API -> report directory
|
||||
REPORT_SOURCES = {
|
||||
"copyparty": COPYPARTY_REPORT_DIR,
|
||||
"npm": NPM_REPORT_DIR,
|
||||
"app": APP_REPORT_DIR,
|
||||
"mikrotik": MIKROTIK_REPORT_DIR,
|
||||
}
|
||||
|
||||
SCAN_KINDS = ("copyparty", "npm", "consolidated")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _count_lines(path: str) -> int | None:
|
||||
"""Return the line count of a file, or None if it cannot be read."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return sum(1 for _ in f)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _read_lines(path: str) -> list[str]:
|
||||
"""Return non-empty lines of a file, or [] if it cannot be read."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return [line.strip() for line in f if line.strip()]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def _read_json(path: str, default: list | dict | None = None) -> list | dict:
|
||||
"""Read a JSON file, returning ``default`` (or []) on failure."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return default if default is not None else []
|
||||
|
||||
|
||||
def _params_from_request() -> ScanParams:
|
||||
"""Build ScanParams from the JSON request body, falling back to defaults."""
|
||||
data = request.get_json(silent=True) or {}
|
||||
|
||||
def _pick(key: str, default):
|
||||
value = data.get(key)
|
||||
return default if value is None or value == "" else value
|
||||
|
||||
return ScanParams(
|
||||
copyparty_log_path=str(
|
||||
_pick("copyparty_log_path", COPYPARTY_LOG_PATH)),
|
||||
copyparty_report_dir=str(
|
||||
_pick("copyparty_report_dir", COPYPARTY_REPORT_DIR)),
|
||||
min_denied=int(_pick("min_denied", DEFAULT_MIN_DENIED)),
|
||||
npm_log_path=str(_pick("npm_log_path", NPM_LOG_PATH)),
|
||||
npm_report_dir=str(_pick("npm_report_dir", NPM_REPORT_DIR)),
|
||||
min_errors=int(_pick("min_errors", DEFAULT_MIN_ERRORS)),
|
||||
error_ratio=float(_pick("error_ratio", DEFAULT_ERROR_RATIO)),
|
||||
report_dir=str(_pick("report_dir", REPORT_DIR)),
|
||||
top=int(_pick("top", DEFAULT_TOP)),
|
||||
quiet=bool(data.get("quiet", False)),
|
||||
full=bool(data.get("full", False)),
|
||||
update_firewall=bool(data.get("update_firewall", False)),
|
||||
dry_run=bool(data.get("dry_run", False)),
|
||||
)
|
||||
|
||||
|
||||
def _with_router(fn):
|
||||
"""Connect to the router, run ``fn(fw)``, always disconnect.
|
||||
|
||||
Returns a JSON error response (502) if the router is unreachable.
|
||||
"""
|
||||
fw = MikrotikFirewall()
|
||||
try:
|
||||
fw.connect()
|
||||
return fn(fw)
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 502
|
||||
finally:
|
||||
fw.disconnect()
|
||||
|
||||
|
||||
def _sync_firewall(dry_run: bool):
|
||||
"""Sync the latest consolidated blocklist to the router (preview or apply)."""
|
||||
v4 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
|
||||
v6 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt"))
|
||||
|
||||
def _sync(fw):
|
||||
changes = fw.sync_blocklist(v4, v6, dry_run=dry_run)
|
||||
return jsonify({**changes, "dry_run": dry_run})
|
||||
|
||||
return _with_router(_sync)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# app factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def create_app() -> Flask:
|
||||
"""Build and configure the Flask application."""
|
||||
_app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
app = Flask(__name__,
|
||||
template_folder=os.path.join(_app_dir, "templates"),
|
||||
static_folder=os.path.join(_app_dir, "static"))
|
||||
|
||||
# Seed the in-memory job store from disk so the dashboard keeps showing
|
||||
# recent scans across server restarts.
|
||||
web_scan.load_persisted()
|
||||
|
||||
from .routes import register_routes
|
||||
|
||||
helpers = {
|
||||
"_count_lines": _count_lines,
|
||||
"_read_lines": _read_lines,
|
||||
"_read_json": _read_json,
|
||||
"_params_from_request": _params_from_request,
|
||||
"_with_router": _with_router,
|
||||
"_sync_firewall": _sync_firewall,
|
||||
}
|
||||
register_routes(app, REPORT_SOURCES, SCAN_KINDS, web_scan, helpers)
|
||||
return app
|
||||
|
||||
|
||||
def serve(host: str = WEB_HOST, port: int = WEB_PORT) -> None:
|
||||
"""Run the web server on localhost at the configured high port."""
|
||||
app = create_app()
|
||||
print(f"Intrusion Detection web UI running at http://{host}:{port}")
|
||||
app.run(host=host, port=port, use_reloader=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve()
|
||||
#!/usr/bin/env python3
|
||||
"""Flask web UI for the intrusion-detection system.
|
||||
|
||||
A localhost-only web server (``127.0.0.1:<high port>``) for running scans,
|
||||
browsing reports, and managing the MikroTik blocklist / firewall.
|
||||
|
||||
Run directly: ``python -m app.web``
|
||||
Or via the entry point: ``python app.py --web`` (see ``serve()``).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from flask import Flask, jsonify, request
|
||||
|
||||
from ..utils.constants import (
|
||||
APP_REPORT_DATA_DIR,
|
||||
APP_REPORT_DIR,
|
||||
COPYPARTY_LOG_PATH,
|
||||
COPYPARTY_REPORT_DIR,
|
||||
DEFAULT_ERROR_RATIO,
|
||||
DEFAULT_MIN_DENIED,
|
||||
DEFAULT_MIN_ERRORS,
|
||||
DEFAULT_TOP,
|
||||
MIKROTIK_REPORT_DIR,
|
||||
NPM_LOG_PATH,
|
||||
NPM_REPORT_DIR,
|
||||
REPORT_DIR,
|
||||
WEB_HOST,
|
||||
WEB_PORT,
|
||||
)
|
||||
from ..firewall import MikrotikFirewall
|
||||
from ..scan import ScanParams
|
||||
from . import web_scan
|
||||
|
||||
#: report sources exposed by the API -> report directory
|
||||
REPORT_SOURCES = {
|
||||
"copyparty": COPYPARTY_REPORT_DIR,
|
||||
"npm": NPM_REPORT_DIR,
|
||||
"app": APP_REPORT_DIR,
|
||||
"mikrotik": MIKROTIK_REPORT_DIR,
|
||||
}
|
||||
|
||||
SCAN_KINDS = ("copyparty", "npm", "consolidated")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _count_lines(path: str) -> int | None:
|
||||
"""Return the line count of a file, or None if it cannot be read."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return sum(1 for _ in f)
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
|
||||
def _read_lines(path: str) -> list[str]:
|
||||
"""Return non-empty lines of a file, or [] if it cannot be read."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return [line.strip() for line in f if line.strip()]
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def _read_json(path: str, default: list | dict | None = None) -> list | dict:
|
||||
"""Read a JSON file, returning ``default`` (or []) on failure."""
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except (OSError, ValueError):
|
||||
return default if default is not None else []
|
||||
|
||||
|
||||
def _params_from_request() -> ScanParams:
|
||||
"""Build ScanParams from the JSON request body, falling back to defaults."""
|
||||
data = request.get_json(silent=True) or {}
|
||||
|
||||
def _pick(key: str, default):
|
||||
value = data.get(key)
|
||||
return default if value is None or value == "" else value
|
||||
|
||||
return ScanParams(
|
||||
copyparty_log_path=str(
|
||||
_pick("copyparty_log_path", COPYPARTY_LOG_PATH)),
|
||||
copyparty_report_dir=str(
|
||||
_pick("copyparty_report_dir", COPYPARTY_REPORT_DIR)),
|
||||
min_denied=int(_pick("min_denied", DEFAULT_MIN_DENIED)),
|
||||
npm_log_path=str(_pick("npm_log_path", NPM_LOG_PATH)),
|
||||
npm_report_dir=str(_pick("npm_report_dir", NPM_REPORT_DIR)),
|
||||
min_errors=int(_pick("min_errors", DEFAULT_MIN_ERRORS)),
|
||||
error_ratio=float(_pick("error_ratio", DEFAULT_ERROR_RATIO)),
|
||||
report_dir=str(_pick("report_dir", REPORT_DIR)),
|
||||
top=int(_pick("top", DEFAULT_TOP)),
|
||||
quiet=bool(data.get("quiet", False)),
|
||||
full=bool(data.get("full", False)),
|
||||
update_firewall=bool(data.get("update_firewall", False)),
|
||||
dry_run=bool(data.get("dry_run", False)),
|
||||
)
|
||||
|
||||
|
||||
def _with_router(fn):
|
||||
"""Connect to the router, run ``fn(fw)``, always disconnect.
|
||||
|
||||
Returns a JSON error response (502) if the router is unreachable.
|
||||
"""
|
||||
fw = MikrotikFirewall()
|
||||
try:
|
||||
fw.connect()
|
||||
return fn(fw)
|
||||
except Exception as e:
|
||||
return jsonify({"error": str(e)}), 502
|
||||
finally:
|
||||
fw.disconnect()
|
||||
|
||||
|
||||
def _sync_firewall(dry_run: bool):
|
||||
"""Sync the latest consolidated blocklist to the router (preview or apply)."""
|
||||
v4 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
|
||||
v6 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt"))
|
||||
|
||||
def _sync(fw):
|
||||
changes = fw.sync_blocklist(v4, v6, dry_run=dry_run)
|
||||
return jsonify({**changes, "dry_run": dry_run})
|
||||
|
||||
return _with_router(_sync)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# app factory
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def create_app() -> Flask:
|
||||
"""Build and configure the Flask application."""
|
||||
_app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
app = Flask(__name__,
|
||||
template_folder=os.path.join(_app_dir, "templates"),
|
||||
static_folder=os.path.join(_app_dir, "static"))
|
||||
|
||||
# Seed the in-memory job store from disk so the dashboard keeps showing
|
||||
# recent scans across server restarts.
|
||||
web_scan.load_persisted()
|
||||
|
||||
from .routes import register_routes
|
||||
|
||||
helpers = {
|
||||
"_count_lines": _count_lines,
|
||||
"_read_lines": _read_lines,
|
||||
"_read_json": _read_json,
|
||||
"_params_from_request": _params_from_request,
|
||||
"_with_router": _with_router,
|
||||
"_sync_firewall": _sync_firewall,
|
||||
}
|
||||
register_routes(app, REPORT_SOURCES, SCAN_KINDS, web_scan, helpers)
|
||||
return app
|
||||
|
||||
|
||||
def serve(host: str = WEB_HOST, port: int = WEB_PORT) -> None:
|
||||
"""Run the web server on localhost at the configured high port."""
|
||||
app = create_app()
|
||||
print(f"Intrusion Detection web UI running at http://{host}:{port}")
|
||||
app.run(host=host, port=port, use_reloader=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
serve()
|
||||
+277
-277
@@ -1,277 +1,277 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Route definitions for the intrusion-detection web UI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from flask import Flask, abort, jsonify, render_template, request
|
||||
|
||||
from ..utils.constants import (
|
||||
APP_REPORT_DATA_DIR,
|
||||
BLOCKLIST_NAME,
|
||||
MIKROTIK_DATA_DIR,
|
||||
WEB_HOST,
|
||||
WEB_PORT,
|
||||
WHITELIST_NAMES,
|
||||
)
|
||||
from ..utils.ip_utils import ip_version
|
||||
|
||||
|
||||
def register_routes(app: Flask,
|
||||
report_sources: dict,
|
||||
scan_kinds: tuple,
|
||||
web_scan,
|
||||
helpers) -> None:
|
||||
"""Register all web UI routes on the Flask application."""
|
||||
|
||||
_count_lines = helpers["_count_lines"]
|
||||
_read_lines = helpers["_read_lines"]
|
||||
_read_json = helpers["_read_json"]
|
||||
_params_from_request = helpers["_params_from_request"]
|
||||
_with_router = helpers["_with_router"]
|
||||
_sync_firewall = helpers["_sync_firewall"]
|
||||
|
||||
# --- pages -------------------------------------------------------------
|
||||
|
||||
@app.get("/")
|
||||
def index():
|
||||
return render_template("dashboard.html")
|
||||
|
||||
@app.get("/scan")
|
||||
def scan_page():
|
||||
return render_template("scan.html")
|
||||
|
||||
@app.get("/reports")
|
||||
def reports_page():
|
||||
return render_template("reports.html")
|
||||
|
||||
@app.get("/blocklist")
|
||||
def blocklist_page():
|
||||
return render_template("blocklist.html")
|
||||
|
||||
@app.get("/firewall")
|
||||
def firewall_page():
|
||||
return render_template("firewall.html")
|
||||
|
||||
@app.get("/whitelist")
|
||||
def whitelist_page():
|
||||
return render_template("whitelist.html", whitelist_names=WHITELIST_NAMES)
|
||||
|
||||
@app.get("/graph")
|
||||
def graph_page():
|
||||
return render_template("graph.html")
|
||||
|
||||
# --- status ------------------------------------------------------------
|
||||
|
||||
@app.get("/api/status")
|
||||
def status():
|
||||
v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
|
||||
v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
|
||||
return jsonify({
|
||||
"host": WEB_HOST,
|
||||
"port": WEB_PORT,
|
||||
"scan_running": web_scan.is_scan_running(),
|
||||
"latest_scans": web_scan.latest_scans(5),
|
||||
"blocklist_ipv4_count": _count_lines(v4_path),
|
||||
"blocklist_ipv6_count": _count_lines(v6_path),
|
||||
"firewall_snapshot": {
|
||||
"exists": os.path.isdir(MIKROTIK_DATA_DIR),
|
||||
"path": MIKROTIK_DATA_DIR,
|
||||
},
|
||||
})
|
||||
|
||||
# --- scans -------------------------------------------------------------
|
||||
|
||||
@app.post("/api/scan/<kind>")
|
||||
def start_scan(kind):
|
||||
if kind not in scan_kinds:
|
||||
abort(404, f"unknown scan kind: {kind}")
|
||||
job = web_scan.start_scan(kind, _params_from_request())
|
||||
if job.get("error"):
|
||||
return jsonify(job), 409
|
||||
return jsonify(job), 202
|
||||
|
||||
@app.get("/api/scan/status")
|
||||
def scan_status():
|
||||
return jsonify({
|
||||
"running": web_scan.is_scan_running(),
|
||||
"latest_scans": web_scan.latest_scans(10),
|
||||
})
|
||||
|
||||
@app.get("/api/scan/<job_id>")
|
||||
def scan_job(job_id):
|
||||
job = web_scan.get_scan(job_id)
|
||||
if job is None:
|
||||
abort(404, f"unknown job: {job_id}")
|
||||
return jsonify(job)
|
||||
|
||||
# --- reports -----------------------------------------------------------
|
||||
|
||||
@app.get("/api/reports/<source>")
|
||||
def list_reports(source):
|
||||
base = report_sources.get(source)
|
||||
if base is None:
|
||||
abort(404, f"unknown report source: {source}")
|
||||
names = sorted(
|
||||
n for n in os.listdir(base)
|
||||
if os.path.isfile(os.path.join(base, n)))
|
||||
return jsonify({"source": source, "files": names})
|
||||
|
||||
@app.get("/api/reports/<source>/<path:name>")
|
||||
def get_report(source, name):
|
||||
base = report_sources.get(source)
|
||||
if base is None:
|
||||
abort(404, f"unknown report source: {source}")
|
||||
full = os.path.realpath(os.path.join(base, name))
|
||||
if not full.startswith(os.path.realpath(base)):
|
||||
abort(403, "invalid report path")
|
||||
if not os.path.isfile(full):
|
||||
abort(404, "report not found")
|
||||
with open(full, "r", encoding="utf-8", errors="replace") as f:
|
||||
content = f.read()
|
||||
return jsonify({"source": source, "name": name, "content": content})
|
||||
|
||||
# --- blocklist ---------------------------------------------------------
|
||||
|
||||
@app.get("/api/blocklist")
|
||||
def blocklist():
|
||||
return jsonify({
|
||||
"ipv4": _read_lines(
|
||||
os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")),
|
||||
"ipv6": _read_lines(
|
||||
os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")),
|
||||
})
|
||||
|
||||
@app.post("/api/blocklist/add")
|
||||
def blocklist_add():
|
||||
data = request.get_json(silent=True) or {}
|
||||
ip = (data.get("ip") or "").strip()
|
||||
version = ip_version(ip)
|
||||
if not ip or version not in (4, 6):
|
||||
return jsonify({"error": "missing or invalid 'ip' field"}), 400
|
||||
|
||||
def _add(fw):
|
||||
if version == 4:
|
||||
fw.write_address_list_ipv4(BLOCKLIST_NAME, [ip])
|
||||
else:
|
||||
fw.write_address_list_ipv6(BLOCKLIST_NAME, [ip])
|
||||
return jsonify({"ok": True, "action": "add", "ip": ip})
|
||||
|
||||
return _with_router(_add)
|
||||
|
||||
@app.post("/api/blocklist/remove")
|
||||
def blocklist_remove():
|
||||
data = request.get_json(silent=True) or {}
|
||||
ip = (data.get("ip") or "").strip()
|
||||
version = ip_version(ip)
|
||||
if not ip or version not in (4, 6):
|
||||
return jsonify({"error": "missing or invalid 'ip' field"}), 400
|
||||
|
||||
def _remove(fw):
|
||||
path = "/ip" if version == 4 else "/ipv6"
|
||||
fw.execute_command(
|
||||
f"{path} firewall address-list remove "
|
||||
f"[find list={BLOCKLIST_NAME} address={ip}]")
|
||||
return jsonify({"ok": True, "action": "remove", "ip": ip})
|
||||
|
||||
return _with_router(_remove)
|
||||
|
||||
# --- graph -------------------------------------------------------------
|
||||
|
||||
@app.get("/api/graph/ipv4")
|
||||
def graph_ipv4():
|
||||
blocked = _read_lines(
|
||||
os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
|
||||
addr_v4 = _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
|
||||
whitelisted = sorted({e.get("address") for e in addr_v4
|
||||
if e.get("list") in WHITELIST_NAMES
|
||||
and e.get("address")})
|
||||
return jsonify({
|
||||
"blocked": blocked,
|
||||
"whitelisted": whitelisted,
|
||||
"counts": {
|
||||
"blocked": len(blocked),
|
||||
"whitelisted": len(whitelisted),
|
||||
},
|
||||
"blocklist_name": BLOCKLIST_NAME,
|
||||
"whitelist_names": WHITELIST_NAMES,
|
||||
})
|
||||
|
||||
# --- firewall ----------------------------------------------------------
|
||||
|
||||
@app.get("/api/firewall/state")
|
||||
def firewall_state():
|
||||
return jsonify({
|
||||
"address_lists": {
|
||||
"ipv4": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json")),
|
||||
"ipv6": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json")),
|
||||
},
|
||||
"firewall_rules": {
|
||||
"ipv4": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv4.json")),
|
||||
"ipv6": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv6.json")),
|
||||
},
|
||||
"snapshot_dir": MIKROTIK_DATA_DIR,
|
||||
})
|
||||
|
||||
@app.post("/api/firewall/sync")
|
||||
def firewall_sync():
|
||||
return _sync_firewall(dry_run=True)
|
||||
|
||||
@app.post("/api/firewall/apply")
|
||||
def firewall_apply():
|
||||
return _sync_firewall(dry_run=False)
|
||||
|
||||
# --- whitelist ---------------------------------------------------------
|
||||
|
||||
@app.get("/api/whitelist")
|
||||
def whitelist():
|
||||
addr_v4 = _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
|
||||
addr_v6 = _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json"))
|
||||
result: dict[str, dict[str, list[str]]] = {}
|
||||
for name in WHITELIST_NAMES:
|
||||
result[name] = {
|
||||
"ipv4": sorted(e.get("address") for e in addr_v4
|
||||
if e.get("list") == name),
|
||||
"ipv6": sorted(e.get("address") for e in addr_v6
|
||||
if e.get("list") == name),
|
||||
}
|
||||
return jsonify({"names": WHITELIST_NAMES, "whitelists": result})
|
||||
|
||||
@app.post("/api/whitelist/update")
|
||||
def whitelist_update():
|
||||
data = request.get_json(silent=True) or {}
|
||||
name = (data.get("name") or "").strip()
|
||||
address = (data.get("address") or "").strip()
|
||||
action = (data.get("action") or "add").strip().lower()
|
||||
version = ip_version(address)
|
||||
|
||||
if name not in WHITELIST_NAMES:
|
||||
return jsonify({"error": f"unknown whitelist: {name}"}), 400
|
||||
if not address or version not in (4, 6):
|
||||
return jsonify({"error": "missing or invalid 'address' field"}), 400
|
||||
if action not in ("add", "remove"):
|
||||
return jsonify({"error": "action must be 'add' or 'remove'"}), 400
|
||||
|
||||
def _update(fw):
|
||||
path = "/ip" if version == 4 else "/ipv6"
|
||||
if action == "add":
|
||||
if version == 4:
|
||||
fw.write_address_list_ipv4(name, [address])
|
||||
else:
|
||||
fw.write_address_list_ipv6(name, [address])
|
||||
else:
|
||||
fw.execute_command(
|
||||
f"{path} firewall address-list remove "
|
||||
f"[find list={name} address={address}]")
|
||||
return jsonify({"ok": True, "action": action,
|
||||
"name": name, "address": address})
|
||||
|
||||
return _with_router(_update)
|
||||
#!/usr/bin/env python3
|
||||
"""Route definitions for the intrusion-detection web UI."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
from flask import Flask, abort, jsonify, render_template, request
|
||||
|
||||
from ..utils.constants import (
|
||||
APP_REPORT_DATA_DIR,
|
||||
BLOCKLIST_NAME,
|
||||
MIKROTIK_DATA_DIR,
|
||||
WEB_HOST,
|
||||
WEB_PORT,
|
||||
WHITELIST_NAMES,
|
||||
)
|
||||
from ..utils.ip_utils import ip_version
|
||||
|
||||
|
||||
def register_routes(app: Flask,
|
||||
report_sources: dict,
|
||||
scan_kinds: tuple,
|
||||
web_scan,
|
||||
helpers) -> None:
|
||||
"""Register all web UI routes on the Flask application."""
|
||||
|
||||
_count_lines = helpers["_count_lines"]
|
||||
_read_lines = helpers["_read_lines"]
|
||||
_read_json = helpers["_read_json"]
|
||||
_params_from_request = helpers["_params_from_request"]
|
||||
_with_router = helpers["_with_router"]
|
||||
_sync_firewall = helpers["_sync_firewall"]
|
||||
|
||||
# --- pages -------------------------------------------------------------
|
||||
|
||||
@app.get("/")
|
||||
def index():
|
||||
return render_template("dashboard.html")
|
||||
|
||||
@app.get("/scan")
|
||||
def scan_page():
|
||||
return render_template("scan.html")
|
||||
|
||||
@app.get("/reports")
|
||||
def reports_page():
|
||||
return render_template("reports.html")
|
||||
|
||||
@app.get("/blocklist")
|
||||
def blocklist_page():
|
||||
return render_template("blocklist.html")
|
||||
|
||||
@app.get("/firewall")
|
||||
def firewall_page():
|
||||
return render_template("firewall.html")
|
||||
|
||||
@app.get("/whitelist")
|
||||
def whitelist_page():
|
||||
return render_template("whitelist.html", whitelist_names=WHITELIST_NAMES)
|
||||
|
||||
@app.get("/graph")
|
||||
def graph_page():
|
||||
return render_template("graph.html")
|
||||
|
||||
# --- status ------------------------------------------------------------
|
||||
|
||||
@app.get("/api/status")
|
||||
def status():
|
||||
v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
|
||||
v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
|
||||
return jsonify({
|
||||
"host": WEB_HOST,
|
||||
"port": WEB_PORT,
|
||||
"scan_running": web_scan.is_scan_running(),
|
||||
"latest_scans": web_scan.latest_scans(5),
|
||||
"blocklist_ipv4_count": _count_lines(v4_path),
|
||||
"blocklist_ipv6_count": _count_lines(v6_path),
|
||||
"firewall_snapshot": {
|
||||
"exists": os.path.isdir(MIKROTIK_DATA_DIR),
|
||||
"path": MIKROTIK_DATA_DIR,
|
||||
},
|
||||
})
|
||||
|
||||
# --- scans -------------------------------------------------------------
|
||||
|
||||
@app.post("/api/scan/<kind>")
|
||||
def start_scan(kind):
|
||||
if kind not in scan_kinds:
|
||||
abort(404, f"unknown scan kind: {kind}")
|
||||
job = web_scan.start_scan(kind, _params_from_request())
|
||||
if job.get("error"):
|
||||
return jsonify(job), 409
|
||||
return jsonify(job), 202
|
||||
|
||||
@app.get("/api/scan/status")
|
||||
def scan_status():
|
||||
return jsonify({
|
||||
"running": web_scan.is_scan_running(),
|
||||
"latest_scans": web_scan.latest_scans(10),
|
||||
})
|
||||
|
||||
@app.get("/api/scan/<job_id>")
|
||||
def scan_job(job_id):
|
||||
job = web_scan.get_scan(job_id)
|
||||
if job is None:
|
||||
abort(404, f"unknown job: {job_id}")
|
||||
return jsonify(job)
|
||||
|
||||
# --- reports -----------------------------------------------------------
|
||||
|
||||
@app.get("/api/reports/<source>")
|
||||
def list_reports(source):
|
||||
base = report_sources.get(source)
|
||||
if base is None:
|
||||
abort(404, f"unknown report source: {source}")
|
||||
names = sorted(
|
||||
n for n in os.listdir(base)
|
||||
if os.path.isfile(os.path.join(base, n)))
|
||||
return jsonify({"source": source, "files": names})
|
||||
|
||||
@app.get("/api/reports/<source>/<path:name>")
|
||||
def get_report(source, name):
|
||||
base = report_sources.get(source)
|
||||
if base is None:
|
||||
abort(404, f"unknown report source: {source}")
|
||||
full = os.path.realpath(os.path.join(base, name))
|
||||
if not full.startswith(os.path.realpath(base)):
|
||||
abort(403, "invalid report path")
|
||||
if not os.path.isfile(full):
|
||||
abort(404, "report not found")
|
||||
with open(full, "r", encoding="utf-8", errors="replace") as f:
|
||||
content = f.read()
|
||||
return jsonify({"source": source, "name": name, "content": content})
|
||||
|
||||
# --- blocklist ---------------------------------------------------------
|
||||
|
||||
@app.get("/api/blocklist")
|
||||
def blocklist():
|
||||
return jsonify({
|
||||
"ipv4": _read_lines(
|
||||
os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")),
|
||||
"ipv6": _read_lines(
|
||||
os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")),
|
||||
})
|
||||
|
||||
@app.post("/api/blocklist/add")
|
||||
def blocklist_add():
|
||||
data = request.get_json(silent=True) or {}
|
||||
ip = (data.get("ip") or "").strip()
|
||||
version = ip_version(ip)
|
||||
if not ip or version not in (4, 6):
|
||||
return jsonify({"error": "missing or invalid 'ip' field"}), 400
|
||||
|
||||
def _add(fw):
|
||||
if version == 4:
|
||||
fw.write_address_list_ipv4(BLOCKLIST_NAME, [ip])
|
||||
else:
|
||||
fw.write_address_list_ipv6(BLOCKLIST_NAME, [ip])
|
||||
return jsonify({"ok": True, "action": "add", "ip": ip})
|
||||
|
||||
return _with_router(_add)
|
||||
|
||||
@app.post("/api/blocklist/remove")
|
||||
def blocklist_remove():
|
||||
data = request.get_json(silent=True) or {}
|
||||
ip = (data.get("ip") or "").strip()
|
||||
version = ip_version(ip)
|
||||
if not ip or version not in (4, 6):
|
||||
return jsonify({"error": "missing or invalid 'ip' field"}), 400
|
||||
|
||||
def _remove(fw):
|
||||
path = "/ip" if version == 4 else "/ipv6"
|
||||
fw.execute_command(
|
||||
f"{path} firewall address-list remove "
|
||||
f"[find list={BLOCKLIST_NAME} address={ip}]")
|
||||
return jsonify({"ok": True, "action": "remove", "ip": ip})
|
||||
|
||||
return _with_router(_remove)
|
||||
|
||||
# --- graph -------------------------------------------------------------
|
||||
|
||||
@app.get("/api/graph/ipv4")
|
||||
def graph_ipv4():
|
||||
blocked = _read_lines(
|
||||
os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
|
||||
addr_v4 = _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
|
||||
whitelisted = sorted({e.get("address") for e in addr_v4
|
||||
if e.get("list") in WHITELIST_NAMES
|
||||
and e.get("address")})
|
||||
return jsonify({
|
||||
"blocked": blocked,
|
||||
"whitelisted": whitelisted,
|
||||
"counts": {
|
||||
"blocked": len(blocked),
|
||||
"whitelisted": len(whitelisted),
|
||||
},
|
||||
"blocklist_name": BLOCKLIST_NAME,
|
||||
"whitelist_names": WHITELIST_NAMES,
|
||||
})
|
||||
|
||||
# --- firewall ----------------------------------------------------------
|
||||
|
||||
@app.get("/api/firewall/state")
|
||||
def firewall_state():
|
||||
return jsonify({
|
||||
"address_lists": {
|
||||
"ipv4": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json")),
|
||||
"ipv6": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json")),
|
||||
},
|
||||
"firewall_rules": {
|
||||
"ipv4": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv4.json")),
|
||||
"ipv6": _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv6.json")),
|
||||
},
|
||||
"snapshot_dir": MIKROTIK_DATA_DIR,
|
||||
})
|
||||
|
||||
@app.post("/api/firewall/sync")
|
||||
def firewall_sync():
|
||||
return _sync_firewall(dry_run=True)
|
||||
|
||||
@app.post("/api/firewall/apply")
|
||||
def firewall_apply():
|
||||
return _sync_firewall(dry_run=False)
|
||||
|
||||
# --- whitelist ---------------------------------------------------------
|
||||
|
||||
@app.get("/api/whitelist")
|
||||
def whitelist():
|
||||
addr_v4 = _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
|
||||
addr_v6 = _read_json(
|
||||
os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json"))
|
||||
result: dict[str, dict[str, list[str]]] = {}
|
||||
for name in WHITELIST_NAMES:
|
||||
result[name] = {
|
||||
"ipv4": sorted(e.get("address") for e in addr_v4
|
||||
if e.get("list") == name),
|
||||
"ipv6": sorted(e.get("address") for e in addr_v6
|
||||
if e.get("list") == name),
|
||||
}
|
||||
return jsonify({"names": WHITELIST_NAMES, "whitelists": result})
|
||||
|
||||
@app.post("/api/whitelist/update")
|
||||
def whitelist_update():
|
||||
data = request.get_json(silent=True) or {}
|
||||
name = (data.get("name") or "").strip()
|
||||
address = (data.get("address") or "").strip()
|
||||
action = (data.get("action") or "add").strip().lower()
|
||||
version = ip_version(address)
|
||||
|
||||
if name not in WHITELIST_NAMES:
|
||||
return jsonify({"error": f"unknown whitelist: {name}"}), 400
|
||||
if not address or version not in (4, 6):
|
||||
return jsonify({"error": "missing or invalid 'address' field"}), 400
|
||||
if action not in ("add", "remove"):
|
||||
return jsonify({"error": "action must be 'add' or 'remove'"}), 400
|
||||
|
||||
def _update(fw):
|
||||
path = "/ip" if version == 4 else "/ipv6"
|
||||
if action == "add":
|
||||
if version == 4:
|
||||
fw.write_address_list_ipv4(name, [address])
|
||||
else:
|
||||
fw.write_address_list_ipv6(name, [address])
|
||||
else:
|
||||
fw.execute_command(
|
||||
f"{path} firewall address-list remove "
|
||||
f"[find list={name} address={address}]")
|
||||
return jsonify({"ok": True, "action": action,
|
||||
"name": name, "address": address})
|
||||
|
||||
return _with_router(_update)
|
||||
+360
-360
@@ -1,360 +1,360 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Background scan runner for the web UI.
|
||||
|
||||
Runs scans in a daemon thread so HTTP requests never block, keeps an
|
||||
in-memory job store, and serializes scans with a busy lock (WHOIS and
|
||||
whitelist refresh must not run concurrently).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from ..utils.constants import (
|
||||
APP_REPORT_DATA_DIR,
|
||||
COPYPARTY_REPORT_DIR,
|
||||
MIKROTIK_REPORT_DIR,
|
||||
NPM_REPORT_DIR,
|
||||
SCAN_JOBS_MAX,
|
||||
SCAN_JOBS_PATH,
|
||||
)
|
||||
from ..copyparty import CopypartyAnalyzer
|
||||
from ..npm import NpmAnalyzer
|
||||
from ..scan import ScanParams, ScanResult, run_scan
|
||||
|
||||
ScanKind = Literal["copyparty", "npm", "consolidated"]
|
||||
|
||||
#: serializes scans — only one runs at a time
|
||||
_scan_lock = threading.Lock()
|
||||
|
||||
#: in-memory job store (job_id -> ScanJob)
|
||||
_jobs: dict[str, "ScanJob"] = {}
|
||||
_jobs_lock = threading.Lock()
|
||||
|
||||
#: True once persisted jobs have been loaded from disk
|
||||
_loaded = False
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanJob:
|
||||
"""One background scan job."""
|
||||
|
||||
job_id: str
|
||||
kind: ScanKind
|
||||
params: ScanParams
|
||||
state: str = "queued" # queued | running | done | error
|
||||
started: str | None = None
|
||||
finished: str | None = None
|
||||
result: ScanResult | None = None
|
||||
error: str | None = None
|
||||
busy: bool = False
|
||||
details: dict = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = self.result
|
||||
return {
|
||||
"job_id": self.job_id,
|
||||
"kind": self.kind,
|
||||
"params": asdict(self.params),
|
||||
"state": self.state,
|
||||
"started": self.started,
|
||||
"finished": self.finished,
|
||||
"error": self.error,
|
||||
"busy": self.busy,
|
||||
"details": self.details,
|
||||
"result": None if result is None else {
|
||||
"ok": result.ok,
|
||||
"cp_exists": result.cp_exists,
|
||||
"npm_exists": result.npm_exists,
|
||||
"cp_candidates": result.cp_candidates,
|
||||
"npm_candidates": result.npm_candidates,
|
||||
"blocked_ips": result.blocked_ips,
|
||||
"blocked_targets": result.blocked_targets,
|
||||
"blocked_ipv4": result.blocked_ipv4,
|
||||
"blocked_ipv6": result.blocked_ipv6,
|
||||
"blocklist_v4_path": result.blocklist_v4_path,
|
||||
"blocklist_v6_path": result.blocklist_v6_path,
|
||||
"firewall_connected": result.firewall_connected,
|
||||
"firewall_changes": result.firewall_changes,
|
||||
"errors": result.errors,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def start_scan(kind: ScanKind, params: ScanParams) -> dict[str, Any]:
|
||||
"""Start a scan in the background and return its job dict.
|
||||
|
||||
If another scan is already running, returns a dict with an ``error`` key.
|
||||
"""
|
||||
if not _scan_lock.acquire(blocking=False):
|
||||
return {"error": "a scan is already running; try again when it finishes"}
|
||||
|
||||
job = ScanJob(job_id=uuid.uuid4().hex[:12], kind=kind,
|
||||
params=params, busy=True)
|
||||
with _jobs_lock:
|
||||
_jobs[job.job_id] = job
|
||||
_save_jobs()
|
||||
|
||||
def _run() -> None:
|
||||
job.started = _now()
|
||||
job.state = "running"
|
||||
try:
|
||||
if kind == "consolidated":
|
||||
job.result = run_scan(params)
|
||||
elif kind == "copyparty":
|
||||
info = CopypartyAnalyzer(
|
||||
log_path=params.copyparty_log_path,
|
||||
report_dir=params.copyparty_report_dir,
|
||||
top=params.top,
|
||||
quiet=params.quiet,
|
||||
full=params.full,
|
||||
min_denied=params.min_denied,
|
||||
).run()
|
||||
job.result = ScanResult(ok=True)
|
||||
job.details = {"report": info.get("report"),
|
||||
"data_dir": info.get("data_dir")}
|
||||
elif kind == "npm":
|
||||
info = NpmAnalyzer(
|
||||
log_path=params.npm_log_path,
|
||||
report_dir=params.npm_report_dir,
|
||||
top=params.top,
|
||||
quiet=params.quiet,
|
||||
full=params.full,
|
||||
min_errors=params.min_errors,
|
||||
error_ratio=params.error_ratio,
|
||||
).run()
|
||||
job.result = ScanResult(ok=True)
|
||||
job.details = {"report": info.get("report"),
|
||||
"data_dir": info.get("data_dir")}
|
||||
else: # pragma: no cover - guarded by type hint
|
||||
raise ValueError(f"unknown scan kind: {kind}")
|
||||
job.state = "done"
|
||||
except Exception as e:
|
||||
job.error = str(e)
|
||||
job.state = "error"
|
||||
finally:
|
||||
job.finished = _now()
|
||||
job.busy = False
|
||||
_scan_lock.release()
|
||||
_save_jobs()
|
||||
|
||||
threading.Thread(target=_run, daemon=True,
|
||||
name=f"scan-{job.job_id}").start()
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
def get_scan(job_id: str) -> dict[str, Any] | None:
|
||||
"""Return a single job as a dict, or None if unknown."""
|
||||
load_persisted()
|
||||
with _jobs_lock:
|
||||
job = _jobs.get(job_id)
|
||||
return job.to_dict() if job else None
|
||||
|
||||
|
||||
def _iso_key(value: str | None) -> float:
|
||||
"""ISO-8601 string -> epoch seconds (0.0 when unparseable/empty)."""
|
||||
if not value:
|
||||
return 0.0
|
||||
try:
|
||||
return datetime.fromisoformat(value).timestamp()
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def latest_file_scan() -> dict[str, Any] | None:
|
||||
"""Build a ``done`` scan entry from the newest report files on disk.
|
||||
|
||||
Lets the dashboard show the latest completed scan even when it was not
|
||||
run through the web UI (server restart, CLI-run scans). Returns None when
|
||||
no scan outputs exist yet.
|
||||
"""
|
||||
|
||||
def newest(dirpath: str, suffix: str = "") -> str | None:
|
||||
"""Newest file in ``dirpath`` (optionally ending in ``suffix``)."""
|
||||
try:
|
||||
names = [n for n in os.listdir(dirpath)
|
||||
if os.path.isfile(os.path.join(dirpath, n))
|
||||
and (not suffix or n.endswith(suffix))]
|
||||
except OSError:
|
||||
return None
|
||||
return os.path.join(dirpath, max(names)) if names else None
|
||||
|
||||
mikrotik = newest(MIKROTIK_REPORT_DIR, ".json")
|
||||
cp_report = newest(COPYPARTY_REPORT_DIR, ".txt")
|
||||
npm_report = newest(NPM_REPORT_DIR, ".txt")
|
||||
v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
|
||||
v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
|
||||
have_outputs = (mikrotik is not None or cp_report is not None
|
||||
or npm_report is not None
|
||||
or os.path.exists(v4_path) or os.path.exists(v6_path))
|
||||
if not have_outputs:
|
||||
return None
|
||||
|
||||
# Timestamp: prefer the newest snapshot's collected_at, else the newest
|
||||
# report file's mtime, else now.
|
||||
timestamp: str | None = None
|
||||
if mikrotik:
|
||||
try:
|
||||
with open(mikrotik, "r", encoding="utf-8") as f:
|
||||
timestamp = json.load(f).get("collected_at")
|
||||
except (OSError, ValueError):
|
||||
timestamp = None
|
||||
if not timestamp:
|
||||
candidates = [p for p in (mikrotik, cp_report, npm_report,
|
||||
v4_path, v6_path)
|
||||
if p and os.path.exists(p)]
|
||||
if candidates:
|
||||
tz = datetime.now().astimezone().tzinfo
|
||||
timestamp = datetime.fromtimestamp(
|
||||
max(os.path.getmtime(p) for p in candidates),
|
||||
tz=tz).isoformat(timespec="seconds")
|
||||
if not timestamp:
|
||||
timestamp = _now()
|
||||
|
||||
blocked_targets = 0
|
||||
for path in (v4_path, v6_path):
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
blocked_targets += sum(1 for _ in f)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return {
|
||||
"job_id": "files-latest",
|
||||
"kind": "consolidated",
|
||||
"state": "done",
|
||||
"started": timestamp,
|
||||
"finished": timestamp,
|
||||
"error": None,
|
||||
"busy": False,
|
||||
"details": {},
|
||||
"result": {
|
||||
"ok": True,
|
||||
"cp_exists": cp_report is not None,
|
||||
"npm_exists": npm_report is not None,
|
||||
"cp_candidates": 0,
|
||||
"npm_candidates": 0,
|
||||
"blocked_ips": blocked_targets,
|
||||
"blocked_targets": blocked_targets,
|
||||
"blocked_ipv4": [],
|
||||
"blocked_ipv6": [],
|
||||
"blocklist_v4_path": v4_path,
|
||||
"blocklist_v6_path": v6_path,
|
||||
"firewall_connected": mikrotik is not None,
|
||||
"firewall_changes": None,
|
||||
"errors": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
|
||||
"""Return the most recently started jobs, newest first.
|
||||
|
||||
Appends a file-derived "latest scan" entry when the newest report files on
|
||||
disk reflect a scan newer than any known job (e.g. after a restart or a
|
||||
CLI run), so the dashboard always shows the latest completed scan.
|
||||
"""
|
||||
load_persisted()
|
||||
with _jobs_lock:
|
||||
jobs = sorted(_jobs.values(),
|
||||
key=lambda j: j.started or "", reverse=True)
|
||||
entries = [j.to_dict() for j in jobs]
|
||||
file_entry = latest_file_scan()
|
||||
if file_entry is not None:
|
||||
newest = jobs[0] if jobs else None
|
||||
newest_ts = _iso_key(
|
||||
(newest.finished or newest.started) if newest else None)
|
||||
# Only surface the file-derived entry when it is genuinely newer than
|
||||
# every known job (tolerance avoids duplicating the run that produced
|
||||
# it, where collected_at lands a moment after the job finished).
|
||||
if newest_ts == 0 or _iso_key(file_entry["started"]) - newest_ts > 2.0:
|
||||
entries.append(file_entry)
|
||||
entries.sort(key=lambda e: _iso_key(e.get("started")), reverse=True)
|
||||
return entries[:limit]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _save_jobs() -> None:
|
||||
"""Persist the newest jobs to ``SCAN_JOBS_PATH`` (atomic tmp+rename)."""
|
||||
with _jobs_lock:
|
||||
jobs = sorted(_jobs.values(),
|
||||
key=lambda j: j.started or "", reverse=True)
|
||||
payload = [j.to_dict() for j in jobs[:SCAN_JOBS_MAX]]
|
||||
os.makedirs(os.path.dirname(SCAN_JOBS_PATH), exist_ok=True)
|
||||
tmp = SCAN_JOBS_PATH + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, indent=2, sort_keys=True)
|
||||
os.replace(tmp, SCAN_JOBS_PATH)
|
||||
|
||||
|
||||
def _load_jobs() -> None:
|
||||
"""Rebuild the in-memory job store from ``SCAN_JOBS_PATH``."""
|
||||
try:
|
||||
with open(SCAN_JOBS_PATH, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
except (OSError, ValueError, TypeError):
|
||||
payload = []
|
||||
if not isinstance(payload, list):
|
||||
return
|
||||
|
||||
jobs: dict[str, ScanJob] = {}
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
try:
|
||||
params = ScanParams(**item.get("params", {}))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
try:
|
||||
job = ScanJob(
|
||||
job_id=str(item["job_id"]),
|
||||
kind=item.get("kind", "consolidated"),
|
||||
params=params,
|
||||
state=item.get("state", "done"),
|
||||
started=item.get("started"),
|
||||
finished=item.get("finished"),
|
||||
error=item.get("error"),
|
||||
busy=False,
|
||||
details=item.get("details", {}) or {},
|
||||
)
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
result = item.get("result")
|
||||
if isinstance(result, dict):
|
||||
try:
|
||||
job.result = ScanResult(**result)
|
||||
except TypeError:
|
||||
job.result = None
|
||||
jobs[job.job_id] = job
|
||||
|
||||
with _jobs_lock:
|
||||
_jobs.clear()
|
||||
_jobs.update(jobs)
|
||||
|
||||
|
||||
def load_persisted() -> None:
|
||||
"""Load persisted jobs from disk once (idempotent)."""
|
||||
global _loaded
|
||||
if _loaded:
|
||||
return
|
||||
_load_jobs()
|
||||
_loaded = True
|
||||
|
||||
|
||||
def is_scan_running() -> bool:
|
||||
"""True if any job currently holds the scan lock."""
|
||||
return _scan_lock.locked()
|
||||
#!/usr/bin/env python3
|
||||
"""Background scan runner for the web UI.
|
||||
|
||||
Runs scans in a daemon thread so HTTP requests never block, keeps an
|
||||
in-memory job store, and serializes scans with a busy lock (WHOIS and
|
||||
whitelist refresh must not run concurrently).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from datetime import datetime
|
||||
from typing import Any, Literal
|
||||
|
||||
from ..utils.constants import (
|
||||
APP_REPORT_DATA_DIR,
|
||||
COPYPARTY_REPORT_DIR,
|
||||
MIKROTIK_REPORT_DIR,
|
||||
NPM_REPORT_DIR,
|
||||
SCAN_JOBS_MAX,
|
||||
SCAN_JOBS_PATH,
|
||||
)
|
||||
from ..copyparty import CopypartyAnalyzer
|
||||
from ..npm import NpmAnalyzer
|
||||
from ..scan import ScanParams, ScanResult, run_scan
|
||||
|
||||
ScanKind = Literal["copyparty", "npm", "consolidated"]
|
||||
|
||||
#: serializes scans — only one runs at a time
|
||||
_scan_lock = threading.Lock()
|
||||
|
||||
#: in-memory job store (job_id -> ScanJob)
|
||||
_jobs: dict[str, "ScanJob"] = {}
|
||||
_jobs_lock = threading.Lock()
|
||||
|
||||
#: True once persisted jobs have been loaded from disk
|
||||
_loaded = False
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now().astimezone().isoformat(timespec="seconds")
|
||||
|
||||
|
||||
@dataclass
|
||||
class ScanJob:
|
||||
"""One background scan job."""
|
||||
|
||||
job_id: str
|
||||
kind: ScanKind
|
||||
params: ScanParams
|
||||
state: str = "queued" # queued | running | done | error
|
||||
started: str | None = None
|
||||
finished: str | None = None
|
||||
result: ScanResult | None = None
|
||||
error: str | None = None
|
||||
busy: bool = False
|
||||
details: dict = field(default_factory=dict)
|
||||
|
||||
def to_dict(self) -> dict[str, Any]:
|
||||
result = self.result
|
||||
return {
|
||||
"job_id": self.job_id,
|
||||
"kind": self.kind,
|
||||
"params": asdict(self.params),
|
||||
"state": self.state,
|
||||
"started": self.started,
|
||||
"finished": self.finished,
|
||||
"error": self.error,
|
||||
"busy": self.busy,
|
||||
"details": self.details,
|
||||
"result": None if result is None else {
|
||||
"ok": result.ok,
|
||||
"cp_exists": result.cp_exists,
|
||||
"npm_exists": result.npm_exists,
|
||||
"cp_candidates": result.cp_candidates,
|
||||
"npm_candidates": result.npm_candidates,
|
||||
"blocked_ips": result.blocked_ips,
|
||||
"blocked_targets": result.blocked_targets,
|
||||
"blocked_ipv4": result.blocked_ipv4,
|
||||
"blocked_ipv6": result.blocked_ipv6,
|
||||
"blocklist_v4_path": result.blocklist_v4_path,
|
||||
"blocklist_v6_path": result.blocklist_v6_path,
|
||||
"firewall_connected": result.firewall_connected,
|
||||
"firewall_changes": result.firewall_changes,
|
||||
"errors": result.errors,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def start_scan(kind: ScanKind, params: ScanParams) -> dict[str, Any]:
|
||||
"""Start a scan in the background and return its job dict.
|
||||
|
||||
If another scan is already running, returns a dict with an ``error`` key.
|
||||
"""
|
||||
if not _scan_lock.acquire(blocking=False):
|
||||
return {"error": "a scan is already running; try again when it finishes"}
|
||||
|
||||
job = ScanJob(job_id=uuid.uuid4().hex[:12], kind=kind,
|
||||
params=params, busy=True)
|
||||
with _jobs_lock:
|
||||
_jobs[job.job_id] = job
|
||||
_save_jobs()
|
||||
|
||||
def _run() -> None:
|
||||
job.started = _now()
|
||||
job.state = "running"
|
||||
try:
|
||||
if kind == "consolidated":
|
||||
job.result = run_scan(params)
|
||||
elif kind == "copyparty":
|
||||
info = CopypartyAnalyzer(
|
||||
log_path=params.copyparty_log_path,
|
||||
report_dir=params.copyparty_report_dir,
|
||||
top=params.top,
|
||||
quiet=params.quiet,
|
||||
full=params.full,
|
||||
min_denied=params.min_denied,
|
||||
).run()
|
||||
job.result = ScanResult(ok=True)
|
||||
job.details = {"report": info.get("report"),
|
||||
"data_dir": info.get("data_dir")}
|
||||
elif kind == "npm":
|
||||
info = NpmAnalyzer(
|
||||
log_path=params.npm_log_path,
|
||||
report_dir=params.npm_report_dir,
|
||||
top=params.top,
|
||||
quiet=params.quiet,
|
||||
full=params.full,
|
||||
min_errors=params.min_errors,
|
||||
error_ratio=params.error_ratio,
|
||||
).run()
|
||||
job.result = ScanResult(ok=True)
|
||||
job.details = {"report": info.get("report"),
|
||||
"data_dir": info.get("data_dir")}
|
||||
else: # pragma: no cover - guarded by type hint
|
||||
raise ValueError(f"unknown scan kind: {kind}")
|
||||
job.state = "done"
|
||||
except Exception as e:
|
||||
job.error = str(e)
|
||||
job.state = "error"
|
||||
finally:
|
||||
job.finished = _now()
|
||||
job.busy = False
|
||||
_scan_lock.release()
|
||||
_save_jobs()
|
||||
|
||||
threading.Thread(target=_run, daemon=True,
|
||||
name=f"scan-{job.job_id}").start()
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
def get_scan(job_id: str) -> dict[str, Any] | None:
|
||||
"""Return a single job as a dict, or None if unknown."""
|
||||
load_persisted()
|
||||
with _jobs_lock:
|
||||
job = _jobs.get(job_id)
|
||||
return job.to_dict() if job else None
|
||||
|
||||
|
||||
def _iso_key(value: str | None) -> float:
|
||||
"""ISO-8601 string -> epoch seconds (0.0 when unparseable/empty)."""
|
||||
if not value:
|
||||
return 0.0
|
||||
try:
|
||||
return datetime.fromisoformat(value).timestamp()
|
||||
except ValueError:
|
||||
return 0.0
|
||||
|
||||
|
||||
def latest_file_scan() -> dict[str, Any] | None:
|
||||
"""Build a ``done`` scan entry from the newest report files on disk.
|
||||
|
||||
Lets the dashboard show the latest completed scan even when it was not
|
||||
run through the web UI (server restart, CLI-run scans). Returns None when
|
||||
no scan outputs exist yet.
|
||||
"""
|
||||
|
||||
def newest(dirpath: str, suffix: str = "") -> str | None:
|
||||
"""Newest file in ``dirpath`` (optionally ending in ``suffix``)."""
|
||||
try:
|
||||
names = [n for n in os.listdir(dirpath)
|
||||
if os.path.isfile(os.path.join(dirpath, n))
|
||||
and (not suffix or n.endswith(suffix))]
|
||||
except OSError:
|
||||
return None
|
||||
return os.path.join(dirpath, max(names)) if names else None
|
||||
|
||||
mikrotik = newest(MIKROTIK_REPORT_DIR, ".json")
|
||||
cp_report = newest(COPYPARTY_REPORT_DIR, ".txt")
|
||||
npm_report = newest(NPM_REPORT_DIR, ".txt")
|
||||
v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
|
||||
v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
|
||||
have_outputs = (mikrotik is not None or cp_report is not None
|
||||
or npm_report is not None
|
||||
or os.path.exists(v4_path) or os.path.exists(v6_path))
|
||||
if not have_outputs:
|
||||
return None
|
||||
|
||||
# Timestamp: prefer the newest snapshot's collected_at, else the newest
|
||||
# report file's mtime, else now.
|
||||
timestamp: str | None = None
|
||||
if mikrotik:
|
||||
try:
|
||||
with open(mikrotik, "r", encoding="utf-8") as f:
|
||||
timestamp = json.load(f).get("collected_at")
|
||||
except (OSError, ValueError):
|
||||
timestamp = None
|
||||
if not timestamp:
|
||||
candidates = [p for p in (mikrotik, cp_report, npm_report,
|
||||
v4_path, v6_path)
|
||||
if p and os.path.exists(p)]
|
||||
if candidates:
|
||||
tz = datetime.now().astimezone().tzinfo
|
||||
timestamp = datetime.fromtimestamp(
|
||||
max(os.path.getmtime(p) for p in candidates),
|
||||
tz=tz).isoformat(timespec="seconds")
|
||||
if not timestamp:
|
||||
timestamp = _now()
|
||||
|
||||
blocked_targets = 0
|
||||
for path in (v4_path, v6_path):
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
blocked_targets += sum(1 for _ in f)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
return {
|
||||
"job_id": "files-latest",
|
||||
"kind": "consolidated",
|
||||
"state": "done",
|
||||
"started": timestamp,
|
||||
"finished": timestamp,
|
||||
"error": None,
|
||||
"busy": False,
|
||||
"details": {},
|
||||
"result": {
|
||||
"ok": True,
|
||||
"cp_exists": cp_report is not None,
|
||||
"npm_exists": npm_report is not None,
|
||||
"cp_candidates": 0,
|
||||
"npm_candidates": 0,
|
||||
"blocked_ips": blocked_targets,
|
||||
"blocked_targets": blocked_targets,
|
||||
"blocked_ipv4": [],
|
||||
"blocked_ipv6": [],
|
||||
"blocklist_v4_path": v4_path,
|
||||
"blocklist_v6_path": v6_path,
|
||||
"firewall_connected": mikrotik is not None,
|
||||
"firewall_changes": None,
|
||||
"errors": [],
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
|
||||
"""Return the most recently started jobs, newest first.
|
||||
|
||||
Appends a file-derived "latest scan" entry when the newest report files on
|
||||
disk reflect a scan newer than any known job (e.g. after a restart or a
|
||||
CLI run), so the dashboard always shows the latest completed scan.
|
||||
"""
|
||||
load_persisted()
|
||||
with _jobs_lock:
|
||||
jobs = sorted(_jobs.values(),
|
||||
key=lambda j: j.started or "", reverse=True)
|
||||
entries = [j.to_dict() for j in jobs]
|
||||
file_entry = latest_file_scan()
|
||||
if file_entry is not None:
|
||||
newest = jobs[0] if jobs else None
|
||||
newest_ts = _iso_key(
|
||||
(newest.finished or newest.started) if newest else None)
|
||||
# Only surface the file-derived entry when it is genuinely newer than
|
||||
# every known job (tolerance avoids duplicating the run that produced
|
||||
# it, where collected_at lands a moment after the job finished).
|
||||
if newest_ts == 0 or _iso_key(file_entry["started"]) - newest_ts > 2.0:
|
||||
entries.append(file_entry)
|
||||
entries.sort(key=lambda e: _iso_key(e.get("started")), reverse=True)
|
||||
return entries[:limit]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# persistence
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
def _save_jobs() -> None:
|
||||
"""Persist the newest jobs to ``SCAN_JOBS_PATH`` (atomic tmp+rename)."""
|
||||
with _jobs_lock:
|
||||
jobs = sorted(_jobs.values(),
|
||||
key=lambda j: j.started or "", reverse=True)
|
||||
payload = [j.to_dict() for j in jobs[:SCAN_JOBS_MAX]]
|
||||
os.makedirs(os.path.dirname(SCAN_JOBS_PATH), exist_ok=True)
|
||||
tmp = SCAN_JOBS_PATH + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(payload, f, indent=2, sort_keys=True)
|
||||
os.replace(tmp, SCAN_JOBS_PATH)
|
||||
|
||||
|
||||
def _load_jobs() -> None:
|
||||
"""Rebuild the in-memory job store from ``SCAN_JOBS_PATH``."""
|
||||
try:
|
||||
with open(SCAN_JOBS_PATH, "r", encoding="utf-8") as f:
|
||||
payload = json.load(f)
|
||||
except (OSError, ValueError, TypeError):
|
||||
payload = []
|
||||
if not isinstance(payload, list):
|
||||
return
|
||||
|
||||
jobs: dict[str, ScanJob] = {}
|
||||
for item in payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
try:
|
||||
params = ScanParams(**item.get("params", {}))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
try:
|
||||
job = ScanJob(
|
||||
job_id=str(item["job_id"]),
|
||||
kind=item.get("kind", "consolidated"),
|
||||
params=params,
|
||||
state=item.get("state", "done"),
|
||||
started=item.get("started"),
|
||||
finished=item.get("finished"),
|
||||
error=item.get("error"),
|
||||
busy=False,
|
||||
details=item.get("details", {}) or {},
|
||||
)
|
||||
except (KeyError, TypeError, ValueError):
|
||||
continue
|
||||
result = item.get("result")
|
||||
if isinstance(result, dict):
|
||||
try:
|
||||
job.result = ScanResult(**result)
|
||||
except TypeError:
|
||||
job.result = None
|
||||
jobs[job.job_id] = job
|
||||
|
||||
with _jobs_lock:
|
||||
_jobs.clear()
|
||||
_jobs.update(jobs)
|
||||
|
||||
|
||||
def load_persisted() -> None:
|
||||
"""Load persisted jobs from disk once (idempotent)."""
|
||||
global _loaded
|
||||
if _loaded:
|
||||
return
|
||||
_load_jobs()
|
||||
_loaded = True
|
||||
|
||||
|
||||
def is_scan_running() -> bool:
|
||||
"""True if any job currently holds the scan lock."""
|
||||
return _scan_lock.locked()
|
||||
Reference in new issue
Block a user