zwitschi
2fb4032766
- R001: app/db/backfill.py imports git-history blocklist -> ledger, firewall snapshots, and per-source requests.csv (idempotent; ran live migrating 705 historical blocklist entries). - S001: web_scan.latest_scans() now merges SQLite 'scans' records so the dashboard surfaces CLI/restart scans (_db_scans). - F001: /api/firewall/snapshots + 'Snapshot history' panel; added recent_firewall_snapshots() query with entry counts.
409 lines
14 KiB
Python
409 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Background scan runner for the web UI.
|
|
|
|
Runs scans in a daemon thread so HTTP requests never block, keeps an
|
|
in-memory job store, and serializes scans with a busy lock (WHOIS and
|
|
whitelist refresh must not run concurrently).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import threading
|
|
import uuid
|
|
from dataclasses import asdict, dataclass, field
|
|
from datetime import datetime
|
|
from typing import Any, Literal
|
|
|
|
from ..utils.constants import (
|
|
APP_REPORT_DATA_DIR,
|
|
COPYPARTY_REPORT_DIR,
|
|
MIKROTIK_REPORT_DIR,
|
|
NPM_REPORT_DIR,
|
|
SCAN_JOBS_MAX,
|
|
SCAN_JOBS_PATH,
|
|
)
|
|
from ..copyparty import CopypartyAnalyzer
|
|
from ..npm import NpmAnalyzer
|
|
from ..scan import ScanParams, ScanResult, run_scan
|
|
|
|
ScanKind = Literal["copyparty", "npm", "consolidated"]
|
|
|
|
#: serializes scans — only one runs at a time
|
|
_scan_lock = threading.Lock()
|
|
|
|
#: in-memory job store (job_id -> ScanJob)
|
|
_jobs: dict[str, "ScanJob"] = {}
|
|
_jobs_lock = threading.Lock()
|
|
|
|
#: True once persisted jobs have been loaded from disk
|
|
_loaded = False
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now().astimezone().isoformat(timespec="seconds")
|
|
|
|
|
|
@dataclass
|
|
class ScanJob:
|
|
"""One background scan job."""
|
|
|
|
job_id: str
|
|
kind: ScanKind
|
|
params: ScanParams
|
|
state: str = "queued" # queued | running | done | error
|
|
started: str | None = None
|
|
finished: str | None = None
|
|
result: ScanResult | None = None
|
|
error: str | None = None
|
|
busy: bool = False
|
|
details: dict = field(default_factory=dict)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
result = self.result
|
|
return {
|
|
"job_id": self.job_id,
|
|
"kind": self.kind,
|
|
"params": asdict(self.params),
|
|
"state": self.state,
|
|
"started": self.started,
|
|
"finished": self.finished,
|
|
"error": self.error,
|
|
"busy": self.busy,
|
|
"details": self.details,
|
|
"result": None if result is None else {
|
|
"ok": result.ok,
|
|
"cp_exists": result.cp_exists,
|
|
"npm_exists": result.npm_exists,
|
|
"cp_candidates": result.cp_candidates,
|
|
"npm_candidates": result.npm_candidates,
|
|
"blocked_ips": result.blocked_ips,
|
|
"blocked_targets": result.blocked_targets,
|
|
"blocked_ipv4": result.blocked_ipv4,
|
|
"blocked_ipv6": result.blocked_ipv6,
|
|
"blocklist_v4_path": result.blocklist_v4_path,
|
|
"blocklist_v6_path": result.blocklist_v6_path,
|
|
"firewall_connected": result.firewall_connected,
|
|
"firewall_changes": result.firewall_changes,
|
|
"errors": result.errors,
|
|
},
|
|
}
|
|
|
|
|
|
def start_scan(kind: ScanKind, params: ScanParams) -> dict[str, Any]:
|
|
"""Start a scan in the background and return its job dict.
|
|
|
|
If another scan is already running, returns a dict with an ``error`` key.
|
|
"""
|
|
if not _scan_lock.acquire(blocking=False):
|
|
return {"error": "a scan is already running; try again when it finishes"}
|
|
|
|
job = ScanJob(job_id=uuid.uuid4().hex[:12], kind=kind,
|
|
params=params, busy=True)
|
|
with _jobs_lock:
|
|
_jobs[job.job_id] = job
|
|
_save_jobs()
|
|
|
|
def _run() -> None:
|
|
job.started = _now()
|
|
job.state = "running"
|
|
try:
|
|
if kind == "consolidated":
|
|
job.result = run_scan(params)
|
|
elif kind == "copyparty":
|
|
info = CopypartyAnalyzer(
|
|
log_path=params.copyparty_log_path,
|
|
report_dir=params.copyparty_report_dir,
|
|
top=params.top,
|
|
quiet=params.quiet,
|
|
full=params.full,
|
|
min_denied=params.min_denied,
|
|
).run()
|
|
job.result = ScanResult(ok=True)
|
|
job.details = {"report": info.get("report"),
|
|
"data_dir": info.get("data_dir")}
|
|
elif kind == "npm":
|
|
info = NpmAnalyzer(
|
|
log_path=params.npm_log_path,
|
|
report_dir=params.npm_report_dir,
|
|
top=params.top,
|
|
quiet=params.quiet,
|
|
full=params.full,
|
|
min_errors=params.min_errors,
|
|
error_ratio=params.error_ratio,
|
|
).run()
|
|
job.result = ScanResult(ok=True)
|
|
job.details = {"report": info.get("report"),
|
|
"data_dir": info.get("data_dir")}
|
|
else: # pragma: no cover - guarded by type hint
|
|
raise ValueError(f"unknown scan kind: {kind}")
|
|
job.state = "done"
|
|
except Exception as e:
|
|
job.error = str(e)
|
|
job.state = "error"
|
|
finally:
|
|
job.finished = _now()
|
|
job.busy = False
|
|
_scan_lock.release()
|
|
_save_jobs()
|
|
|
|
threading.Thread(target=_run, daemon=True,
|
|
name=f"scan-{job.job_id}").start()
|
|
return job.to_dict()
|
|
|
|
|
|
def get_scan(job_id: str) -> dict[str, Any] | None:
|
|
"""Return a single job as a dict, or None if unknown."""
|
|
load_persisted()
|
|
with _jobs_lock:
|
|
job = _jobs.get(job_id)
|
|
return job.to_dict() if job else None
|
|
|
|
|
|
def _iso_key(value: str | None) -> float:
|
|
"""ISO-8601 string -> epoch seconds (0.0 when unparseable/empty)."""
|
|
if not value:
|
|
return 0.0
|
|
try:
|
|
return datetime.fromisoformat(value).timestamp()
|
|
except ValueError:
|
|
return 0.0
|
|
|
|
|
|
def latest_file_scan() -> dict[str, Any] | None:
|
|
"""Build a ``done`` scan entry from the newest report files on disk.
|
|
|
|
Lets the dashboard show the latest completed scan even when it was not
|
|
run through the web UI (server restart, CLI-run scans). Returns None when
|
|
no scan outputs exist yet.
|
|
"""
|
|
|
|
def newest(dirpath: str, suffix: str = "") -> str | None:
|
|
"""Newest file in ``dirpath`` (optionally ending in ``suffix``)."""
|
|
try:
|
|
names = [n for n in os.listdir(dirpath)
|
|
if os.path.isfile(os.path.join(dirpath, n))
|
|
and (not suffix or n.endswith(suffix))]
|
|
except OSError:
|
|
return None
|
|
return os.path.join(dirpath, max(names)) if names else None
|
|
|
|
mikrotik = newest(MIKROTIK_REPORT_DIR, ".json")
|
|
cp_report = newest(COPYPARTY_REPORT_DIR, ".txt")
|
|
npm_report = newest(NPM_REPORT_DIR, ".txt")
|
|
v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
|
|
v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
|
|
have_outputs = (mikrotik is not None or cp_report is not None
|
|
or npm_report is not None
|
|
or os.path.exists(v4_path) or os.path.exists(v6_path))
|
|
if not have_outputs:
|
|
return None
|
|
|
|
# Timestamp: prefer the newest snapshot's collected_at, else the newest
|
|
# report file's mtime, else now.
|
|
timestamp: str | None = None
|
|
if mikrotik:
|
|
try:
|
|
with open(mikrotik, "r", encoding="utf-8") as f:
|
|
timestamp = json.load(f).get("collected_at")
|
|
except (OSError, ValueError):
|
|
timestamp = None
|
|
if not timestamp:
|
|
candidates = [p for p in (mikrotik, cp_report, npm_report,
|
|
v4_path, v6_path)
|
|
if p and os.path.exists(p)]
|
|
if candidates:
|
|
tz = datetime.now().astimezone().tzinfo
|
|
timestamp = datetime.fromtimestamp(
|
|
max(os.path.getmtime(p) for p in candidates),
|
|
tz=tz).isoformat(timespec="seconds")
|
|
if not timestamp:
|
|
timestamp = _now()
|
|
|
|
blocked_targets = 0
|
|
for path in (v4_path, v6_path):
|
|
if os.path.exists(path):
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as f:
|
|
blocked_targets += sum(1 for _ in f)
|
|
except OSError:
|
|
pass
|
|
|
|
return {
|
|
"job_id": "files-latest",
|
|
"kind": "consolidated",
|
|
"state": "done",
|
|
"started": timestamp,
|
|
"finished": timestamp,
|
|
"error": None,
|
|
"busy": False,
|
|
"details": {},
|
|
"result": {
|
|
"ok": True,
|
|
"cp_exists": cp_report is not None,
|
|
"npm_exists": npm_report is not None,
|
|
"cp_candidates": 0,
|
|
"npm_candidates": 0,
|
|
"blocked_ips": blocked_targets,
|
|
"blocked_targets": blocked_targets,
|
|
"blocked_ipv4": [],
|
|
"blocked_ipv6": [],
|
|
"blocklist_v4_path": v4_path,
|
|
"blocklist_v6_path": v6_path,
|
|
"firewall_connected": mikrotik is not None,
|
|
"firewall_changes": None,
|
|
"errors": [],
|
|
},
|
|
}
|
|
|
|
|
|
def _db_scans(limit: int = 10) -> list[dict[str, Any]]:
|
|
"""Recent scan records from the SQLite ``scans`` table (S001).
|
|
|
|
Converts DB rows into the web job-dict shape so the dashboard can surface
|
|
scans that were persisted by ``run_scan`` / analyzers even if they were
|
|
never run through the in-memory job store (e.g. CLI scans or restarts).
|
|
Returns [] when the DB is unavailable.
|
|
"""
|
|
try:
|
|
from ..db import get_connection
|
|
from ..db import repositories as repo
|
|
conn = get_connection()
|
|
rows = repo.latest_scans(conn, limit=limit)
|
|
except Exception:
|
|
return []
|
|
out = []
|
|
for r in rows:
|
|
try:
|
|
result_json = r.get("result_json")
|
|
result = json.loads(result_json) if result_json else {}
|
|
except (TypeError, ValueError):
|
|
result = {}
|
|
out.append({
|
|
"job_id": f"db-scan-{r['id']}",
|
|
"kind": r.get("kind") or "consolidated",
|
|
"state": r.get("status") or "done",
|
|
"started": r.get("started_at"),
|
|
"finished": r.get("finished_at"),
|
|
"error": r.get("error"),
|
|
"busy": False,
|
|
"details": {},
|
|
"result": result or None,
|
|
})
|
|
return out
|
|
|
|
|
|
def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
|
|
"""Return the most recently started jobs, newest first.
|
|
|
|
Merges three sources, newest-first:
|
|
1. the in-memory job store (currently running / last-run jobs),
|
|
2. the SQLite ``scans`` table (persisted by run_scan/analyzers — S001),
|
|
3. a file-derived "latest scan" fallback for pre-DB runs.
|
|
Any DB-backed scan not already represented in memory is appended so the
|
|
dashboard shows scans from CLI runs and across restarts.
|
|
"""
|
|
load_persisted()
|
|
with _jobs_lock:
|
|
jobs = sorted(_jobs.values(),
|
|
key=lambda j: j.started or "", reverse=True)
|
|
entries = [j.to_dict() for j in jobs]
|
|
|
|
# Add DB scan records that aren't already in the in-memory store. Key on
|
|
# (kind, started_at) to avoid duplicating a job that run_scan also logged.
|
|
known = {(e.get("kind"), e.get("started")) for e in entries}
|
|
for db_row in _db_scans(limit=limit * 2):
|
|
if (db_row.get("kind"), db_row.get("started")) not in known:
|
|
entries.append(db_row)
|
|
|
|
file_entry = latest_file_scan()
|
|
if file_entry is not None:
|
|
newest = (entries or [None])[0]
|
|
newest_key = (
|
|
(newest.get("finished") or newest.get("started")) if newest else None)
|
|
newest_ts = _iso_key(newest_key)
|
|
# Only surface the file-derived entry when it is genuinely newer than
|
|
# every known job (tolerance avoids duplicating the run that produced
|
|
# it, where collected_at lands a moment after the job finished).
|
|
if newest_ts == 0 or _iso_key(file_entry["started"]) - newest_ts > 2.0:
|
|
entries.append(file_entry)
|
|
entries.sort(key=lambda e: _iso_key(e.get("started")), reverse=True)
|
|
return entries[:limit]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# persistence
|
|
# ---------------------------------------------------------------------------
|
|
|
|
def _save_jobs() -> None:
|
|
"""Persist the newest jobs to ``SCAN_JOBS_PATH`` (atomic tmp+rename)."""
|
|
with _jobs_lock:
|
|
jobs = sorted(_jobs.values(),
|
|
key=lambda j: j.started or "", reverse=True)
|
|
payload = [j.to_dict() for j in jobs[:SCAN_JOBS_MAX]]
|
|
os.makedirs(os.path.dirname(SCAN_JOBS_PATH), exist_ok=True)
|
|
tmp = SCAN_JOBS_PATH + ".tmp"
|
|
with open(tmp, "w", encoding="utf-8") as f:
|
|
json.dump(payload, f, indent=2, sort_keys=True)
|
|
os.replace(tmp, SCAN_JOBS_PATH)
|
|
|
|
|
|
def _load_jobs() -> None:
|
|
"""Rebuild the in-memory job store from ``SCAN_JOBS_PATH``."""
|
|
try:
|
|
with open(SCAN_JOBS_PATH, "r", encoding="utf-8") as f:
|
|
payload = json.load(f)
|
|
except (OSError, ValueError, TypeError):
|
|
payload = []
|
|
if not isinstance(payload, list):
|
|
return
|
|
|
|
jobs: dict[str, ScanJob] = {}
|
|
for item in payload:
|
|
if not isinstance(item, dict):
|
|
continue
|
|
try:
|
|
params = ScanParams(**item.get("params", {}))
|
|
except (TypeError, ValueError):
|
|
continue
|
|
try:
|
|
job = ScanJob(
|
|
job_id=str(item["job_id"]),
|
|
kind=item.get("kind", "consolidated"),
|
|
params=params,
|
|
state=item.get("state", "done"),
|
|
started=item.get("started"),
|
|
finished=item.get("finished"),
|
|
error=item.get("error"),
|
|
busy=False,
|
|
details=item.get("details", {}) or {},
|
|
)
|
|
except (KeyError, TypeError, ValueError):
|
|
continue
|
|
result = item.get("result")
|
|
if isinstance(result, dict):
|
|
try:
|
|
job.result = ScanResult(**result)
|
|
except TypeError:
|
|
job.result = None
|
|
jobs[job.job_id] = job
|
|
|
|
with _jobs_lock:
|
|
_jobs.clear()
|
|
_jobs.update(jobs)
|
|
|
|
|
|
def load_persisted() -> None:
|
|
"""Load persisted jobs from disk once (idempotent)."""
|
|
global _loaded
|
|
if _loaded:
|
|
return
|
|
_load_jobs()
|
|
_loaded = True
|
|
|
|
|
|
def is_scan_running() -> bool:
|
|
"""True if any job currently holds the scan lock."""
|
|
return _scan_lock.locked()
|