zwitschi
51f57f7a04
Track last processed byte offset per log file so scans only read new entries, add --full flag to force a rescan, and persist state atomically.
156 lines
5.2 KiB
Python
156 lines
5.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Background scan runner for the web UI.
|
|
|
|
Runs scans in a daemon thread so HTTP requests never block, keeps an
|
|
in-memory job store, and serializes scans with a busy lock (WHOIS and
|
|
whitelist refresh must not run concurrently).
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import threading
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime
|
|
from typing import Any, Literal
|
|
|
|
from .copyparty import CopypartyAnalyzer
|
|
from .npm import NpmAnalyzer
|
|
from .scan import ScanParams, ScanResult, run_scan
|
|
|
|
ScanKind = Literal["copyparty", "npm", "consolidated"]
|
|
|
|
#: serializes scans — only one runs at a time
|
|
_scan_lock = threading.Lock()
|
|
|
|
#: in-memory job store (job_id -> ScanJob)
|
|
_jobs: dict[str, "ScanJob"] = {}
|
|
_jobs_lock = threading.Lock()
|
|
|
|
|
|
def _now() -> str:
|
|
return datetime.now().astimezone().isoformat(timespec="seconds")
|
|
|
|
|
|
@dataclass
|
|
class ScanJob:
|
|
"""One background scan job."""
|
|
|
|
job_id: str
|
|
kind: ScanKind
|
|
params: ScanParams
|
|
state: str = "queued" # queued | running | done | error
|
|
started: str | None = None
|
|
finished: str | None = None
|
|
result: ScanResult | None = None
|
|
error: str | None = None
|
|
busy: bool = False
|
|
details: dict = field(default_factory=dict)
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
result = self.result
|
|
return {
|
|
"job_id": self.job_id,
|
|
"kind": self.kind,
|
|
"state": self.state,
|
|
"started": self.started,
|
|
"finished": self.finished,
|
|
"error": self.error,
|
|
"busy": self.busy,
|
|
"details": self.details,
|
|
"result": None if result is None else {
|
|
"ok": result.ok,
|
|
"cp_exists": result.cp_exists,
|
|
"npm_exists": result.npm_exists,
|
|
"cp_candidates": result.cp_candidates,
|
|
"npm_candidates": result.npm_candidates,
|
|
"blocked_ips": result.blocked_ips,
|
|
"blocked_targets": result.blocked_targets,
|
|
"blocklist_v4_path": result.blocklist_v4_path,
|
|
"blocklist_v6_path": result.blocklist_v6_path,
|
|
"firewall_connected": result.firewall_connected,
|
|
"firewall_changes": result.firewall_changes,
|
|
"errors": result.errors,
|
|
},
|
|
}
|
|
|
|
|
|
def start_scan(kind: ScanKind, params: ScanParams) -> dict[str, Any]:
|
|
"""Start a scan in the background and return its job dict.
|
|
|
|
If another scan is already running, returns a dict with an ``error`` key.
|
|
"""
|
|
if not _scan_lock.acquire(blocking=False):
|
|
return {"error": "a scan is already running; try again when it finishes"}
|
|
|
|
job = ScanJob(job_id=uuid.uuid4().hex[:12], kind=kind,
|
|
params=params, busy=True)
|
|
with _jobs_lock:
|
|
_jobs[job.job_id] = job
|
|
|
|
def _run() -> None:
|
|
job.started = _now()
|
|
job.state = "running"
|
|
try:
|
|
if kind == "consolidated":
|
|
job.result = run_scan(params)
|
|
elif kind == "copyparty":
|
|
info = CopypartyAnalyzer(
|
|
log_path=params.copyparty_log_path,
|
|
report_dir=params.copyparty_report_dir,
|
|
top=params.top,
|
|
quiet=params.quiet,
|
|
full=params.full,
|
|
min_denied=params.min_denied,
|
|
).run()
|
|
job.result = ScanResult(ok=True)
|
|
job.details = {"report": info.get("report"),
|
|
"data_dir": info.get("data_dir")}
|
|
elif kind == "npm":
|
|
info = NpmAnalyzer(
|
|
log_path=params.npm_log_path,
|
|
report_dir=params.npm_report_dir,
|
|
top=params.top,
|
|
quiet=params.quiet,
|
|
full=params.full,
|
|
min_errors=params.min_errors,
|
|
error_ratio=params.error_ratio,
|
|
).run()
|
|
job.result = ScanResult(ok=True)
|
|
job.details = {"report": info.get("report"),
|
|
"data_dir": info.get("data_dir")}
|
|
else: # pragma: no cover - guarded by type hint
|
|
raise ValueError(f"unknown scan kind: {kind}")
|
|
job.state = "done"
|
|
except Exception as e:
|
|
job.error = str(e)
|
|
job.state = "error"
|
|
finally:
|
|
job.finished = _now()
|
|
job.busy = False
|
|
_scan_lock.release()
|
|
|
|
threading.Thread(target=_run, daemon=True,
|
|
name=f"scan-{job.job_id}").start()
|
|
return job.to_dict()
|
|
|
|
|
|
def get_scan(job_id: str) -> dict[str, Any] | None:
|
|
"""Return a single job as a dict, or None if unknown."""
|
|
with _jobs_lock:
|
|
job = _jobs.get(job_id)
|
|
return job.to_dict() if job else None
|
|
|
|
|
|
def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
|
|
"""Return the most recently started jobs, newest first."""
|
|
with _jobs_lock:
|
|
jobs = sorted(_jobs.values(),
|
|
key=lambda j: j.started or "", reverse=True)
|
|
return [j.to_dict() for j in jobs[:limit]]
|
|
|
|
|
|
def is_scan_running() -> bool:
|
|
"""True if any job currently holds the scan lock."""
|
|
return _scan_lock.locked()
|