feat(db): backfill, DB-backed web scans, firewall snapshot view (R001/S001/F001)

- R001: app/db/backfill.py imports git-history blocklist -> ledger, firewall
  snapshots, and per-source requests.csv (idempotent; ran live migrating 705
  historical blocklist entries).
- S001: web_scan.latest_scans() now merges SQLite 'scans' records so the
  dashboard surfaces CLI/restart scans (_db_scans).
- F001: /api/firewall/snapshots + 'Snapshot history' panel; added
  recent_firewall_snapshots() query with entry counts.
This commit is contained in:
zwitschi committed 2026-10-24 19:56:20 +02:00
1 parent ba48054293
commit 2fb4032766
4 files changed
+308 -6

No files matched your search

+13
View File
@@ -265,6 +265,19 @@ def register_routes(app: Flask,
"snapshot_dir": MIKROTIK_DATA_DIR,
})
@app.get("/api/firewall/snapshots")
def firewall_snapshots():
"""Recent firewall snapshots persisted in the SQLite store (F001)."""
try:
from ..db import get_connection
from ..db import repositories as repo
conn = get_connection()
limit = request.args.get("limit", 10, type=int)
return jsonify({"snapshots": repo.recent_firewall_snapshots(
conn, limit=limit)})
except Exception as exc:
return jsonify({"error": str(exc), "snapshots": []}), 503
@app.post("/api/firewall/sync")
def firewall_sync():
return _sync_firewall(dry_run=True)
+54 -6
View File
@@ -258,23 +258,71 @@ def latest_file_scan() -> dict[str, Any] | None:
}
def _db_scans(limit: int = 10) -> list[dict[str, Any]]:
"""Recent scan records from the SQLite ``scans`` table (S001).
Converts DB rows into the web job-dict shape so the dashboard can surface
scans that were persisted by ``run_scan`` / analyzers even if they were
never run through the in-memory job store (e.g. CLI scans or restarts).
Returns [] when the DB is unavailable.
"""
try:
from ..db import get_connection
from ..db import repositories as repo
conn = get_connection()
rows = repo.latest_scans(conn, limit=limit)
except Exception:
return []
out = []
for r in rows:
try:
result_json = r.get("result_json")
result = json.loads(result_json) if result_json else {}
except (TypeError, ValueError):
result = {}
out.append({
"job_id": f"db-scan-{r['id']}",
"kind": r.get("kind") or "consolidated",
"state": r.get("status") or "done",
"started": r.get("started_at"),
"finished": r.get("finished_at"),
"error": r.get("error"),
"busy": False,
"details": {},
"result": result or None,
})
return out
def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
"""Return the most recently started jobs, newest first.
Appends a file-derived "latest scan" entry when the newest report files on
disk reflect a scan newer than any known job (e.g. after a restart or a
CLI run), so the dashboard always shows the latest completed scan.
Merges three sources, newest-first:
1. the in-memory job store (currently running / last-run jobs),
2. the SQLite ``scans`` table (persisted by run_scan/analyzers — S001),
3. a file-derived "latest scan" fallback for pre-DB runs.
Any DB-backed scan not already represented in memory is appended so the
dashboard shows scans from CLI runs and across restarts.
"""
load_persisted()
with _jobs_lock:
jobs = sorted(_jobs.values(),
key=lambda j: j.started or "", reverse=True)
entries = [j.to_dict() for j in jobs]
# Add DB scan records that aren't already in the in-memory store. Key on
# (kind, started_at) to avoid duplicating a job that run_scan also logged.
known = {(e.get("kind"), e.get("started")) for e in entries}
for db_row in _db_scans(limit=limit * 2):
if (db_row.get("kind"), db_row.get("started")) not in known:
entries.append(db_row)
file_entry = latest_file_scan()
if file_entry is not None:
newest = jobs[0] if jobs else None
newest_ts = _iso_key(
(newest.finished or newest.started) if newest else None)
newest = (entries or [None])[0]
newest_key = (
(newest.get("finished") or newest.get("started")) if newest else None)
newest_ts = _iso_key(newest_key)
# Only surface the file-derived entry when it is genuinely newer than
# every known job (tolerance avoids duplicating the run that produced
# it, where collected_at lands a moment after the job finished).