feat(db): backfill, DB-backed web scans, firewall snapshot view (R001/S001/F001)
- R001: app/db/backfill.py imports git-history blocklist -> ledger, firewall snapshots, and per-source requests.csv (idempotent; ran live migrating 705 historical blocklist entries). - S001: web_scan.latest_scans() now merges SQLite 'scans' records so the dashboard surfaces CLI/restart scans (_db_scans). - F001: /api/firewall/snapshots + 'Snapshot history' panel; added recent_firewall_snapshots() query with entry counts.
This commit is contained in:
1 parent
ba48054293
commit
2fb4032766
4 files changed
+308
-6
No files matched your search
+54
-6
@@ -258,23 +258,71 @@ def latest_file_scan() -> dict[str, Any] | None:
|
||||
}
|
||||
|
||||
|
||||
def _db_scans(limit: int = 10) -> list[dict[str, Any]]:
|
||||
"""Recent scan records from the SQLite ``scans`` table (S001).
|
||||
|
||||
Converts DB rows into the web job-dict shape so the dashboard can surface
|
||||
scans that were persisted by ``run_scan`` / analyzers even if they were
|
||||
never run through the in-memory job store (e.g. CLI scans or restarts).
|
||||
Returns [] when the DB is unavailable.
|
||||
"""
|
||||
try:
|
||||
from ..db import get_connection
|
||||
from ..db import repositories as repo
|
||||
conn = get_connection()
|
||||
rows = repo.latest_scans(conn, limit=limit)
|
||||
except Exception:
|
||||
return []
|
||||
out = []
|
||||
for r in rows:
|
||||
try:
|
||||
result_json = r.get("result_json")
|
||||
result = json.loads(result_json) if result_json else {}
|
||||
except (TypeError, ValueError):
|
||||
result = {}
|
||||
out.append({
|
||||
"job_id": f"db-scan-{r['id']}",
|
||||
"kind": r.get("kind") or "consolidated",
|
||||
"state": r.get("status") or "done",
|
||||
"started": r.get("started_at"),
|
||||
"finished": r.get("finished_at"),
|
||||
"error": r.get("error"),
|
||||
"busy": False,
|
||||
"details": {},
|
||||
"result": result or None,
|
||||
})
|
||||
return out
|
||||
|
||||
|
||||
def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
|
||||
"""Return the most recently started jobs, newest first.
|
||||
|
||||
Appends a file-derived "latest scan" entry when the newest report files on
|
||||
disk reflect a scan newer than any known job (e.g. after a restart or a
|
||||
CLI run), so the dashboard always shows the latest completed scan.
|
||||
Merges three sources, newest-first:
|
||||
1. the in-memory job store (currently running / last-run jobs),
|
||||
2. the SQLite ``scans`` table (persisted by run_scan/analyzers — S001),
|
||||
3. a file-derived "latest scan" fallback for pre-DB runs.
|
||||
Any DB-backed scan not already represented in memory is appended so the
|
||||
dashboard shows scans from CLI runs and across restarts.
|
||||
"""
|
||||
load_persisted()
|
||||
with _jobs_lock:
|
||||
jobs = sorted(_jobs.values(),
|
||||
key=lambda j: j.started or "", reverse=True)
|
||||
entries = [j.to_dict() for j in jobs]
|
||||
|
||||
# Add DB scan records that aren't already in the in-memory store. Key on
|
||||
# (kind, started_at) to avoid duplicating a job that run_scan also logged.
|
||||
known = {(e.get("kind"), e.get("started")) for e in entries}
|
||||
for db_row in _db_scans(limit=limit * 2):
|
||||
if (db_row.get("kind"), db_row.get("started")) not in known:
|
||||
entries.append(db_row)
|
||||
|
||||
file_entry = latest_file_scan()
|
||||
if file_entry is not None:
|
||||
newest = jobs[0] if jobs else None
|
||||
newest_ts = _iso_key(
|
||||
(newest.finished or newest.started) if newest else None)
|
||||
newest = (entries or [None])[0]
|
||||
newest_key = (
|
||||
(newest.get("finished") or newest.get("started")) if newest else None)
|
||||
newest_ts = _iso_key(newest_key)
|
||||
# Only surface the file-derived entry when it is genuinely newer than
|
||||
# every known job (tolerance avoids duplicating the run that produced
|
||||
# it, where collected_at lands a moment after the job finished).
|
||||
|
||||
Reference in new issue
Block a user