From ba48054293687c532685c681b89e1f6d9e4badb4 Mon Sep 17 00:00:00 2001 From: Zwitschi Date: Sat, 24 Oct 2026 19:56:17 +0200 Subject: [PATCH] feat(blocklist): make SQLite ledger authoritative for the cumulative set (B001) - Add repo.record_blocklist(): grow-only, never auto-removes (mirrors router cumulative model; preserves entries that leave the NPM log window). - scan.py uses it on the scan path and derives the report's complete set from the ledger (_db_complete_blocklist), falling back to git history only if the DB is unavailable. --- app/db/repositories.py | 40 ++++++++++++++++++++++++++++++++++++++ app/scan/__init__.py | 44 +++++++++++++++++++++++++++++++++++------- 2 files changed, 77 insertions(+), 7 deletions(-) diff --git a/app/db/repositories.py b/app/db/repositories.py index 070fb1f..e5f353d 100644 --- a/app/db/repositories.py +++ b/app/db/repositories.py @@ -322,6 +322,32 @@ def unblock_entry(conn: sqlite3.Connection, cidr: str, *, return removed +def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *, + added_by: str | None = None, reasons=None) -> dict[str, Any]: + """Grow-only record: add any ``cidrs`` not already active to the ledger. + + Unlike :func:`sync_blocklist`, this **never removes** entries. It mirrors + the router's cumulative model: once an IP/subnet has ever been blocked it + stays active even after it leaves the scan window (e.g. NPM ~30-day log + retention), exactly like the old git-history backfill. Returns the added + delta and the resulting active count. + """ + cidr_set = set(cidrs) + before = {r["cidr"] for r in active_blocklist(conn)} + to_add = sorted(cidr_set - before) + now = utcnow() + with transaction(conn): + for cidr in to_add: + _block_entry_inner(conn, cidr, added_by=added_by, last_seen=now, + reasons=reasons) + # Refresh last_seen on entries that are still desired so the ledger + # reflects recency for existing rows too. + conn.executemany( + "UPDATE blocklist_entries SET last_seen=? WHERE cidr=? AND status='active'", + [(now, cidr) for cidr in cidr_set.intersection(before)]) + return {"added": to_add, "active": len(before) + len(to_add)} + + def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *, added_by: str | None = None) -> dict[str, Any]: """Reconcile active blocklist to ``desired_cidrs``. @@ -465,6 +491,20 @@ def save_firewall_snapshot(conn: sqlite3.Connection, *, return snapshot_id +def recent_firewall_snapshots(conn: sqlite3.Connection, + limit: int = 10) -> list[dict[str, Any]]: + """Most recent firewall snapshots with entry counts (F001).""" + rows = conn.execute( + "SELECT fs.id, fs.captured_at, fs.kind, fs.trigger, " + "(SELECT COUNT(*) FROM firewall_address_list_entries e " + " WHERE e.snapshot_id = fs.id) AS addr_entries, " + "(SELECT COUNT(*) FROM firewall_rule_entries r " + " WHERE r.snapshot_id = fs.id) AS rule_entries " + "FROM firewall_snapshots fs ORDER BY fs.id DESC LIMIT ?", + (limit,)).fetchall() + return [dict(r) for r in rows] + + # --------------------------------------------------------------------------- # WHOIS cache # --------------------------------------------------------------------------- diff --git a/app/scan/__init__.py b/app/scan/__init__.py index 2a778ca..99586ee 100644 --- a/app/scan/__init__.py +++ b/app/scan/__init__.py @@ -42,18 +42,42 @@ from ..utils.whois import consolidate_by_subnet def _sync_blocklist_to_db(connected: bool, desired_v4: list[str], desired_v6: list[str], added_by: str | None = None) -> None: - """Best-effort sync of the consolidated blocklist targets into SQLite. + """Best-effort grow-only sync of block targets into the SQLite ledger. - Keep the DB ledger in sync regardless of whether a router sync happened so - analysis always sees the current policy.""" + Uses ``record_blocklist`` (add-only) so the ledger is the authoritative + cumulative record: once an IP is ever blocked it stays active even after it + leaves the scan window, matching the router's cumulative model. Returns the + added delta or None if the DB is unavailable. + """ try: from ..db import get_connection from ..db import repositories as repo conn = get_connection() - repo.sync_blocklist(conn, list(desired_v4) + list(desired_v6), - added_by=added_by) + return repo.record_blocklist( + conn, list(desired_v4) + list(desired_v6), added_by=added_by) except Exception as exc: # never break the scan for a DB issue print(f"WARN: could not sync blocklist to db ({exc})") + return None + + +def _db_complete_blocklist(desired_v4: list[str], desired_v6: list[str]): + """Complete cumulative blocklist from the DB ledger (B001). + + Returns ``(complete_v4, complete_v6)`` derived from the active blocklist + ledger plus this scan's newly desired targets (in case the DB write for + this scan has not run yet). Falls back to the git-history reconstruction + when the DB is unavailable, so nothing regresses. + """ + try: + from ..db import get_connection + from ..db import repositories as repo + conn = get_connection() + active = [e["cidr"] for e in repo.active_blocklist(conn)] + except Exception: + active = [] + union = set(active) | set(desired_v4) | set(desired_v6) + v4, v6 = split_cidr_versions(union) + return v4, v6 @dataclass @@ -256,9 +280,15 @@ def run_scan(params: ScanParams) -> ScanResult: f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.") # Timestamped change report — only when the router was reachable. - complete_v4, complete_v6 = cumulative_blocklist( - blocklist_v4_path, blocklist_v6_path, + # Source of truth for the cumulative set: the SQLite |ledger| first + # (B001), falling back to the git-history reconstruction only if the + # DB is unavailable. + complete_v4, complete_v6 = _db_complete_blocklist( blocked_ipv4, blocked_ipv6) + if not complete_v4 and not complete_v6: + complete_v4, complete_v6 = cumulative_blocklist( + blocklist_v4_path, blocklist_v6_path, + blocked_ipv4, blocked_ipv6) blocklist_report_path = write_blocklist_report( params.report_dir, history_v4=history_v4, history_v6=history_v6,