feat(blocklist): make SQLite ledger authoritative for the cumulative set (B001)

- Add repo.record_blocklist(): grow-only, never auto-removes (mirrors router
  cumulative model; preserves entries that leave the NPM log window).
- scan.py uses it on the scan path and derives the report's complete set from
  the ledger (_db_complete_blocklist), falling back to git history only if the
  DB is unavailable.
This commit is contained in:
zwitschi committed 2026-10-24 19:56:17 +02:00
1 parent 4a30f6b582
commit ba48054293
2 files changed
+77 -7

No files matched your search

+40
View File
@@ -322,6 +322,32 @@ def unblock_entry(conn: sqlite3.Connection, cidr: str, *,
return removed
def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *,
added_by: str | None = None, reasons=None) -> dict[str, Any]:
"""Grow-only record: add any ``cidrs`` not already active to the ledger.
Unlike :func:`sync_blocklist`, this **never removes** entries. It mirrors
the router's cumulative model: once an IP/subnet has ever been blocked it
stays active even after it leaves the scan window (e.g. NPM ~30-day log
retention), exactly like the old git-history backfill. Returns the added
delta and the resulting active count.
"""
cidr_set = set(cidrs)
before = {r["cidr"] for r in active_blocklist(conn)}
to_add = sorted(cidr_set - before)
now = utcnow()
with transaction(conn):
for cidr in to_add:
_block_entry_inner(conn, cidr, added_by=added_by, last_seen=now,
reasons=reasons)
# Refresh last_seen on entries that are still desired so the ledger
# reflects recency for existing rows too.
conn.executemany(
"UPDATE blocklist_entries SET last_seen=? WHERE cidr=? AND status='active'",
[(now, cidr) for cidr in cidr_set.intersection(before)])
return {"added": to_add, "active": len(before) + len(to_add)}
def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *,
added_by: str | None = None) -> dict[str, Any]:
"""Reconcile active blocklist to ``desired_cidrs``.
@@ -465,6 +491,20 @@ def save_firewall_snapshot(conn: sqlite3.Connection, *,
return snapshot_id
def recent_firewall_snapshots(conn: sqlite3.Connection,
limit: int = 10) -> list[dict[str, Any]]:
"""Most recent firewall snapshots with entry counts (F001)."""
rows = conn.execute(
"SELECT fs.id, fs.captured_at, fs.kind, fs.trigger, "
"(SELECT COUNT(*) FROM firewall_address_list_entries e "
" WHERE e.snapshot_id = fs.id) AS addr_entries, "
"(SELECT COUNT(*) FROM firewall_rule_entries r "
" WHERE r.snapshot_id = fs.id) AS rule_entries "
"FROM firewall_snapshots fs ORDER BY fs.id DESC LIMIT ?",
(limit,)).fetchall()
return [dict(r) for r in rows]
# ---------------------------------------------------------------------------
# WHOIS cache
# ---------------------------------------------------------------------------
+37 -7
View File
@@ -42,18 +42,42 @@ from ..utils.whois import consolidate_by_subnet
def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
desired_v6: list[str], added_by: str | None = None) -> None:
"""Best-effort sync of the consolidated blocklist targets into SQLite.
"""Best-effort grow-only sync of block targets into the SQLite ledger.
Keep the DB ledger in sync regardless of whether a router sync happened so
analysis always sees the current policy."""
Uses ``record_blocklist`` (add-only) so the ledger is the authoritative
cumulative record: once an IP is ever blocked it stays active even after it
leaves the scan window, matching the router's cumulative model. Returns the
added delta or None if the DB is unavailable.
"""
try:
from ..db import get_connection
from ..db import repositories as repo
conn = get_connection()
repo.sync_blocklist(conn, list(desired_v4) + list(desired_v6),
added_by=added_by)
return repo.record_blocklist(
conn, list(desired_v4) + list(desired_v6), added_by=added_by)
except Exception as exc: # never break the scan for a DB issue
print(f"WARN: could not sync blocklist to db ({exc})")
return None
def _db_complete_blocklist(desired_v4: list[str], desired_v6: list[str]):
"""Complete cumulative blocklist from the DB ledger (B001).
Returns ``(complete_v4, complete_v6)`` derived from the active blocklist
ledger plus this scan's newly desired targets (in case the DB write for
this scan has not run yet). Falls back to the git-history reconstruction
when the DB is unavailable, so nothing regresses.
"""
try:
from ..db import get_connection
from ..db import repositories as repo
conn = get_connection()
active = [e["cidr"] for e in repo.active_blocklist(conn)]
except Exception:
active = []
union = set(active) | set(desired_v4) | set(desired_v6)
v4, v6 = split_cidr_versions(union)
return v4, v6
@dataclass
@@ -256,9 +280,15 @@ def run_scan(params: ScanParams) -> ScanResult:
f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.")
# Timestamped change report — only when the router was reachable.
complete_v4, complete_v6 = cumulative_blocklist(
blocklist_v4_path, blocklist_v6_path,
# Source of truth for the cumulative set: the SQLite |ledger| first
# (B001), falling back to the git-history reconstruction only if the
# DB is unavailable.
complete_v4, complete_v6 = _db_complete_blocklist(
blocked_ipv4, blocked_ipv6)
if not complete_v4 and not complete_v6:
complete_v4, complete_v6 = cumulative_blocklist(
blocklist_v4_path, blocklist_v6_path,
blocked_ipv4, blocked_ipv6)
blocklist_report_path = write_blocklist_report(
params.report_dir,
history_v4=history_v4, history_v6=history_v6,