feat(blocklist): make SQLite ledger authoritative for the cumulative set (B001)
- Add repo.record_blocklist(): grow-only, never auto-removes (mirrors router cumulative model; preserves entries that leave the NPM log window). - scan.py uses it on the scan path and derives the report's complete set from the ledger (_db_complete_blocklist), falling back to git history only if the DB is unavailable.
This commit is contained in:
1 parent
4a30f6b582
commit
ba48054293
2 files changed
+77
-7
No files matched your search
@@ -322,6 +322,32 @@ def unblock_entry(conn: sqlite3.Connection, cidr: str, *,
|
||||
return removed
|
||||
|
||||
|
||||
def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *,
|
||||
added_by: str | None = None, reasons=None) -> dict[str, Any]:
|
||||
"""Grow-only record: add any ``cidrs`` not already active to the ledger.
|
||||
|
||||
Unlike :func:`sync_blocklist`, this **never removes** entries. It mirrors
|
||||
the router's cumulative model: once an IP/subnet has ever been blocked it
|
||||
stays active even after it leaves the scan window (e.g. NPM ~30-day log
|
||||
retention), exactly like the old git-history backfill. Returns the added
|
||||
delta and the resulting active count.
|
||||
"""
|
||||
cidr_set = set(cidrs)
|
||||
before = {r["cidr"] for r in active_blocklist(conn)}
|
||||
to_add = sorted(cidr_set - before)
|
||||
now = utcnow()
|
||||
with transaction(conn):
|
||||
for cidr in to_add:
|
||||
_block_entry_inner(conn, cidr, added_by=added_by, last_seen=now,
|
||||
reasons=reasons)
|
||||
# Refresh last_seen on entries that are still desired so the ledger
|
||||
# reflects recency for existing rows too.
|
||||
conn.executemany(
|
||||
"UPDATE blocklist_entries SET last_seen=? WHERE cidr=? AND status='active'",
|
||||
[(now, cidr) for cidr in cidr_set.intersection(before)])
|
||||
return {"added": to_add, "active": len(before) + len(to_add)}
|
||||
|
||||
|
||||
def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *,
|
||||
added_by: str | None = None) -> dict[str, Any]:
|
||||
"""Reconcile active blocklist to ``desired_cidrs``.
|
||||
@@ -465,6 +491,20 @@ def save_firewall_snapshot(conn: sqlite3.Connection, *,
|
||||
return snapshot_id
|
||||
|
||||
|
||||
def recent_firewall_snapshots(conn: sqlite3.Connection,
|
||||
limit: int = 10) -> list[dict[str, Any]]:
|
||||
"""Most recent firewall snapshots with entry counts (F001)."""
|
||||
rows = conn.execute(
|
||||
"SELECT fs.id, fs.captured_at, fs.kind, fs.trigger, "
|
||||
"(SELECT COUNT(*) FROM firewall_address_list_entries e "
|
||||
" WHERE e.snapshot_id = fs.id) AS addr_entries, "
|
||||
"(SELECT COUNT(*) FROM firewall_rule_entries r "
|
||||
" WHERE r.snapshot_id = fs.id) AS rule_entries "
|
||||
"FROM firewall_snapshots fs ORDER BY fs.id DESC LIMIT ?",
|
||||
(limit,)).fetchall()
|
||||
return [dict(r) for r in rows]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# WHOIS cache
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
+37
-7
@@ -42,18 +42,42 @@ from ..utils.whois import consolidate_by_subnet
|
||||
|
||||
def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
|
||||
desired_v6: list[str], added_by: str | None = None) -> None:
|
||||
"""Best-effort sync of the consolidated blocklist targets into SQLite.
|
||||
"""Best-effort grow-only sync of block targets into the SQLite ledger.
|
||||
|
||||
Keep the DB ledger in sync regardless of whether a router sync happened so
|
||||
analysis always sees the current policy."""
|
||||
Uses ``record_blocklist`` (add-only) so the ledger is the authoritative
|
||||
cumulative record: once an IP is ever blocked it stays active even after it
|
||||
leaves the scan window, matching the router's cumulative model. Returns the
|
||||
added delta or None if the DB is unavailable.
|
||||
"""
|
||||
try:
|
||||
from ..db import get_connection
|
||||
from ..db import repositories as repo
|
||||
conn = get_connection()
|
||||
repo.sync_blocklist(conn, list(desired_v4) + list(desired_v6),
|
||||
added_by=added_by)
|
||||
return repo.record_blocklist(
|
||||
conn, list(desired_v4) + list(desired_v6), added_by=added_by)
|
||||
except Exception as exc: # never break the scan for a DB issue
|
||||
print(f"WARN: could not sync blocklist to db ({exc})")
|
||||
return None
|
||||
|
||||
|
||||
def _db_complete_blocklist(desired_v4: list[str], desired_v6: list[str]):
|
||||
"""Complete cumulative blocklist from the DB ledger (B001).
|
||||
|
||||
Returns ``(complete_v4, complete_v6)`` derived from the active blocklist
|
||||
ledger plus this scan's newly desired targets (in case the DB write for
|
||||
this scan has not run yet). Falls back to the git-history reconstruction
|
||||
when the DB is unavailable, so nothing regresses.
|
||||
"""
|
||||
try:
|
||||
from ..db import get_connection
|
||||
from ..db import repositories as repo
|
||||
conn = get_connection()
|
||||
active = [e["cidr"] for e in repo.active_blocklist(conn)]
|
||||
except Exception:
|
||||
active = []
|
||||
union = set(active) | set(desired_v4) | set(desired_v6)
|
||||
v4, v6 = split_cidr_versions(union)
|
||||
return v4, v6
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -256,9 +280,15 @@ def run_scan(params: ScanParams) -> ScanResult:
|
||||
f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.")
|
||||
|
||||
# Timestamped change report — only when the router was reachable.
|
||||
complete_v4, complete_v6 = cumulative_blocklist(
|
||||
blocklist_v4_path, blocklist_v6_path,
|
||||
# Source of truth for the cumulative set: the SQLite |ledger| first
|
||||
# (B001), falling back to the git-history reconstruction only if the
|
||||
# DB is unavailable.
|
||||
complete_v4, complete_v6 = _db_complete_blocklist(
|
||||
blocked_ipv4, blocked_ipv6)
|
||||
if not complete_v4 and not complete_v6:
|
||||
complete_v4, complete_v6 = cumulative_blocklist(
|
||||
blocklist_v4_path, blocklist_v6_path,
|
||||
blocked_ipv4, blocked_ipv6)
|
||||
blocklist_report_path = write_blocklist_report(
|
||||
params.report_dir,
|
||||
history_v4=history_v4, history_v6=history_v6,
|
||||
|
||||
Reference in new issue
Block a user