feat(prune): add functionality to remove old single-host IP blocks while keeping CIDR ranges intact

app/db/repositories.py

@@ -11,7 +11,8 @@ from __future__ import annotations
 import json
 import sqlite3
 from contextlib import contextmanager
-from datetime import datetime, timezone
+from datetime import datetime, timedelta, timezone
+import ipaddress
 from typing import Any, Iterator, Sequence
 
 from ..utils.models import IPStats
@@ -104,7 +105,8 @@ def _parse_ts(ts: str) -> tuple[str, float | None]:
     cleaned = ts.strip()
     # Accept a trailing timezone like +HH:MM or Z; fall back to naive parse.
     try:
-        epoch = datetime.fromisoformat(cleaned.replace("Z", "+00:00")).timestamp()
+        epoch = datetime.fromisoformat(
+            cleaned.replace("Z", "+00:00")).timestamp()
     except ValueError:
         try:
             epoch = datetime.fromisoformat(cleaned).timestamp()
@@ -348,6 +350,52 @@ def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *,
     return {"added": to_add, "active": len(before) + len(to_add)}
 
 
+def prune_old_single_ip_blocks(conn: sqlite3.Connection, *,
+                               older_than_days: int = 30,
+                               actor: str | None = "retention") -> list[str]:
+    """Remove active single-host entries older than ``older_than_days``.
+
+    Keeps CIDR ranges/subnets intact. Host entries are detected via
+    ``ipaddress.ip_network(..., strict=False)`` and only removed when the
+    network is exactly one address (/32 for IPv4, /128 for IPv6).
+    """
+    cutoff = datetime.now(timezone.utc) - \
+        timedelta(days=max(0, older_than_days))
+    rows = conn.execute(
+        "SELECT cidr, added_at, last_seen FROM blocklist_entries "
+        "WHERE status='active'").fetchall()
+
+    to_remove: list[str] = []
+    for row in rows:
+        cidr = str(row["cidr"])
+        if not _is_single_host_cidr(cidr):
+            continue
+        seen = _parse_iso_ts(
+            row["last_seen"]) or _parse_iso_ts(row["added_at"])
+        if seen is None:
+            continue
+        if seen < cutoff:
+            to_remove.append(cidr)
+
+    if not to_remove:
+        return []
+
+    now = utcnow()
+    with transaction(conn):
+        conn.executemany(
+            "UPDATE blocklist_entries SET status='removed', removed_at=? "
+            "WHERE cidr=? AND status='active'",
+            [(now, cidr) for cidr in to_remove],
+        )
+        conn.executemany(
+            "INSERT INTO blocklist_events (action, cidr, actor, fired_at, detail) "
+            "VALUES ('remove', ?, ?, ?, ?)",
+            [(cidr, actor, now, f"single-ip-retention>{older_than_days}d")
+             for cidr in to_remove],
+        )
+    return sorted(to_remove)
+
+
 def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *,
                    added_by: str | None = None) -> dict[str, Any]:
     """Reconcile active blocklist to ``desired_cidrs``.
@@ -402,6 +450,26 @@ def _cidr_version(cidr: str) -> int:
         return 4 if ":" not in str(cidr) else 6
 
 
+def _is_single_host_cidr(cidr: str) -> bool:
+    try:
+        net = ipaddress.ip_network(cidr, strict=False)
+        return net.num_addresses == 1
+    except ValueError:
+        return False
+
+
+def _parse_iso_ts(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
 # ---------------------------------------------------------------------------
 # Whitelists
 # ---------------------------------------------------------------------------

app/scan/__init__.py

@@ -41,7 +41,8 @@ from ..utils.whois import consolidate_by_subnet
 
 
 def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
-                          desired_v6: list[str], added_by: str | None = None) -> None:
+                          desired_v6: list[str],
+                          added_by: str | None = None) -> dict | None:
     """Best-effort grow-only sync of block targets into the SQLite ledger.
 
     Uses ``record_blocklist`` (add-only) so the ledger is the authoritative
@@ -53,8 +54,13 @@ def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
         from ..db import get_connection
         from ..db import repositories as repo
         conn = get_connection()
-        return repo.record_blocklist(
+        result = repo.record_blocklist(
             conn, list(desired_v4) + list(desired_v6), added_by=added_by)
+        # Retention policy: expire stale single-host entries after 30 days
+        # while keeping CIDR ranges/subnets intact.
+        pruned = repo.prune_old_single_ip_blocks(conn, older_than_days=30)
+        result["pruned_single_ips"] = pruned
+        return result
     except Exception as exc:  # never break the scan for a DB issue
         print(f"WARN: could not sync blocklist to db ({exc})")
         return None

tests/test_repositories.py

@@ -13,6 +13,7 @@ import os
 import sqlite3
 import tempfile
 import unittest
+from datetime import datetime, timedelta, timezone
 
 from app.db import connect
 from app.db import repositories as repo
@@ -110,7 +111,8 @@ class BlocklistTest(unittest.TestCase):
                                    added_by="scan:1")
         self.assertEqual(r1["active"], 2)
         # a later scan only sees 5.5.5.5; 6.6.6.6 stays (cumulative).
-        r2 = repo.record_blocklist(self.conn, ["5.5.5.5/32"], added_by="scan:2")
+        r2 = repo.record_blocklist(
+            self.conn, ["5.5.5.5/32"], added_by="scan:2")
         active = [e["cidr"] for e in repo.active_blocklist(self.conn)]
         self.assertIn("6.6.6.6/32", active)
         self.assertEqual(r2["active"], 2)
@@ -122,6 +124,38 @@ class BlocklistTest(unittest.TestCase):
                          [e["cidr"] for e in repo.active_blocklist(self.conn)])
         self.assertFalse(repo.unblock_entry(self.conn, "7.7.7.7/32"))
 
+    def test_prune_old_single_ip_blocks_keeps_ranges(self):
+        old = (datetime.now(timezone.utc) - timedelta(days=31)).isoformat(
+            timespec="seconds")
+        with repo.transaction(self.conn):
+            repo._block_entry_inner(self.conn, "1.2.3.4/32", last_seen=old)
+            repo._block_entry_inner(
+                self.conn, "2001:db8::1/128", last_seen=old)
+            repo._block_entry_inner(self.conn, "1.2.3.0/24", last_seen=old)
+            repo._block_entry_inner(self.conn, "2001:db8::/64", last_seen=old)
+
+        removed = repo.prune_old_single_ip_blocks(
+            self.conn, older_than_days=30)
+        self.assertCountEqual(removed, ["1.2.3.4/32", "2001:db8::1/128"])
+
+        active = {e["cidr"] for e in repo.active_blocklist(self.conn)}
+        self.assertIn("1.2.3.0/24", active)
+        self.assertIn("2001:db8::/64", active)
+        self.assertNotIn("1.2.3.4/32", active)
+        self.assertNotIn("2001:db8::1/128", active)
+
+    def test_prune_old_single_ip_blocks_respects_recency(self):
+        fresh = (datetime.now(timezone.utc) - timedelta(days=5)).isoformat(
+            timespec="seconds")
+        with repo.transaction(self.conn):
+            repo._block_entry_inner(self.conn, "9.9.9.9/32", last_seen=fresh)
+
+        removed = repo.prune_old_single_ip_blocks(
+            self.conn, older_than_days=30)
+        self.assertEqual(removed, [])
+        self.assertIn("9.9.9.9/32",
+                      {e["cidr"] for e in repo.active_blocklist(self.conn)})
+
 
 class WhitelistTest(unittest.TestCase):
     def setUp(self):