feat(reports): store and browse report artifacts

B001 change caused 'unsupported operand type(s) for -: list and set' during a
full consolidated scan: _db_complete_blocklist returned split_cidr_versions()
output, which is tuple[list, list], but write_blocklist_report()/blocklist_delta()
expect set operands. Return set(v4), set(v6) to match cumulative_blocklist's
contract. Reproduced the exact report path with a populated ledger and
verified it no longer crashes.

.env.example

@@ -60,6 +60,12 @@ SUBNET_THRESHOLD=3
 REPORT_RETENTION_DAYS=30
 REPORT_RETENTION_MAX_FILES=10
 
+# --- SQLite store ---------------------------------------------------------
+
+# Local SQLite database canonical store (logs, scans, blocklist/whitelist and
+# firewall history). Created automatically on first run. Default: data/ids.db
+#IDS_DB_PATH=/absolute/path/to/ids.db
+
 # --- Scan / flagging defaults ----------------------------------------------
 
 # Rows in the top-IP report tables.

.gitignore

@@ -1,16 +1,31 @@
-# instructions for agents
+# instructions for agents
-.github/instructions/
+.github/instructions/
-.agents
+.agents
-skills-lock.json
+skills-lock.json
-
+
-# environment variables
+# environment variables
-.env
+.env
-
+
-# python cache
+# python cache
-__pycache__/
+__pycache__/
-
+.mypy_cache/
-# app runtime data (feeds, caches, scan state - regenerated each run)
+
-app/data/
+# python virtual environment
-
+.venv/
-# generated reports
+
-app/reports/
+# app runtime data (feeds, caches, scan state - regenerated each run).
+# All of this must stay UNTRACKED so ``git pull`` never conflicts with files a
+# scan has written since the last pull (see .github/instructions/DEPLOYMENT.md).
+# The single allowed tracked seed is the read-only PaloAlto scanner-ips.txt
+# (never rewritten at runtime); the Cloudflare ips-*.txt are runtime-refreshed
+# downloads, so they are NOT tracked and are re-downloaded if absent.
+app/data/*
+!app/data/paloalto/scanner-ips.txt
+
+# SQLite database (canonical store) + WAL/SHM sidecars
+app/data/*.db
+app/data/*.db-wal
+app/data/*.db-shm
+
+# generated reports
+app/reports/

README.md

@@ -49,6 +49,10 @@ intrusion-detection/
 │   ├── scan/            scan pipeline
 │   │   ├── __init__.py  consolidated scanner (run_scan, ScanParams, ScanResult)
 │   │   └── scan_state.py incremental log scan state (load/save)
+│   ├── db/              SQLite store (see .github/instructions/DATABASE.md)
+│   │   ├── __init__.py     connection factory + migrations runner
+│   │   ├── schema.py       DDL for all tables (PRAGMA user_version migrations)
+│   │   └── repositories.py  queries: logs, scans, blocklist, whitelist, firewall
 │   ├── utils/           shared utilities
 │   │   ├── constants.py     all config constants (paths, env vars, thresholds)
 │   │   ├── files.py         read_lines, find_logs, write_csv/json/lines
@@ -67,8 +71,9 @@ intrusion-detection/
 │   ├── cloudflare/
 │   │   ├── ips-v4.txt
 │   │   └── ips-v6.txt
-│   └── paloalto/
+│   ├── paloalto/
-│       └── scanner-ips.txt
+│   │   └── scanner-ips.txt
+│   └── ids.db           local SQLite store (log/scan/blocklist/whitelist history; auto-created)
 ├── reports/             generated reports (per-run + per-source)
 │   ├── copyparty/
 │   ├── npm/
@@ -167,6 +172,11 @@ python app.py scan --quiet
 python app.py scan --update-firewall --dry-run
 python app.py scan --update-firewall
 
+# detailed log-analysis report from the SQLite store (needs scans run first)
+python app.py analyze                  # top IPs + cross-source + blocklist
+python app.py analyze --ip 1.2.3.4     # drill into one IP's request history
+python app.py analyze --json           # machine-readable output
+
 # individual scanners (still runnable directly)
 python -m app.copyparty
 python -m app.npm
@@ -209,29 +219,36 @@ time and is never trimmed automatically.
   it stops appearing in recent logs (for example after it leaves the NPM
   ~30-day log-retention window, or once WHOIS subnet consolidation groups it
   into a larger range).
-- The **source of truth** for "all blocked IPs" is the git history of the
+- The **source of truth** for "all blocked IPs" is the **SQLite blocklist
-  consolidated blocklist files `reports/app/data/blocklist_ipv4.txt` and
+  ledger** (`app.db` → `blocklist_entries` / `blocklist_events`), not git
-  `blocklist_ipv6.txt` (these files are git-tracked). After each scan the
+  history. Each scan syncs the desired blocklist into the ledger; the router's
-  updated blocklist files should be committed so the history keeps growing.
+  cumulative state is derived from it. `app/utils/blocklist_history.py` is a
+  read-only fallback only.
 - The sync only ever **adds** entries. The only automatic removals are router
-  entries that were never blocked by the system — i.e. absent from every git
+  entries that were never blocked by the system — i.e. absent from the ledger
-  version of the blocklist files (manual / rogue entries).
+  (manual / rogue entries).
 - To explicitly unblock an IP, remove it manually via the web UI
   (`/blocklist` → remove) or directly on the router; the automated sync will
   not re-add entries that are not part of a scan.
 
+> **Deployment note:** runtime data (blocklists, reports, the DB) is **never
+> committed to git**. Keeping it untracked is what makes `git pull` updates
+> conflict-free — see [`DEPLOYMENT.md`](.github/instructions/DEPLOYMENT.md)
+> and the `deploy/` scripts.
+
 Each consolidated scan with `--update-firewall` also writes a timestamped
 change report to `reports/app/YYYY-MM-DD-hh-mm-ss.json`: the number and list
 of entries added / removed (complete-set delta), the full cumulative blocked
-list, and the router sync changes.
+list, and the router sync changes. It also persists a firewall snapshot into
+the `firewall_snapshots` SQLite tables for long-term audit (F001).
 
 ### Report retention
 
 Generated files under `reports/` are pruned automatically after each
 consolidated scan: every subdirectory keeps at most
 `REPORT_RETENTION_MAX_FILES` files (default 10), all newer than
-`REPORT_RETENTION_DAYS` days (default 30). The blocklist source of truth
+`REPORT_RETENTION_DAYS` days (default 30). The blocklist files
-(`reports/app/data/blocklist_ipv4.txt` / `blocklist_ipv6.txt`) is always
+(`reports/app/data/blocklist_ipv4.txt` / `blocklist_ipv6.txt`) are always
 protected and never pruned. Tune via `REPORT_RETENTION_DAYS` /
 `REPORT_RETENTION_MAX_FILES`; `--dry-run` previews the cleanup without
 deleting.
@@ -253,6 +270,8 @@ Pages:
 | `/blocklist` | view the consolidated blocklist, add/remove entries   |
 | `/firewall`  | view address lists + rules, preview/apply a sync      |
 | `/whitelist` | view/manage provider whitelist CIDRs                  |
+| `/graph`      | visualize the 0.0.0.0/8 address space (blocked/whitelisted pixels) |
+| `/analysis`   | detailed persisted-log analysis (top IPs, drill-down, cross-source) |
 
 Scans run in a background thread (only one at a time) and report progress via
 the JSON API, so the UI never blocks. Firewall sync is **dry-run by default**
@@ -261,6 +280,35 @@ from the UI — applying changes is an explicit action. The main JSON endpoints
 `/api/firewall/*`, `/api/whitelist`) are also usable directly from a REST
 client.
 
+## Deployment & updating (`git pull`)
+
+The app runs scans continuously (web UI + cron + CLI), and all of it is
+intended to be updated on the server with a plain `git pull`. To make that
+**safe** — i.e. never conflict with a file a scan wrote since the last pull —
+runtime data is strictly **never committed to git**:
+
+- `app/data/*`, `app/reports/*` and the SQLite DB (`app/data/ids.db`) are
+gitignored. The only tracked file under `data/` is the read-only
+`app/data/paloalto/scanner-ips.txt` seed.
+- The Cloudflare `ips-*.txt` whitelist seeds are **not** tracked: they are
+runtime-refreshed downloads and are re-fetched if absent (built-in ranges are
+the fallback).
+- History that must survive — the cumulative blocklist and firewall audit —
+lives in the SQLite store, not in git revisions.
+
+Update a deployed instance with:
+
+```sh
+sudo deploy/deploy.sh
+```
+
+See [`DEPLOYMENT.md`](.github/instructions/DEPLOYMENT.md) for the full plan,
+and `deploy/` for the scripts (`deploy.sh`, `restart-service.sh`,
+`check_runtime_git.sh`, `ids.service` systemd template). The wrapper
+fast-forwards only (`git pull --ff-only`), refuses to run on an unclean tree,
+reinstalls deps if `requirements.txt` changed, applies DB migrations
+(`python -m app.db --migrate`), and restarts the `ids` unit.
+
 ## Output
 
 Each script writes to its own report directory. `app.py` also writes
@@ -311,9 +359,11 @@ reports/
 ```
 
 The current router state (address lists + firewall rules) is also mirrored to
-`data/mikrotik/` on every `--update-firewall` run. Those files are overwritten
+`data/mikrotik/` on every `--update-firewall` run as a scratch pointer to the
-each run and are **git-visible**, so `git diff` shows how the firewall rules
+latest snapshot. Those files are overwritten each run and are **not** tracked;
-and address lists change over time.
+the long-term audit trail is the SQLite `firewall_snapshots` tables plus the
+timestamped reports under `reports/mikrotik/`. `git pull` is therefore never
+blocked by firewall-state files.
 
 ## Detection
 

REPORTS.md

@@ -0,0 +1,142 @@
+# Report Inventory
+
+This inventory records the current report surface before moving report reads
+or writes into SQLite. Generated paths are rooted at `REPORT_DIR` (normally
+`reports/`). The SQLite tables listed below already exist in schema v1 unless
+noted otherwise.
+
+## Artifacts
+
+| Area | Producer and formats | Current role and consumers | Classification |
+| --- | --- | --- | --- |
+| `copyparty/` | `Analyzer.write_report()` creates timestamped `.txt`; `write_data()` creates per-IP CSV/JSON, denied-path CSV, flagged/banned IP TXT, `requests.csv`, `authed_users.json`, and `mikrotik-block.rsc`. | Reports API serves top-level text reports. `requests.csv` is imported by `app.db.backfill`; per-run data supports inspection and legacy workflows. Retention prunes generated files. | Request rows and per-scan IP stats are canonical in `requests` and `ip_stats`; rendered summaries and exports are derived. |
+| `npm/` | Same analyzer outputs, plus user-agent and host summary CSV files. | Same consumers as copyparty. `requests.csv` is a legacy backfill input; retention prunes generated files. | Request rows and per-scan IP stats are canonical in `requests` and `ip_stats`; summaries and exports are derived. |
+| `app/data/` | Consolidated scan writes `blocklist_ipv4.txt` and `blocklist_ipv6.txt`. | Blocklist UI/API reads these files; scan history reconstruction and firewall sync use them as current/compatibility inputs. Retention explicitly protects them. | Operational compatibility exports. Canonical blocked targets and events belong in `blocklist_entries` and `blocklist_events`; keep files until every consumer uses the DB and git-history contract is replaced. |
+| `app/` timestamped JSON | `write_blocklist_report()` emits scan/firewall change reports when the router is reachable. | Human-readable audit trail containing run delta, cumulative targets, router changes, dry-run state, and errors; retained under report retention. | Derived audit artifact. Preserve structured fields and source scan association in the DB before considering file retirement. |
+| `mikrotik/` | `MikrotikFirewall.save_state_report()` writes timestamped JSON snapshots and report data; latest state is also written to `data/mikrotik/*.json`. | Firewall UI/API displays current state and snapshot history. `app.db.backfill` currently imports only the latest `data/mikrotik` snapshot; runtime snapshot persistence uses `firewall_snapshots`, `firewall_address_list_entries`, and `firewall_rule_entries`. | Firewall rows and snapshots are canonical in the DB once persisted. Keep JSON exports during migration and retain any git-diff inputs required by deployment/audit. |
+
+## Consumers
+
+- `GET /api/reports/<source>` lists top-level files in the selected report
+  directory; `GET /api/reports/<source>/<path>` reads a file below it.
+- The reports page consumes those endpoints. Scan status and dashboard helpers
+  select the newest source and MikroTik report files for display.
+- `/api/blocklist` reads the consolidated IPv4/IPv6 text files. Scan history
+  reconstruction and firewall sync also consume those files alongside the
+  SQLite ledger and git history.
+- RouterOS `.rsc` exports and firewall synchronization consume generated
+  targets; these are operational outputs, not disposable presentation files.
+- `app.db.backfill` imports per-source `requests.csv`, blocklist history, and
+  the latest common MikroTik JSON snapshot. It does not currently import all
+  timestamped reports or all historical MikroTik snapshots.
+- Report retention removes old generated artifacts after consolidated scans,
+  subject to configured age/file-count limits; the consolidated blocklist
+  files are protected.
+
+## Restructuring Implications
+
+- Keep `requests`, `ip_stats`, blocklist tables, and firewall snapshot tables
+  as the queryable source of truth for facts they already represent. Avoid
+  copying those facts into a generic report blob.
+- A future report-artifact record should store metadata and structured fields
+  that have no canonical home yet. Rendered text/CSV/JSON should be generated
+  from those records or remain an explicit export.
+- Preserve source paths, generated timestamps, scan IDs, and hashes during
+  import so DB records can be traced back to original files.
+- Do not retire blocklist exports, RouterOS scripts, or source logs until
+  their consumers and audit requirements have explicit replacements.
+
+## Database and API Design
+
+### Canonical Data
+
+Continue treating normalized domain tables as canonical for facts they
+already represent:
+
+- `requests` for parsed request events.
+- `ip_stats` for per-IP/per-scan aggregates.
+- `scans` and `scans_sources` for run status, parameters, and source counts.
+- `blocklist_entries`, `blocklist_reasons`, and `blocklist_events` for block
+  decisions and history.
+- `firewall_snapshots`, `firewall_address_list_entries`, and
+  `firewall_rule_entries` for router state.
+
+Add a `report_artifacts` table only for report-specific summaries that cannot
+be reconstructed from those canonical rows. Proposed columns:
+
+| Column | Purpose |
+| --- | --- |
+| `id` | Integer primary key. |
+| `source` | `copyparty`, `npm`, `app`, or `mikrotik`. Keep as text because `log_sources` currently covers only request sources. |
+| `scan_id` | Nullable link to the generating scan; legacy imports may not have a known scan. |
+| `report_type` | Stable type such as `analyzer_summary`, `blocklist_change`, or `firewall_snapshot_summary`. |
+| `generated_at` | Original generation timestamp when known; otherwise import time plus an explicit unknown/original-time flag in metadata. |
+| `schema_version` | Version of the structured payload shape. |
+| `payload_json` | Structured report-specific summary, not a duplicate of requests or raw logs. |
+| `original_path` | Relative source path for provenance and legacy lookup. |
+| `content_sha256` | Hash of imported source content for integrity and idempotent import. |
+| `created_at` | Database import/creation timestamp. |
+
+Use a unique index on `(original_path, content_sha256)` for idempotent file
+imports. Keep payload schemas type-specific and versioned. Do not copy
+`requests.csv`, `ips.json`, firewall rows, or blocklists wholesale into
+`payload_json`; query their canonical tables and generate exports from those
+tables. If a rendered text report is retained as a database artifact, store
+its report-specific structured inputs and render text on demand rather than
+using opaque text as the only source of truth.
+
+### API Transition Contract
+
+- Add `GET /api/reports` with optional `source`, `type`, `since`, `until`,
+  `limit`, and opaque `cursor` filters. Return a stable list of artifact
+  metadata: `id`, `source`, `scan_id`, `type`, `generated_at`, and available
+  export formats. Enforce a bounded default and maximum page size.
+- Add `GET /api/report-artifacts/<artifact_id>` for the structured payload and
+  provenance metadata. Use a distinct path because `/api/reports/<source>`
+  already owns the one-segment route shape.
+- Add `GET /api/report-artifacts/<artifact_id>/export` for supported report types and formats
+  (initially JSON and, where useful, CSV/TXT). Generate exports from canonical
+  tables and versioned artifact payloads; do not accept filesystem paths from
+  a database-backed export request.
+- Preserve the current source/path routes as a read-only compatibility
+  fallback while files remain. The reports page should consume DB metadata
+  first, then offer legacy files when no imported artifact exists. Return
+  stable JSON errors for unknown IDs, unsupported formats, and unavailable
+  DB state.
+- Keep ordering deterministic (`generated_at DESC`, then `id DESC`), validate
+  filter values, and use cursor pagination for large histories. Do not expose
+  absolute server paths in list responses.
+
+### Sensitive and Large Content
+
+Do not include raw request lines or full request histories in report payloads;
+those belong in the dedicated log-storage design and require separate access
+and retention controls. Keep high-volume CSV exports generated on demand. Treat
+IP addresses, paths/query strings, user names, user agents, and firewall state
+as sensitive operational data: the existing web server must remain
+localhost-only, payloads should include only fields needed by the report, and
+new endpoints must not expose raw log data by default. Backups and exported
+files inherit the same access and retention requirements as the database.
+
+## Migration Implementation Status
+
+- Schema v2 adds `report_artifacts` with source, scan link, report type,
+  generation time, payload version, original relative path, and SHA-256 hash.
+- `python -m app.db.backfill --dry-run --reports-only` inventories supported
+  report files by source and format. The importer is idempotent, preserves
+  malformed JSON as text with a parse error, reports unreadable/oversized
+  files, and never modifies source files. It skips exports owned by canonical
+  tables (for example `requests.csv` and current blocklists).
+- New analyzer text summaries and consolidated blocklist change reports are
+  written to SQLite and their existing files. Firewall snapshots continue to
+  be persisted in the normalized firewall tables.
+- The Reports page lists database artifacts first, supports cursor pagination
+  and JSON/CSV/TXT downloads, and retains the existing file-backed routes as a
+  fallback for unmigrated files.
+- Validation covers idempotency, malformed/oversized inputs, source/format
+  counts, backup/restore, dual-write payloads, and API fallback. A workspace
+  dry-run found 76 files: 60 eligible, 16 canonical exports skipped, and none
+  unmappable at the time of the check.
+- No report files have been retired. Keep this gate closed until every listed
+  consumer has moved or has an explicit compatibility requirement, and a
+  production backup/restore and parity review has been approved.

app.py

@@ -40,12 +40,22 @@ def _run_scan(argv: list[str]) -> None:
     scan_main(argv)
 
 
+def _run_analyze(argv: list[str]) -> None:
+    from app.analyze import main as analyze_main
+    analyze_main(argv)
+
+
 def main() -> None:
     # `python app.py scan ...` runs the consolidated CLI scan.
     if len(sys.argv) > 1 and sys.argv[1] == "scan":
         _run_scan(sys.argv[2:])
         return
 
+    # `python app.py analyze ...` prints a detailed report from the SQLite store.
+    if len(sys.argv) > 1 and sys.argv[1] == "analyze":
+        _run_analyze(sys.argv[2:])
+        return
+
     # Default: start the web UI (parse_known_args tolerates stray flags).
     args, _ = _web_parser().parse_known_args()
     _run_web(args.web_host, args.web_port)

app/analyze.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""CLI detailed log-analysis report (L002).
+
+``python app.py analyze [--ip IP] [--since ISO] [--top N] [--json]`` reads the
+persisted SQLite store (request history, per-IP aggregates, blocklist ledger)
+and prints a detailed, interactive analysis. It complements the web UI's
+``/analysis`` page for use on a headless server or in scripts.
+
+No scan is run here: this only queries what ``scan`` has already persisted.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime
+
+
+def _fmt_ts(epoch: float | None, default: str = "-") -> str:
+    if not epoch:
+        return default
+    try:
+        return datetime.fromtimestamp(epoch).isoformat(timespec="seconds")
+    except (ValueError, OverflowError, OSError):
+        return default
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="python app.py analyze",
+        description="Detailed log-analysis report from the SQLite store.")
+    p.add_argument("--ip", metavar="IP", default=None,
+                   help="focus the report on a single IP (drill-down)")
+    p.add_argument("--since", metavar="ISO", default=None,
+                   help="only analyse requests at/after this ISO timestamp")
+    p.add_argument("--top", type=int, default=20,
+                   help="rows in the top-IP / threat tables (default: 20)")
+    p.add_argument("--json", action="store_true",
+                   help="emit raw JSON instead of a human-readable report")
+    return p
+
+
+def _report(conn, opts) -> dict:
+    from .db import repositories as repo
+
+    data: dict = {}
+    data["scans"] = repo.latest_scans(conn, limit=10)
+    data["totals"] = {
+        "requests": repo.count_requests(conn),
+        "scans": repo.count_scans(conn),
+    }
+
+    if opts.ip:
+        data["ip"] = {
+            "ip": opts.ip,
+            "timeline": repo.ip_timeline(conn, ip=opts.ip),
+            "active_blocklist": [e for e in repo.active_blocklist(conn)
+                                 if e["cidr"].startswith(opts.ip)],
+        }
+    else:
+        data["top_ips"] = repo.top_ips(conn, since=opts.since, limit=opts.top)
+        data["cross_source"] = repo.correlate_across_sources(
+            conn, limit=opts.top)
+        data["blocklist"] = repo.active_blocklist(conn)
+        data["recent_errors"] = repo.query_requests(
+            conn, since=opts.since, limit=opts.top)
+    return data
+
+
+def _render_text(data: dict, opts) -> str:
+    lines = ["Detailed log analysis", "=" * 40, ""]
+    totals = data["totals"]
+    lines.append(f"SQLite store: {totals['requests']} requests across "
+                 f"{totals['scans']} scan(s)")
+    if data["scans"]:
+        lines.append("Latest scans:")
+        for s in data["scans"]:
+            lines.append(f"  #{s['id']:<4} {s['kind']:<8} {s['status']:<7} "
+                         f"{s.get('started') or s.get('started_at') or '-'}")
+    lines.append("")
+
+    if "ip" in data:
+        ip = data["ip"]
+        lines.append(f"== Request history for {ip['ip']} ==")
+        bl = ip["active_blocklist"]
+        if bl:
+            lines.append(f"  (source: active blocklist entries "
+                         f"{', '.join(e['cidr'] for e in bl)})")
+        lines.append(f"{'time':<24} {'src':<9} {'method':<8} {'status':<6} path")
+        for r in ip["timeline"][:opts.top]:
+            lines.append(
+                f"{r.get('ts') or '-':<24} {str(r.get('source')):<9} "
+                f"{str(r.get('method') or '-'):<8} "
+                f"{str(r.get('status') or '-'):<6} {r.get('path') or '-'}")
+    else:
+        lines.append("== Top IPs ==")
+        lines.append(
+            f"{'ip':<18} {'ver':<3} {'reqs':<6} {'err':<5} {'mal':<5} "
+            f"{'first':<22} last")
+        for r in data["top_ips"]:
+            lines.append(
+                f"{str(r['ip']):<18} {r.get('version') or '-':<3} "
+                f"{r['total']:<6} {r['errors']:<5} {r['malicious_hits']:<5} "
+                f"{_fmt_ts(r['first_epoch']):<22} {_fmt_ts(r['last_epoch'])}")
+        lines.append("")
+
+        lines.append("== Cross-source IPs ==")
+        if data["cross_source"]:
+            for r in data["cross_source"]:
+                lines.append(
+                    f"  {r['ip']:<18} sources={r['sources']} "
+                    f"({r['source_list']}) total={r['total']}")
+        else:
+            lines.append("  (none — no IP seen in more than one log source)")
+
+        lines.append("")
+        lines.append("== Active blocklist (ledger) ==")
+        if data["blocklist"]:
+            for e in data["blocklist"]:
+                lines.append(f"  {e['cidr']:<22} added={e['added_at']}")
+        else:
+            lines.append("  (empty)")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> None:
+    opts = _build_parser().parse_args(argv)
+    try:
+        from .db import get_connection
+        conn = get_connection()
+        data = _report(conn, opts)
+    except Exception as exc:
+        print(f"error: could not read SQLite store ({exc}); "
+              f"run a scan first so data exists.", file=sys.stderr)
+        sys.exit(1)
+
+    if opts.json:
+        print(json.dumps(data, indent=2, sort_keys=True, default=str))
+    else:
+        print(_render_text(data, opts))
+
+
+if __name__ == "__main__":
+    main()

app/analyzer.py

@@ -7,6 +7,7 @@
 
 from __future__ import annotations
 
+import hashlib
 import os
 import re
 from collections import Counter
@@ -60,11 +61,66 @@ class Analyzer:
         self.flagged: dict[str, IPStats] = {}
         self.per_file: dict[str, int] = {}
 
+    # --- sqlite persistence ------------------------------------------------
+
+    def persist_to_db(self, scan_id: int | None = None) -> None:
+        """Ingest this analyzer's requests + per-IP stats into SQLite.
+
+        Lazily imported to avoid cycles. When ``scan_id`` is None a standalone
+        scan record is created for this analyzer so individual
+        (non-consolidated) runs are still tracked. Never raises: a DB failure
+        logs a warning but does not break the scan.
+        """
+        if not self.requests and not self.stats:
+            return
+        try:
+            from .db import get_connection
+            from .db import repositories as repo
+            from .utils.ip_utils import ip_version as _ip_ver
+            conn = get_connection()
+            if scan_id is None:
+                scan_id = repo.start_scan(conn, self.name,
+                                          params={"full": self.full})
+            source = self.name
+            rows: list[dict] = []
+            for r in self.requests:
+                rows.append({
+                    "source": source, "scan_id": scan_id,
+                    "log_file": ((r.extra or {}).get("log_file")
+                                  or (self.files[0] if self.files else None)),
+                    "ts": r.ts, "ip": r.ip,
+                    "version": _ip_ver(r.ip) or 0,
+                    "method": r.method, "path": r.path,
+                    "status": r.status,
+                    "upstream_status": r.upstream_status,
+                    "host": r.host, "user": r.user, "ua": r.ua,
+                    "referer": r.referer, "denied": r.denied,
+                    "banned": r.banned, "is_error": r.is_error,
+                    "extra": r.extra,
+                })
+            repo.insert_requests(conn, rows)
+            repo.insert_ip_stats(conn, scan_id,
+                                 list(self.stats.values()),
+                                 set(self.block_candidates()))
+            repo.set_scan_source_counts(
+                conn, scan_id, source, len(self.requests), len(self.files))
+            repo.finish_scan(conn, scan_id, status="done",
+                             log_files=len(self.files),
+                             request_count=len(self.requests))
+        except Exception as exc:  # never break the scan because of DB issues
+            if not self.quiet:
+                print(f"WARN: sqlite ingest skipped ({exc})")
     # --- hooks -------------------------------------------------------------
 
     def parse_line(self, line: str) -> Request | None:
         raise NotImplementedError
 
+    def attr_source(self, req: Request, log_file: str) -> None:
+        """Record the originating log file onto a parsed request."""
+        if req.extra is None:
+            req.extra = {}
+        req.extra["log_file"] = req.extra.get("log_file", log_file)
+
     def flag(self, stats: dict[str, IPStats]) -> dict[str, IPStats]:
         raise NotImplementedError
 
@@ -111,6 +167,7 @@ class Analyzer:
                 for line in read_lines(full_path):
                     req = self.parse_line(line)
                     if req is not None:
+                        self.attr_source(req, name)
                         self.requests.append(req)
                         count += 1
             else:
@@ -125,6 +182,7 @@ class Analyzer:
                 for line, resume in read_lines_from(full_path, start):
                     req = self.parse_line(line)
                     if req is not None:
+                        self.attr_source(req, name)
                         self.requests.append(req)
                         count += 1
                     last_resume = resume
@@ -262,7 +320,7 @@ class Analyzer:
 
     # --- entrypoint --------------------------------------------------------
 
-    def run(self) -> dict[str, Any]:
+    def run(self, db_scan_id: int | None = None) -> dict[str, Any]:
         validate_dir(self.log_path)
         ensure_dirs(self.report_dir, self.data_dir)
         refresh_whitelists()
@@ -273,8 +331,13 @@ class Analyzer:
         if not self.quiet:
             print(self.text_report())
 
+        # Ingest into the SQLite store (creates its own scan record when no
+        # consolidated ``db_scan_id`` is supplied).
+        self.persist_to_db(scan_id=db_scan_id)
+
         self.write_data()
         report_path = self.write_report()
+        self.persist_report_artifact(report_path, db_scan_id)
         print(f"data files:     {self.data_dir}")
         print(f"summary report: {report_path}")
 
@@ -288,3 +351,28 @@ class Analyzer:
             result.update(rsc=rsc, blocked=n_blocked,
                           whitelisted=n_whitelisted)
         return result
+
+    def persist_report_artifact(self, path: str,
+                                scan_id: int | None = None) -> None:
+        """Store the rendered summary in SQLite while keeping its file export."""
+        try:
+            from .db import get_connection
+            from .db import repositories as repo
+            with open(path, "rb") as file:
+                content = file.read()
+            stat = os.stat(path)
+            relative = os.path.relpath(path, self.report_dir).replace(os.sep, "/")
+            repo.insert_report_artifact(
+                get_connection(), source=self.name,
+                report_type="analyzer_summary",
+                generated_at=datetime.fromtimestamp(
+                    stat.st_mtime).astimezone().isoformat(timespec="seconds"),
+                payload={"format": "text",
+                         "content": content.decode("utf-8", errors="replace")},
+                original_path=f"{self.name}/{relative}",
+                content_sha256=hashlib.sha256(content).hexdigest(),
+                scan_id=scan_id,
+            )
+        except Exception as exc:
+            if not self.quiet:
+                print(f"WARN: report artifact DB write skipped ({exc})")

app/data/paloalto/scanner-ips.txt

@@ -0,0 +1,12 @@
+35.203.210.0/23
+144.86.173.0/24
+147.185.132.0/23
+162.216.149.0/24
+162.216.150.0/24
+172.105.147.0/24
+198.235.24.0/24
+205.210.31.0/24
+216.25.88.0/21
+2604:a940:300:5b6:0:0:0:0/64 
+2604:a940:301:225:0:0:0:0/64 
+2604:a940:302:118:0:0:0:0/64

app/db/__init__.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Local SQLite database layer for the intrusion-detection system.
+
+Owns the shared connection factory (WAL mode, busy timeout, single mutable
+connection reused across the background-scan thread and web readers) and
+applies schema migrations on open. Query helpers live in :mod:`repositories`.
+"""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+import threading
+
+from ..utils.constants import DATA_DB_PATH
+from .schema import apply_migrations
+
+_connection: sqlite3.Connection | None = None
+_lock = threading.RLock()
+
+
+def connect(path: str | None = None) -> sqlite3.Connection:
+    """Open an initialized SQLite connection at ``path``.
+
+    Enables WAL for concurrent readers during the single-writer background
+    scan and sets a busy timeout so short write contention does not raise.
+    Foreign keys are on and migrations are applied.
+    """
+    db_path = path or DATA_DB_PATH
+    os.makedirs(os.path.dirname(os.path.abspath(db_path)) or ".", exist_ok=True)
+    conn = sqlite3.connect(
+        db_path,
+        check_same_thread=False,  # background scan thread + web handlers
+        isolation_level=None,     # autocommit; use explicit BEGIN for batches
+    )
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=5000")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.row_factory = sqlite3.Row
+    apply_migrations(conn)
+    return conn
+
+
+def get_connection() -> sqlite3.Connection:
+    """Return the shared, lazily-created connection (thread-safe)."""
+    global _connection
+    with _lock:
+        if _connection is None:
+            _connection = connect()
+        return _connection
+
+
+def close() -> None:
+    """Close the shared connection, if any."""
+    global _connection
+    with _lock:
+        if _connection is not None:
+            try:
+                _connection.close()
+            except sqlite3.Error:
+                pass
+            _connection = None

app/db/__main__.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""CLI entry point for the SQLite store.
+
+Usage:
+    python -m app.db --migrate        applies pending migrations (idempotent)
+    python -m app.db --info           prints schema version + table counts, exits
+    python -m app.db --path           prints the resolved DB path, exits
+
+Used by ``deploy.sh`` (D004) to ensure the DB is up to date after a ``git pull``
+without needing the web server up.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from ..utils.constants import DATA_DB_PATH
+from .schema import SCHEMA_VERSION, apply_migrations
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m app.db",
+        description="Intrusion-detection SQLite store utilities.")
+    parser.add_argument("--migrate", action="store_true",
+                        help="apply pending schema migrations (idempotent)")
+    parser.add_argument("--info", action="store_true",
+                        help="print schema version + table/row counts")
+    parser.add_argument("--path", action="store_true",
+                        help="print the resolved DB path and exit")
+    args = parser.parse_args(argv)
+
+    if args.path:
+        print(DATA_DB_PATH)
+        return 0
+
+    from . import get_connection
+
+    conn = get_connection()
+    current = conn.execute("PRAGMA user_version").fetchone()[0]
+
+    if args.info:
+        counts = {}
+        tables = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' "
+            "AND name NOT LIKE 'sqlite_%' ORDER BY name").fetchall()
+        for t in tables:
+            name = t["name"]
+            try:
+                counts[name] = conn.execute(
+                    f"SELECT COUNT(*) FROM \"{name}\"").fetchone()[0]
+            except Exception:
+                counts[name] = None
+        print(f"db_path:  {DATA_DB_PATH}")
+        print(f"version:  {current} (latest {SCHEMA_VERSION})")
+        print("tables:")
+        for name, n in counts.items():
+            print(f"  {name:<32} {n} rows")
+        return 0
+
+    if args.migrate or current < SCHEMA_VERSION:
+        before = current
+        apply_migrations(conn)
+        print(f"schema version {before} -> "
+              f"{conn.execute('PRAGMA user_version').fetchone()[0]}")
+        return 0
+
+    print(f"schema is already up to date (version {current})")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

app/db/backfill.py

@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+"""R001 — one-time backfill of legacy runtime data into the SQLite store.
+
+Imports existing JSON/CSV/git-history data so the DB becomes the authoritative
+source without losing anything already accumulated on a deployed instance:
+
+- cumulative blocklist from the git-history reconstruction (old source of
+  truth) -> ``blocklist_entries`` (grow-only ledger)
+- firewall snapshots from ``data/mikrotik/*.json`` -> ``firewall_snapshots``
+- per-source requests from ``reports/<source>/data/requests.csv`` (best effort)
+
+Idempotent: skips rows already present. Safe to run repeatedly.
+
+Usage:
+    python -m app.db.backfill [--dry-run]
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import os
+import re
+import sys
+from datetime import datetime, timezone
+
+from ..utils.constants import (
+    APP_REPORT_DIR,
+    APP_REPORT_DATA_DIR,
+    DATA_DIR,
+    MIKROTIK_DATA_DIR,
+    MIKROTIK_REPORT_DIR,
+    NPM_REPORT_DIR,
+    COPYPARTY_REPORT_DIR,
+)
+
+MAX_REPORT_ARTIFACT_BYTES = 10 * 1024 * 1024
+CANONICAL_REPORT_EXPORTS = {
+    "requests.csv", "ips.json", "ips_ipv4.csv", "ips_ipv6.csv",
+    "blocked_ipv4.txt", "blocked_ipv6.txt", "address_list_ipv4.json",
+    "address_list_ipv6.json", "firewall_ipv4.json", "firewall_ipv6.json",
+}
+
+
+def utcnow() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _read_cidr_lines(path: str) -> set[str]:
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return {line.strip() for line in f if line.strip()}
+    except OSError:
+        return set()
+
+
+def backfill_blocklist(conn, dry_run: bool = False) -> dict:
+    """Import the legacy cumulative blocklist into the ledger."""
+    from . import repositories as repo
+    from ..utils.blocklist_history import (
+        committed_lines, current_file_lines,
+    )
+
+    v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
+    v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
+    # Legacy source of truth: union of all git revisions + working file +
+    # (there is no earlier scan target to add here).
+    complete_v4 = committed_lines(v4_path) | current_file_lines(v4_path)
+    complete_v6 = committed_lines(v6_path) | current_file_lines(v6_path)
+    cidrs = sorted(complete_v4 | complete_v6)
+    if not cidrs:
+        return {"blocklist": {"cidrs": 0, "added": 0}}
+    if dry_run:
+        return {"blocklist": {"cidrs": len(cidrs), "added": None, "dry_run": True}}
+    result = repo.record_blocklist(conn, cidrs, added_by="backfill")
+    return {"blocklist": result}
+
+
+def backfill_firewall(conn, dry_run: bool = False) -> dict:
+    """Import the latest firewall snapshot from ``data/mikrotik/*.json``."""
+    from . import repositories as repo
+
+    def _load(name: str) -> list:
+        try:
+            with open(os.path.join(MIKROTIK_DATA_DIR, name), "r",
+                      encoding="utf-8") as f:
+                data = json.load(f)
+            return data if isinstance(data, list) else []
+        except (OSError, ValueError):
+            return []
+
+    addr_v4 = _load("address_list_ipv4.json")
+    addr_v6 = _load("address_list_ipv6.json")
+    fw_v4 = _load("firewall_ipv4.json")
+    fw_v6 = _load("firewall_ipv6.json")
+    if not (addr_v4 or addr_v6 or fw_v4 or fw_v6):
+        return {"firewall": {"dry_run": dry_run, "snapshot": None}}
+    # Idempotent: if a firewall snapshot already exists (previous backfill),
+    # don't create another copy of the same latest state.
+    n = conn.execute(
+        "SELECT COUNT(*) FROM firewall_snapshots").fetchone()[0]
+    if n:
+        return {"firewall": {"dry_run": dry_run, "snapshot": "exists",
+                             "skipped": True}}
+    if dry_run:
+        return {"firewall": {"dry_run": True, "snapshot": None}}
+    sid = repo.save_firewall_snapshot(
+        conn, address_lists_v4=addr_v4, address_lists_v6=addr_v6,
+        rules_v4=fw_v4, rules_v6=fw_v6)
+    return {"firewall": {"snapshot": sid}}
+
+
+def backfill_requests(conn, dry_run: bool = False) -> dict:
+    """Best-effort import of per-source ``requests.csv`` files.
+
+    Only ingests rows that don't already exist (keyed on source+log_file+ts+ip).
+    Legacy CSV columns vary by source; we keep a conservative mapping and never
+    fail the whole backfill over a single bad file.
+    """
+    from . import repositories as repo
+
+    seen = set()
+    for r in conn.execute(
+            "SELECT source, log_file, ts, ip FROM requests"):
+        seen.add((r["source"], r["log_file"], r["ts"], r["ip"]))
+
+    candidates = [
+        (os.path.join(COPYPARTY_REPORT_DIR, "data", "requests.csv"), "copyparty"),
+        (os.path.join(NPM_REPORT_DIR, "data", "requests.csv"), "npm"),
+    ]
+    total = 0
+    for path, source in candidates:
+        if not os.path.exists(path):
+            continue
+        try:
+            with open(path, "r", encoding="utf-8", errors="replace") as f:
+                reader = csv.DictReader(f)
+                rows = []
+                for row in reader:
+                    ip = (row.get("ip") or "").strip()
+                    ts = row.get("ts") or row.get("timestamp") or ""
+                    log_file = row.get("log_file") or row.get("file") or ""
+                    if not ip:
+                        continue
+                    key = (source, log_file, ts, ip)
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    rows.append({
+                        "source": source, "scan_id": None, "log_file": log_file,
+                        "ts": ts, "ip": ip,
+                        "version": 4 if ":" not in ip else 6,
+                        "method": row.get("method"), "path": row.get("path"),
+                        "status": _int_or_none(row.get("status")),
+                        "upstream_status": _int_or_none(
+                            row.get("upstream_status")),
+                        "host": row.get("host"), "user": row.get("user"),
+                        "ua": row.get("ua") or row.get("user_agent"),
+                        "referer": row.get("referer"),
+                        "denied": _bool_str(row.get("denied")),
+                        "banned": _bool_str(row.get("banned")),
+                        "is_error": _bool_str(row.get("is_error")),
+                        "extra": None,
+                    })
+                    total += 1
+            if not dry_run and total:
+                repo.insert_requests(conn, rows)
+        except Exception as exc:
+            print(f"  WARN: requests.csv backfill {source}: {exc}")
+    return {"requests": {"imported": total, "dry_run": dry_run}}
+
+
+def backfill_report_artifacts(conn, dry_run: bool = False,
+                              roots: dict[str, str] | None = None) -> dict:
+    """Import supported report files as versioned, hash-addressed artifacts.
+
+    Canonical event exports and current firewall/blocklist data are skipped;
+    those belong in their normalized tables. Original files are never changed
+    or removed. Oversized or unreadable files are reported for manual review.
+    """
+    from . import repositories as repo
+
+    roots = roots or {
+        "copyparty": COPYPARTY_REPORT_DIR,
+        "npm": NPM_REPORT_DIR,
+        "app": APP_REPORT_DIR,
+        "mikrotik": MIKROTIK_REPORT_DIR,
+    }
+    result = {
+        "discovered": 0, "imported": 0, "would_import": 0,
+        "already_imported": 0, "skipped_canonical": [], "unmapped": [],
+        "by_source": {}, "by_format": {}, "dry_run": dry_run,
+    }
+
+    def count(bucket: str, key: str, field: str) -> None:
+        stats = result[bucket].setdefault(key, {})
+        stats[field] = stats.get(field, 0) + 1
+
+    for source, root in roots.items():
+        if not os.path.isdir(root):
+            continue
+        for directory, _, filenames in os.walk(root):
+            for filename in sorted(filenames):
+                path = os.path.join(directory, filename)
+                suffix = os.path.splitext(filename)[1].lower()
+                if suffix not in (".txt", ".json", ".csv", ".rsc"):
+                    continue
+                relative = os.path.relpath(path, root).replace(os.sep, "/")
+                result["discovered"] += 1
+                report_format = suffix.lstrip(".")
+                count("by_source", source, "discovered")
+                count("by_format", report_format, "discovered")
+                if filename.lower() in CANONICAL_REPORT_EXPORTS:
+                    result["skipped_canonical"].append(
+                        {"source": source, "path": relative})
+                    count("by_source", source, "skipped_canonical")
+                    count("by_format", report_format, "skipped_canonical")
+                    continue
+
+                try:
+                    size = os.path.getsize(path)
+                    if size > MAX_REPORT_ARTIFACT_BYTES:
+                        raise ValueError(
+                            f"file exceeds {MAX_REPORT_ARTIFACT_BYTES} bytes")
+                    with open(path, "rb") as file:
+                        raw = file.read()
+                    text = raw.decode("utf-8", errors="replace")
+                    payload = _report_payload(suffix, text)
+                    digest = hashlib.sha256(raw).hexdigest()
+                    modified = datetime.fromtimestamp(
+                        os.path.getmtime(path), timezone.utc)
+                    generated_at = _report_generated_at(filename, modified)
+                except (OSError, ValueError) as exc:
+                    result["unmapped"].append(
+                        {"source": source, "path": relative,
+                         "reason": str(exc)})
+                    count("by_source", source, "unmapped")
+                    count("by_format", report_format, "unmapped")
+                    continue
+
+                original_path = f"{source}/{relative}"
+                exists = conn.execute(
+                    "SELECT 1 FROM report_artifacts "
+                    "WHERE original_path=? AND content_sha256=?",
+                    (original_path, digest),
+                ).fetchone()
+                if exists:
+                    result["already_imported"] += 1
+                    count("by_source", source, "already_imported")
+                    count("by_format", report_format, "already_imported")
+                    continue
+                if dry_run:
+                    result["would_import"] += 1
+                    count("by_source", source, "would_import")
+                    count("by_format", report_format, "would_import")
+                    continue
+
+                try:
+                    repo.insert_report_artifact(
+                        conn, source=source,
+                        report_type=_report_type(suffix),
+                        generated_at=generated_at, payload=payload,
+                        original_path=original_path,
+                        content_sha256=digest,
+                    )
+                    result["imported"] += 1
+                    count("by_source", source, "imported")
+                    count("by_format", report_format, "imported")
+                except (OSError, ValueError, TypeError) as exc:
+                    result["unmapped"].append(
+                        {"source": source, "path": relative,
+                         "reason": str(exc)})
+                    count("by_source", source, "unmapped")
+                    count("by_format", report_format, "unmapped")
+    return {"reports": result}
+
+
+def _report_payload(suffix: str, text: str) -> dict:
+    if suffix == ".json":
+        try:
+            return {"format": "json", "data": json.loads(text)}
+        except ValueError as exc:
+            return {"format": "json", "content": text,
+                    "parse_error": str(exc)}
+    if suffix == ".csv":
+        try:
+            reader = csv.DictReader(text.splitlines())
+            return {"format": "csv", "columns": reader.fieldnames or [],
+                    "rows": list(reader)}
+        except csv.Error as exc:
+            return {"format": "csv", "content": text,
+                    "parse_error": str(exc)}
+    return {"format": "text", "content": text}
+
+
+def _report_type(suffix: str) -> str:
+    return {".json": "json_report", ".csv": "csv_report",
+            ".rsc": "routeros_script"}.get(suffix, "text_report")
+
+
+def _report_generated_at(filename: str, modified: datetime) -> str:
+    match = re.search(
+        r"(?P<date>\d{4}-\d{2}-\d{2})(?:[-_T](?P<hour>\d{2})[-_:]"
+        r"(?P<minute>\d{2})[-_:](?P<second>\d{2}))?", filename)
+    if not match:
+        return modified.isoformat(timespec="seconds")
+    values = match.groupdict()
+    hour = int(values["hour"] or 0)
+    minute = int(values["minute"] or 0)
+    second = int(values["second"] or 0)
+    try:
+        parsed = datetime.strptime(values["date"], "%Y-%m-%d").replace(
+            hour=hour, minute=minute, second=second, tzinfo=timezone.utc)
+    except ValueError:
+        return modified.isoformat(timespec="seconds")
+    return parsed.isoformat(timespec="seconds")
+
+
+def _int_or_none(v):
+    try:
+        if v in (None, "", "-"):
+            return None
+        return int(v)
+    except (TypeError, ValueError):
+        return None
+
+
+def _bool_str(v):
+    if isinstance(v, str):
+        return int(v.strip().lower() in ("1", "true", "yes", "t"))
+    return int(bool(v))
+
+
+def main(argv: list[str] | None = None) -> None:
+    p = argparse.ArgumentParser(prog="python -m app.db.backfill",
+                                description="Backfill legacy runtime data into SQLite.")
+    p.add_argument("--dry-run", action="store_true",
+                   help="report what would be imported without writing")
+    p.add_argument("--reports-only", action="store_true",
+                   help="import legacy report artifacts only")
+    opts = p.parse_args(argv)
+
+    from . import get_connection
+    conn = get_connection()
+    print(f"== backfill (dry_run={opts.dry_run}) ==")
+    if opts.reports_only:
+        print("reports :", backfill_report_artifacts(
+            conn, dry_run=opts.dry_run))
+        print("== done ==")
+        return
+    print("blocklist:", backfill_blocklist(conn, dry_run=opts.dry_run))
+    print("firewall :", backfill_firewall(conn, dry_run=opts.dry_run))
+    print("requests :", backfill_requests(conn, dry_run=opts.dry_run))
+    print("reports  :", backfill_report_artifacts(conn, dry_run=opts.dry_run))
+    print("== done ==")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

app/db/repositories.py

@@ -0,0 +1,690 @@
+#!/usr/bin/env python3
+"""Query helpers for the intrusion-detection SQLite store.
+
+Functions are grouped by domain and take an explicit ``sqlite3.Connection`` so
+callers control transaction boundaries. Bulk inserts are batched with
+``executemany`` inside a single transaction for speed.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import base64
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+import ipaddress
+from typing import Any, Iterator, Sequence
+
+from ..utils.models import IPStats
+
+
+def utcnow() -> str:
+    """ISO-8601 timestamp with seconds precision (UTC)."""
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+@contextmanager
+def transaction(conn: sqlite3.Connection) -> Iterator[sqlite3.Connection]:
+    """Context manager for an explicit transaction (commit or rollback)."""
+    conn.execute("BEGIN")
+    try:
+        yield conn
+    except Exception:
+        conn.execute("ROLLBACK")
+        raise
+    else:
+        conn.execute("COMMIT")
+
+
+# ---------------------------------------------------------------------------
+# Scans
+# ---------------------------------------------------------------------------
+
+def start_scan(conn: sqlite3.Connection, kind: str, *,
+               params: dict[str, Any] | None = None,
+               full: bool = False, dry_run: bool = False) -> int:
+    """Record a scan start and return its id."""
+    cur = conn.execute(
+        "INSERT INTO scans (kind, started_at, status, full, dry_run, params_json) "
+        "VALUES (?, ?, 'running', ?, ?, ?)",
+        (kind, utcnow(), int(full), int(dry_run),
+         json.dumps(params) if params is not None else None))
+    return int(cur.lastrowid)
+
+
+def finish_scan(conn: sqlite3.Connection, scan_id: int, *,
+                status: str, result: dict[str, Any] | None = None,
+                log_files: int = 0, request_count: int = 0,
+                error: str | None = None) -> None:
+    """Mark a scan complete with outcome counters."""
+    conn.execute(
+        "UPDATE scans SET finished_at=?, status=?, result_json=?, "
+        "log_files=?, request_count=?, error=? WHERE id=?",
+        (utcnow(), status,
+         json.dumps(result) if result is not None else None,
+         log_files, request_count, error, scan_id))
+
+
+def set_scan_source_counts(conn: sqlite3.Connection, scan_id: int,
+                           source: str, requests: int, files: int) -> None:
+    """Record per-source request/file counts for a scan."""
+    conn.execute(
+        "INSERT INTO scans_sources (scan_id, source, requests, files) "
+        "VALUES (?, ?, ?, ?) "
+        "ON CONFLICT(scan_id, source) DO UPDATE SET "
+        "requests=excluded.requests, files=excluded.files",
+        (scan_id, source, requests, files))
+
+
+def latest_scans(conn: sqlite3.Connection, limit: int = 10) -> list[dict[str, Any]]:
+    """Most recent scan metadata rows."""
+    rows = conn.execute(
+        "SELECT id, kind, started_at, finished_at, status, full, dry_run, "
+        "log_files, request_count, error FROM scans "
+        "ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def get_scan(conn: sqlite3.Connection, scan_id: int) -> dict[str, Any] | None:
+    row = conn.execute(
+        "SELECT id, kind, started_at, finished_at, status, full, dry_run, "
+        "params_json, result_json, log_files, request_count, error "
+        "FROM scans WHERE id=?", (scan_id,)).fetchone()
+    return dict(row) if row is not None else None
+
+
+# ---------------------------------------------------------------------------
+# Logs + raw requests
+# ---------------------------------------------------------------------------
+
+def _parse_ts(ts: str) -> tuple[str, float | None]:
+    """Best-effort ISO timestamp extraction from a log line timestamp string."""
+    if not ts:
+        return (ts, None)
+    epoch = None
+    cleaned = ts.strip()
+    # Accept a trailing timezone like +HH:MM or Z; fall back to naive parse.
+    try:
+        epoch = datetime.fromisoformat(
+            cleaned.replace("Z", "+00:00")).timestamp()
+    except ValueError:
+        try:
+            epoch = datetime.fromisoformat(cleaned).timestamp()
+        except ValueError:
+            epoch = None
+    return (ts, epoch)
+
+
+def insert_requests(conn: sqlite3.Connection, rows: Sequence[dict[str, Any]]) -> int:
+    """Bulk-insert ``requests`` rows. Returns inserted count.
+
+    Each ``row`` maps keys: source, scan_id, log_file, ts, ip, version,
+    method, path, status, upstream_status, host, user, ua, referer, denied,
+    banned, is_error, malicious_hits, extra(dict|None) — matching
+    :mod:`app.utils.models.Request` plus provenance fields.
+    """
+    if not rows:
+        return 0
+    created = utcnow()
+    tuples: list[tuple] = []
+    for r in rows:
+        ts, epoch = _parse_ts(r.get("ts") or "")
+        extra = r.get("extra")
+        tuples.append((
+            r.get("source"), r.get("scan_id"), r.get("log_file"), ts, epoch,
+            r.get("ip"), r.get("version"),
+            r.get("method"), r.get("path"),
+            r.get("status"), r.get("upstream_status"),
+            r.get("host"), r.get("user"), r.get("ua"), r.get("referer"),
+            int(bool(r.get("denied"))), int(bool(r.get("banned"))),
+            int(bool(r.get("is_error"))),
+            int(r.get("malicious_hits") or 0),
+            json.dumps(extra) if extra is not None else None,
+            created,
+        ))
+    with transaction(conn):
+        conn.executemany(
+            "INSERT INTO requests (source, scan_id, log_file, ts, ts_epoch, ip, "
+            "version, method, path, status, upstream_status, host, user, ua, "
+            "referer, denied, banned, is_error, malicious_hits, extra, created_at) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            tuples)
+    return len(tuples)
+
+
+def query_requests(conn: sqlite3.Connection, *,
+                   ip: str | None = None, source: str | None = None,
+                   since: str | None = None, until: str | None = None,
+                   path: str | None = None, limit: int = 500) -> list[dict[str, Any]]:
+    """Return matching request rows (optionally filtered), newest last."""
+    clauses: list[str] = []
+    args: list[Any] = []
+    if ip:
+        clauses.append("ip = ?")
+        args.append(ip)
+    if source:
+        clauses.append("source = ?")
+        args.append(source)
+    if since:
+        clauses.append("COALESCE(ts, '') >= ?")
+        args.append(since)
+    if until:
+        clauses.append("COALESCE(ts, '') <= ?")
+        args.append(until)
+    if path:
+        clauses.append("path LIKE ?")
+        args.append(path)
+    sql = "SELECT * FROM requests"
+    if clauses:
+        sql += " WHERE " + " AND ".join(clauses)
+    sql += " ORDER BY id LIMIT ?"
+    args.append(int(limit))
+    return [dict(r) for r in conn.execute(sql, args).fetchall()]
+
+
+# ---------------------------------------------------------------------------
+# IP aggregates
+# ---------------------------------------------------------------------------
+
+def insert_ip_stats(conn: sqlite3.Connection, scan_id: int,
+                    stats: Sequence[IPStats],
+                    blocked: set[str]) -> int:
+    """Persist per-IP aggregates for a scan. ``blocked`` are direct blocked IPs."""
+    if not stats:
+        return 0
+    tuples: list[tuple] = []
+    for s in stats:
+        tuples.append((
+            scan_id, s.ip, s.version, s.total, s.errors, s.c4xx, s.c5xx,
+            s.denied, s.banned, s.first, s.last, s.malicious_hits,
+            json.dumps(dict(s.paths)), json.dumps(dict(s.methods)),
+            json.dumps(dict(s.hosts)), json.dumps(dict(s.uas)),
+            json.dumps(sorted(s.users)), s.whitelist or None,
+            int(s.ip in blocked),
+        ))
+    with transaction(conn):
+        conn.executemany(
+            "INSERT INTO ip_stats (scan_id, ip, version, total, errors, c4xx, "
+            "c5xx, denied, banned, first_ts, last_ts, malicious_hits, paths_json, "
+            "methods_json, hosts_json, uas_json, users_json, whitelist, blocked) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            tuples)
+    return len(tuples)
+
+
+def ip_timeline(conn: sqlite3.Connection, ip: str,
+                limit: int = 2000) -> list[dict[str, Any]]:
+    """Chronological request history for one IP across all sources."""
+    rows = conn.execute(
+        "SELECT source, log_file, ts, ts_epoch, method, path, status, host, "
+        "ua, denied, banned, is_error, malicious_hits FROM requests "
+        "WHERE ip = ? ORDER BY COALESCE(ts_epoch, 0), id LIMIT ?",
+        (ip, limit)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def count_requests(conn: sqlite3.Connection) -> int:
+    """Total persisted request records."""
+    return int(conn.execute("SELECT COUNT(*) FROM requests").fetchone()[0])
+
+
+def count_scans(conn: sqlite3.Connection) -> int:
+    """Total persisted scan records."""
+    return int(conn.execute("SELECT COUNT(*) FROM scans").fetchone()[0])
+
+
+# ---------------------------------------------------------------------------
+# Report artifacts
+# ---------------------------------------------------------------------------
+
+def insert_report_artifact(conn: sqlite3.Connection, *, source: str,
+                           report_type: str, generated_at: str,
+                           payload: Any, original_path: str,
+                           content_sha256: str, scan_id: int | None = None,
+                           schema_version: int = 1) -> int:
+    """Insert a structured report artifact, idempotent by path and hash."""
+    with transaction(conn):
+        conn.execute(
+            "INSERT OR IGNORE INTO report_artifacts "
+            "(source, scan_id, report_type, generated_at, schema_version, "
+            "payload_json, original_path, content_sha256, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (source, scan_id, report_type, generated_at, schema_version,
+             json.dumps(payload, ensure_ascii=False), original_path,
+             content_sha256, utcnow()),
+        )
+        row = conn.execute(
+            "SELECT id FROM report_artifacts "
+            "WHERE original_path=? AND content_sha256=?",
+            (original_path, content_sha256),
+        ).fetchone()
+    return int(row["id"])
+
+
+def list_report_artifacts(conn: sqlite3.Connection, *,
+                          source: str | None = None,
+                          report_type: str | None = None,
+                          since: str | None = None,
+                          until: str | None = None,
+                          limit: int = 50,
+                          cursor: str | None = None) -> dict[str, Any]:
+    """List report metadata using bounded keyset pagination."""
+    limit = min(max(int(limit), 1), 200)
+    clauses: list[str] = []
+    args: list[Any] = []
+    if source:
+        clauses.append("source=?")
+        args.append(source)
+    if report_type:
+        clauses.append("report_type=?")
+        args.append(report_type)
+    if since:
+        clauses.append("generated_at>=?")
+        args.append(since)
+    if until:
+        clauses.append("generated_at<=?")
+        args.append(until)
+    if cursor:
+        generated_at, artifact_id = _decode_report_cursor(cursor)
+        clauses.append("(generated_at<? OR (generated_at=? AND id<?))")
+        args.extend((generated_at, generated_at, artifact_id))
+
+    sql = (
+        "SELECT id, source, scan_id, report_type, generated_at, "
+        "schema_version, original_path, content_sha256 "
+        "FROM report_artifacts"
+    )
+    if clauses:
+        sql += " WHERE " + " AND ".join(clauses)
+    sql += " ORDER BY generated_at DESC, id DESC LIMIT ?"
+    rows = [dict(row) for row in conn.execute(sql, (*args, limit + 1))]
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    next_cursor = (
+        _encode_report_cursor(rows[-1]["generated_at"], rows[-1]["id"])
+        if has_more and rows else None
+    )
+    return {"items": rows, "next_cursor": next_cursor}
+
+
+def get_report_artifact(conn: sqlite3.Connection,
+                        artifact_id: int) -> dict[str, Any] | None:
+    """Fetch one artifact and decode its structured payload."""
+    row = conn.execute(
+        "SELECT id, source, scan_id, report_type, generated_at, "
+        "schema_version, payload_json, original_path, content_sha256, created_at "
+        "FROM report_artifacts WHERE id=?", (artifact_id,)).fetchone()
+    if row is None:
+        return None
+    artifact = dict(row)
+    artifact["payload"] = json.loads(artifact.pop("payload_json"))
+    return artifact
+
+
+def _encode_report_cursor(generated_at: str, artifact_id: int) -> str:
+    raw = f"{generated_at}\n{artifact_id}".encode("utf-8")
+    return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
+
+
+def _decode_report_cursor(cursor: str) -> tuple[str, int]:
+    try:
+        padded = cursor + "=" * (-len(cursor) % 4)
+        generated_at, artifact_id = base64.urlsafe_b64decode(
+            padded.encode("ascii")).decode("utf-8").split("\n", 1)
+        return generated_at, int(artifact_id)
+    except (ValueError, UnicodeError) as exc:
+        raise ValueError("invalid report cursor") from exc
+
+
+def correlate_across_sources(conn: sqlite3.Connection,
+                             limit: int = 100) -> list[dict[str, Any]]:
+    """IPs seen in more than one log source, with per-source counts."""
+    rows = conn.execute(
+        "SELECT ip, COUNT(DISTINCT source) AS sources, "
+        "GROUP_CONCAT(DISTINCT source) AS source_list, COUNT(*) AS total "
+        "FROM requests WHERE ip IS NOT NULL AND ip != '' "
+        "GROUP BY ip HAVING sources > 1 "
+        "ORDER BY total DESC LIMIT ?", (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def top_ips(conn: sqlite3.Connection, since: str | None = None,
+            limit: int = 50) -> list[dict[str, Any]]:
+    """Top IPs by request count within an optional time window."""
+    rows = conn.execute(
+        "SELECT ip, version, COUNT(*) AS total, "
+        "SUM(is_error) AS errors, SUM(malicious_hits) AS malicious_hits, "
+        "MIN(COALESCE(ts_epoch, 0)) AS first_epoch, "
+        "MAX(COALESCE(ts_epoch, 0)) AS last_epoch "
+        "FROM requests WHERE ip IS NOT NULL AND ip != '' "
+        + ("AND COALESCE(ts, '') >= ? " if since else "")
+        + "GROUP BY ip ORDER BY total DESC LIMIT ?",
+        ((since,) if since else ()) + (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+# ---------------------------------------------------------------------------
+# Blocklist ledger
+# ---------------------------------------------------------------------------
+
+def active_blocklist(conn: sqlite3.Connection) -> list[dict[str, Any]]:
+    rows = conn.execute(
+        "SELECT id, cidr, version, added_at, added_by, last_seen "
+        "FROM blocklist_entries WHERE status='active' ORDER BY cidr").fetchall()
+    return [dict(r) for r in rows]
+
+
+def block_entry(conn: sqlite3.Connection, cidr: str, *,
+                added_by: str | None = None, last_seen: str | None = None,
+                reasons: list[dict[str, Any]] | None = None) -> None:
+    """Add a CIDR to the grow-only ledger (idempotent when already active).
+
+    Manages its own transaction for standalone use."""
+    with transaction(conn):
+        _block_entry_inner(conn, cidr, added_by=added_by,
+                           last_seen=last_seen, reasons=reasons)
+
+
+def _block_entry_inner(conn: sqlite3.Connection, cidr: str, *,
+                       added_by: str | None = None,
+                       last_seen: str | None = None,
+                       reasons: list[dict[str, Any]] | None = None) -> None:
+    """Low-level block insert; caller owns the transaction."""
+    now = utcnow()
+    version = _cidr_version(cidr)
+    existing = conn.execute(
+        "SELECT id FROM blocklist_entries WHERE cidr = ? AND status='active'",
+        (cidr,)).fetchone()
+    if existing is not None:
+        entry_id = int(existing["id"])
+        conn.execute(
+            "UPDATE blocklist_entries SET last_seen = COALESCE(?, last_seen) "
+            "WHERE id = ?", (last_seen, entry_id))
+        if reasons:
+            insert_reasons(conn, entry_id, last_seen or "", reasons)
+        return
+    cur = conn.execute(
+        "INSERT INTO blocklist_entries (cidr, version, status, added_at, "
+        "added_by, last_seen) VALUES (?, ?, 'active', ?, ?, ?)",
+        (cidr, version, now, added_by, last_seen))
+    entry_id = int(cur.lastrowid)
+    if reasons:
+        insert_reasons(conn, entry_id, last_seen or "", reasons)
+
+
+def unblock_entry(conn: sqlite3.Connection, cidr: str, *,
+                  actor: str | None = None) -> bool:
+    """Remove a CIDR from active blocklist (idempotent). Returns True if removed."""
+    now = utcnow()
+    cur = conn.execute(
+        "UPDATE blocklist_entries SET status='removed', removed_at=? "
+        "WHERE cidr=? AND status='active'", (now, cidr))
+    removed = cur.rowcount > 0
+    conn.execute(
+        "INSERT INTO blocklist_events (action, cidr, actor, fired_at, detail) "
+        "VALUES ('remove', ?, ?, ?, ?)",
+        (cidr, actor, now, None))
+    return removed
+
+
+def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *,
+                     added_by: str | None = None, reasons=None) -> dict[str, Any]:
+    """Grow-only record: add any ``cidrs`` not already active to the ledger.
+
+    Unlike :func:`sync_blocklist`, this **never removes** entries. It mirrors
+    the router's cumulative model: once an IP/subnet has ever been blocked it
+    stays active even after it leaves the scan window (e.g. NPM ~30-day log
+    retention), exactly like the old git-history backfill. Returns the added
+    delta and the resulting active count.
+    """
+    cidr_set = set(cidrs)
+    before = {r["cidr"] for r in active_blocklist(conn)}
+    to_add = sorted(cidr_set - before)
+    now = utcnow()
+    with transaction(conn):
+        for cidr in to_add:
+            _block_entry_inner(conn, cidr, added_by=added_by, last_seen=now,
+                               reasons=reasons)
+        # Refresh last_seen on entries that are still desired so the ledger
+        # reflects recency for existing rows too.
+        conn.executemany(
+            "UPDATE blocklist_entries SET last_seen=? WHERE cidr=? AND status='active'",
+            [(now, cidr) for cidr in cidr_set.intersection(before)])
+    return {"added": to_add, "active": len(before) + len(to_add)}
+
+
+def prune_old_single_ip_blocks(conn: sqlite3.Connection, *,
+                               older_than_days: int = 30,
+                               actor: str | None = "retention") -> list[str]:
+    """Remove active single-host entries older than ``older_than_days``.
+
+    Keeps CIDR ranges/subnets intact. Host entries are detected via
+    ``ipaddress.ip_network(..., strict=False)`` and only removed when the
+    network is exactly one address (/32 for IPv4, /128 for IPv6).
+    """
+    cutoff = datetime.now(timezone.utc) - \
+        timedelta(days=max(0, older_than_days))
+    rows = conn.execute(
+        "SELECT cidr, added_at, last_seen FROM blocklist_entries "
+        "WHERE status='active'").fetchall()
+
+    to_remove: list[str] = []
+    for row in rows:
+        cidr = str(row["cidr"])
+        if not _is_single_host_cidr(cidr):
+            continue
+        seen = _parse_iso_ts(
+            row["last_seen"]) or _parse_iso_ts(row["added_at"])
+        if seen is None:
+            continue
+        if seen < cutoff:
+            to_remove.append(cidr)
+
+    if not to_remove:
+        return []
+
+    now = utcnow()
+    with transaction(conn):
+        conn.executemany(
+            "UPDATE blocklist_entries SET status='removed', removed_at=? "
+            "WHERE cidr=? AND status='active'",
+            [(now, cidr) for cidr in to_remove],
+        )
+        conn.executemany(
+            "INSERT INTO blocklist_events (action, cidr, actor, fired_at, detail) "
+            "VALUES ('remove', ?, ?, ?, ?)",
+            [(cidr, actor, now, f"single-ip-retention>{older_than_days}d")
+             for cidr in to_remove],
+        )
+    return sorted(to_remove)
+
+
+def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *,
+                   added_by: str | None = None) -> dict[str, Any]:
+    """Reconcile active blocklist to ``desired_cidrs``.
+
+    Adds missing entries; marks active entries absent from ``desired_cidrs``
+    as removed (only ones the system added) so the ledger stays grow-only but
+    reflects the current policy. Returns the add/remove delta.
+    """
+    desired = set(desired_cidrs)
+    current = {r["cidr"] for r in active_blocklist(conn)}
+    added = sorted(desired - current)
+    removed = sorted(current - desired)
+    now = utcnow()
+    with transaction(conn):
+        for cidr in added:
+            _block_entry_inner(conn, cidr, added_by=added_by, last_seen=now)
+        conn.executemany(
+            "UPDATE blocklist_entries SET status='removed', removed_at=? "
+            "WHERE cidr=? AND status='active'",
+            [(now, cidr) for cidr in removed])
+    return {"added": added, "removed": removed}
+
+
+def insert_reasons(conn: sqlite3.Connection, entry_id: int, scan_session: str,
+                   reasons: list[dict[str, Any]]) -> None:
+    tuples = [
+        (entry_id, r.get("scan_id"), r.get("ip") or "", r.get("reason"),
+         r.get("pattern"), r.get("severity"), r.get("sample_path"),
+         int(r.get("hits") or 0))
+        for r in reasons
+    ]
+    conn.executemany(
+        "INSERT INTO blocklist_reasons (entry_id, scan_id, ip, reason, pattern, "
+        "severity, sample_path, hits) VALUES (?,?,?,?,?,?,?,?)",
+        tuples)
+
+
+def blocklist_history(conn: sqlite3.Connection, limit: int = 500) -> list[dict[str, Any]]:
+    rows = conn.execute(
+        "SELECT id, cidr, version, status, added_at, removed_at, added_by, "
+        "last_seen FROM blocklist_entries ORDER BY id DESC LIMIT ?",
+        (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def _cidr_version(cidr: str) -> int:
+    try:
+        import ipaddress
+        net = ipaddress.ip_network(cidr, strict=False)
+        return net.version
+    except ValueError:
+        return 4 if ":" not in str(cidr) else 6
+
+
+def _is_single_host_cidr(cidr: str) -> bool:
+    try:
+        net = ipaddress.ip_network(cidr, strict=False)
+        return net.num_addresses == 1
+    except ValueError:
+        return False
+
+
+def _parse_iso_ts(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+# ---------------------------------------------------------------------------
+# Whitelists
+# ---------------------------------------------------------------------------
+
+def set_whitelist(conn: sqlite3.Connection, provider: str,
+                  cidrs: Sequence[str], source: str | None = None) -> None:
+    """Replace the CIDR set for a provider (mark old inactive, upsert new)."""
+    now = utcnow()
+    with transaction(conn):
+        conn.execute(
+            "UPDATE whitelist_entries SET active=0 WHERE provider=?", (provider,))
+        for cidr in cidrs:
+            version = _cidr_version(cidr)
+            conn.execute(
+                "INSERT INTO whitelist_entries (provider, cidr, version, source, "
+                "active, updated_at) VALUES (?,?,?,?,1,?) "
+                "ON CONFLICT(provider, cidr) DO UPDATE SET active=1, "
+                "source=excluded.source, updated_at=excluded.updated_at",
+                (provider, cidr, version, source, now))
+        conn.execute(
+            "INSERT INTO whitelist_sources (provider, source_type, last_refresh_at, "
+            "entry_count) VALUES (?, ?, ?, ?) ON CONFLICT(provider) DO UPDATE SET "
+            "source_type=excluded.source_type, last_refresh_at=excluded.last_refresh_at, "
+            "entry_count=excluded.entry_count",
+            (provider, source, now, len(cidrs)))
+
+
+def is_whitelisted_db(conn: sqlite3.Connection, ip: str) -> bool:
+    """Check whether an IP falls in any active whitelist CIDR."""
+    import ipaddress
+    try:
+        addr = ipaddress.ip_address(ip)
+    except ValueError:
+        return False
+    rows = conn.execute(
+        "SELECT cidr FROM whitelist_entries WHERE active=1").fetchall()
+    for r in rows:
+        try:
+            net = ipaddress.ip_network(r["cidr"], strict=False)
+        except ValueError:
+            continue
+        if addr.version == net.version and addr in net:
+            return True
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Firewall snapshots
+# ---------------------------------------------------------------------------
+
+def save_firewall_snapshot(conn: sqlite3.Connection, *,
+                           address_lists_v4: list[dict],
+                           address_lists_v6: list[dict],
+                           rules_v4: list[dict], rules_v6: list[dict]) -> int:
+    """Persist a full firewall snapshot; returns the snapshot id."""
+    now = utcnow()
+    cur = conn.execute(
+        "INSERT INTO firewall_snapshots (captured_at, kind, trigger) "
+        "VALUES (?, 'snapshot', 'scan')", (now,))
+    snapshot_id = int(cur.lastrowid)
+    with transaction(conn):
+        tuples = [
+            (snapshot_id, "ipv4", e.get("list"), e.get("address"),
+             int(bool(e.get("dynamic"))), e.get("timeout"))
+            for e in address_lists_v4
+        ] + [
+            (snapshot_id, "ipv6", e.get("list"), e.get("address"),
+             int(bool(e.get("dynamic"))), e.get("timeout"))
+            for e in address_lists_v6
+        ]
+        conn.executemany(
+            "INSERT INTO firewall_address_list_entries (snapshot_id, network, "
+            "list, address, dynamic, timeout) VALUES (?,?,?,?,?,?)", tuples)
+        rule_tuples = [
+            (snapshot_id, "ipv4", idx, e.get("chain"), e.get("action"),
+             e.get("dst-list"), None, e.get("comment"), json.dumps(e))
+            for idx, e in enumerate(rules_v4)
+        ] + [
+            (snapshot_id, "ipv6", idx, e.get("chain"), e.get("action"),
+             e.get("dst-list"), None, e.get("comment"), json.dumps(e))
+            for idx, e in enumerate(rules_v6)
+        ]
+        conn.executemany(
+            "INSERT INTO firewall_rule_entries (snapshot_id, network, rule_index, "
+            "chain, action, dst_list, proto_port, comment, raw_json) "
+            "VALUES (?,?,?,?,?,?,?,?,?)", rule_tuples)
+    return snapshot_id
+
+
+def recent_firewall_snapshots(conn: sqlite3.Connection,
+                              limit: int = 10) -> list[dict[str, Any]]:
+    """Most recent firewall snapshots with entry counts (F001)."""
+    rows = conn.execute(
+        "SELECT fs.id, fs.captured_at, fs.kind, fs.trigger, "
+        "(SELECT COUNT(*) FROM firewall_address_list_entries e "
+        " WHERE e.snapshot_id = fs.id) AS addr_entries, "
+        "(SELECT COUNT(*) FROM firewall_rule_entries r "
+        " WHERE r.snapshot_id = fs.id) AS rule_entries "
+        "FROM firewall_snapshots fs ORDER BY fs.id DESC LIMIT ?",
+        (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+# ---------------------------------------------------------------------------
+# WHOIS cache
+# ---------------------------------------------------------------------------
+
+def set_whois(conn: sqlite3.Connection, ip: str, name: str | None,
+              cidr: str | None) -> None:
+    conn.execute(
+        "INSERT INTO whois_info (ip, name, cidr, fetched_at) VALUES (?,?,?,?) "
+        "ON CONFLICT(ip) DO UPDATE SET name=excluded.name, cidr=excluded.cidr, "
+        "fetched_at=excluded.fetched_at",
+        (ip, name, cidr, utcnow()))

app/db/schema.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""SQLite schema and migrations for the intrusion-detection system.
+
+Definitions live here as the single source of truth and are applied through
+``PRAGMA user_version`` migrations (see ``apply_migrations``). Each entry is an
+ordered list of DDL statements; the version is bumped in lock-step so the
+migrations are idempotent.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+from typing import Iterable
+
+#: Latest schema version. Bump by one whenever a new migration is appended.
+SCHEMA_VERSION = 2
+
+#: Ordered migrations: index -> list of DDL statements. A migration N upgrades a
+#: database from user_version N-1 to N. Never edit an older migration in place;
+#: append a new one instead.
+MIGRATIONS: dict[int, list[str]] = {
+    1: [
+        # ------------------------------------------------------------------
+        # log_sources — canonical provenance
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS log_sources (
+            id INTEGER PRIMARY KEY,
+            name TEXT UNIQUE NOT NULL,
+            log_path TEXT,
+            enabled INTEGER NOT NULL DEFAULT 1
+        )
+        """
+    ],
+
+    2: [
+        """
+        CREATE TABLE IF NOT EXISTS report_artifacts (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source TEXT NOT NULL,
+            scan_id INTEGER,
+            report_type TEXT NOT NULL,
+            generated_at TEXT NOT NULL,
+            schema_version INTEGER NOT NULL DEFAULT 1,
+            payload_json TEXT NOT NULL,
+            original_path TEXT NOT NULL,
+            content_sha256 TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            UNIQUE (original_path, content_sha256)
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_report_artifacts_source_time "
+        "ON report_artifacts (source, generated_at DESC, id DESC)",
+        "CREATE INDEX IF NOT EXISTS idx_report_artifacts_type_time "
+        "ON report_artifacts (report_type, generated_at DESC, id DESC)",
+        """
+        CREATE TABLE IF NOT EXISTS requests (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source TEXT NOT NULL,
+            scan_id INTEGER,
+            log_file TEXT,
+            ts TEXT,
+            ts_epoch REAL,
+            ip TEXT NOT NULL,
+            version INTEGER,
+            method TEXT,
+            path TEXT,
+            status INTEGER,
+            upstream_status INTEGER,
+            host TEXT,
+            user TEXT,
+            ua TEXT,
+            referer TEXT,
+            denied INTEGER NOT NULL DEFAULT 0,
+            banned INTEGER NOT NULL DEFAULT 0,
+            is_error INTEGER NOT NULL DEFAULT 0,
+            malicious_hits INTEGER NOT NULL DEFAULT 0,
+            extra TEXT,
+            created_at TEXT NOT NULL
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_requests_ip_ts ON requests (ip, ts)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_source_ts ON requests (source, ts)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_ts_epoch ON requests (ts_epoch)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_path ON requests (path)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_scan ON requests (scan_id)",
+        # ------------------------------------------------------------------
+        # ip_stats — per-IP aggregate per scan
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS ip_stats (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            scan_id INTEGER,
+            ip TEXT NOT NULL,
+            version INTEGER,
+            total INTEGER NOT NULL DEFAULT 0,
+            errors INTEGER NOT NULL DEFAULT 0,
+            c4xx INTEGER NOT NULL DEFAULT 0,
+            c5xx INTEGER NOT NULL DEFAULT 0,
+            denied INTEGER NOT NULL DEFAULT 0,
+            banned INTEGER NOT NULL DEFAULT 0,
+            first_ts TEXT,
+            last_ts TEXT,
+            malicious_hits INTEGER NOT NULL DEFAULT 0,
+            paths_json TEXT,
+            methods_json TEXT,
+            hosts_json TEXT,
+            uas_json TEXT,
+            users_json TEXT,
+            whitelist TEXT,
+            blocked INTEGER NOT NULL DEFAULT 0
+        )
+        """,
+        "CREATE UNIQUE INDEX IF NOT EXISTS idx_ip_stats_scan_ip ON ip_stats (scan_id, ip)",
+        # ------------------------------------------------------------------
+        # scans + scans_sources — one row per analysis run
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS scans (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            kind TEXT NOT NULL,
+            started_at TEXT NOT NULL,
+            finished_at TEXT,
+            status TEXT NOT NULL DEFAULT 'running',
+            full INTEGER NOT NULL DEFAULT 0,
+            dry_run INTEGER NOT NULL DEFAULT 0,
+            params_json TEXT,
+            result_json TEXT,
+            log_files INTEGER NOT NULL DEFAULT 0,
+            request_count INTEGER NOT NULL DEFAULT 0,
+            error TEXT
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_scans_started ON scans (started_at)",
+        """
+        CREATE TABLE IF NOT EXISTS scans_sources (
+            scan_id INTEGER NOT NULL,
+            source TEXT NOT NULL,
+            requests INTEGER NOT NULL DEFAULT 0,
+            files INTEGER NOT NULL DEFAULT 0,
+            PRIMARY KEY (scan_id, source)
+        )
+        """,
+        # ------------------------------------------------------------------
+        # blocklist — grow-only ledger + reasons + audit events
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS blocklist_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            cidr TEXT NOT NULL,
+            version INTEGER,
+            status TEXT NOT NULL DEFAULT 'active',
+            added_at TEXT NOT NULL,
+            removed_at TEXT,
+            added_by TEXT,
+            last_seen TEXT
+        )
+        """,
+        """CREATE UNIQUE INDEX IF NOT EXISTS
+           idx_blocklist_active_cidr ON blocklist_entries (cidr) WHERE status='active'""",
+        """
+        CREATE TABLE IF NOT EXISTS blocklist_reasons (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            entry_id INTEGER,
+            scan_id INTEGER,
+            ip TEXT NOT NULL,
+            reason TEXT,
+            pattern TEXT,
+            severity INTEGER,
+            sample_path TEXT,
+            hits INTEGER NOT NULL DEFAULT 0
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_blereasons_ip ON blocklist_reasons (ip)",
+        """
+        CREATE TABLE IF NOT EXISTS blocklist_events (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            action TEXT NOT NULL,
+            cidr TEXT NOT NULL,
+            actor TEXT,
+            fired_at TEXT NOT NULL,
+            detail TEXT
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_blevents_cidr ON blocklist_events (cidr)",
+        # ------------------------------------------------------------------
+        # whitelist — current entries + source refresh tracking
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS whitelist_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            provider TEXT NOT NULL,
+            cidr TEXT NOT NULL,
+            version INTEGER,
+            source TEXT,
+            active INTEGER NOT NULL DEFAULT 1,
+            updated_at TEXT
+        )
+        """,
+        "CREATE UNIQUE INDEX IF NOT EXISTS idx_wl_provider_cidr ON whitelist_entries (provider, cidr)",
+        """
+        CREATE TABLE IF NOT EXISTS whitelist_sources (
+            provider TEXT PRIMARY KEY,
+            source_type TEXT,
+            url TEXT,
+            file TEXT,
+            last_refresh_at TEXT,
+            entry_count INTEGER NOT NULL DEFAULT 0,
+            refresh_error TEXT
+        )
+        """,
+        # ------------------------------------------------------------------
+        # firewall — snapshots of address lists + rules
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS firewall_snapshots (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            captured_at TEXT NOT NULL,
+            kind TEXT NOT NULL DEFAULT 'snapshot',
+            trigger TEXT NOT NULL DEFAULT 'scan'
+        )
+        """,
+        """
+        CREATE TABLE IF NOT EXISTS firewall_address_list_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            snapshot_id INTEGER,
+            network TEXT,
+            list TEXT,
+            address TEXT,
+            dynamic INTEGER NOT NULL DEFAULT 0,
+            timeout INTEGER
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_fwal_snapshot ON firewall_address_list_entries (snapshot_id)",
+        "CREATE INDEX IF NOT EXISTS idx_fwal_list_addr ON firewall_address_list_entries (list, address)",
+        """
+        CREATE TABLE IF NOT EXISTS firewall_rule_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            snapshot_id INTEGER,
+            network TEXT,
+            rule_index INTEGER,
+            chain TEXT,
+            action TEXT,
+            dst_list TEXT,
+            proto_port TEXT,
+            comment TEXT,
+            raw_json TEXT
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_fwr_snapshot ON firewall_rule_entries (snapshot_id)",
+        # ------------------------------------------------------------------
+        # whois — structured cache
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS whois_info (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            ip TEXT UNIQUE NOT NULL,
+            name TEXT,
+            cidr TEXT,
+            fetched_at TEXT NOT NULL
+        )
+        """,
+    ],
+}
+
+
+def apply_migrations(conn: sqlite3.Connection) -> None:
+    """Apply all pending migrations, bumping ``user_version`` in lock-step.
+
+    Safe to call on every process start: it only runs migrations whose version
+    is greater than the current ``PRAGMA user_version``.
+    """
+    conn.execute("PRAGMA foreign_keys=ON")
+    current = conn.execute("PRAGMA user_version").fetchone()[0]
+    for version in range(current + 1, SCHEMA_VERSION + 1):
+        statements = MIGRATIONS.get(version)
+        if statements is None:
+            raise RuntimeError(
+                f"missing migration for schema version {version}")
+        with conn:
+            for ddl in statements:
+                conn.execute(ddl)
+            conn.execute(f"PRAGMA user_version={version}")

app/firewall.py

@@ -143,9 +143,11 @@ class MikrotikFirewall:
         """Save current address lists and firewall rules for auditing.
 
         Writes structured snapshots to ``<report_dir>/mikrotik/data/`` and to
-        the common ``data/mikrotik/`` directory (overwritten every run so git
+        the common ``data/mikrotik/`` directory (scratch mirror for debugging;
-        can diff changes to the firewall state over time), plus a timestamped
+        **not** a git audit trail), plus a timestamped JSON report under
-        JSON report under ``<report_dir>/mikrotik/``.
+        ``<report_dir>/mikrotik/``. The authoritative long-term audit trail is
+        the SQLite ``firewall_snapshots`` tables (see ``app.db``), which are
+        appended here regardless of the file writes.
         """
         state = {
             "router": self.router_ip,
@@ -174,11 +176,31 @@ class MikrotikFirewall:
                     os.path.join(data_dir, f"firewall_{version}.json"),
                     state["firewall_rules"][version])
 
+        # Persist a SQLite snapshot (authoritative audit trail, independent of
+        # git). Best-effort: never breaks a router sync if the DB is down.
+        self._persist_snapshot_to_db(state)
+
         timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
         report_path = os.path.join(report_dir, "mikrotik", f"{timestamp}.json")
         self._write_json(report_path, state)
         return report_path
 
+    @staticmethod
+    def _persist_snapshot_to_db(state: dict) -> None:
+        """Write the firewall snapshot into the SQLite store (best-effort)."""
+        try:
+            from .db import get_connection
+            from .db import repositories as repo
+            repo.save_firewall_snapshot(
+                get_connection(),
+                address_lists_v4=state["address_lists"]["ipv4"],
+                address_lists_v6=state["address_lists"]["ipv6"],
+                rules_v4=state["firewall_rules"]["ipv4"],
+                rules_v6=state["firewall_rules"]["ipv6"],
+            )
+        except Exception as exc:
+            print(f"WARN: could not persist firewall snapshot to db ({exc})")
+
     def sync_blocklist(self, blocked_ipv4: Iterable[str],
                        blocked_ipv6: Iterable[str],
                        dry_run: bool = False,
@@ -186,10 +208,12 @@ class MikrotikFirewall:
         """Synchronize the Blocked list cumulatively and audit the result.
 
         The blocklist is grow-only: targets are added when present in the
-        complete set (all git history of the blocklist files + the current
+        complete cumulative set (git history of the blocklist files + the
-        working files + this scan's targets) and never removed automatically.
+        current working files + this scan's targets) and never removed
-        Only router entries absent from the complete set — i.e. never blocked
+        automatically. Only router entries absent from the complete set — i.e.
-        by the system — are pruned. See ``app.blocklist_history``.
+        never blocked by the system — are pruned. The history now lives in the
+        SQLite blocklist ledger (``app.db`` ``blocklist_entries``); git history
+        remains a read-only fallback. See ``app.blocklist_history``.
         """
 
         desired_ipv4 = set(blocked_ipv4)

app/scan/__init__.py

@@ -1,322 +1,431 @@
-#!/usr/bin/env python3
+#!/usr/bin/env python3
-"""Consolidated intrusion detection log scanner.
+"""Consolidated intrusion detection log scanner.
-
+
-Runs both copyparty and Nginx Proxy Manager (NPM) log analyzers,
+Runs both copyparty and Nginx Proxy Manager (NPM) log analyzers,
-and outputs a single consolidated MikroTik RouterOS blocklist.
+and outputs a single consolidated MikroTik RouterOS blocklist.
-"""
+"""
-
+
-from __future__ import annotations
+from __future__ import annotations
-
+
-import argparse
+import argparse
-import os
+import os
-from dataclasses import dataclass, field
+from dataclasses import dataclass, field
-
+
-from ..utils.blocklist_history import (
+from ..utils.blocklist_history import (
-    committed_lines,
+    committed_lines,
-    cumulative_blocklist,
+    cumulative_blocklist,
-    current_file_lines,
+    current_file_lines,
-    write_blocklist_report,
+    write_blocklist_report,
-)
+)
-from ..utils.constants import (
+from ..utils.constants import (
-    COPYPARTY_LOG_PATH,
+    COPYPARTY_LOG_PATH,
-    COPYPARTY_REPORT_DIR,
+    COPYPARTY_REPORT_DIR,
-    DEFAULT_ERROR_RATIO,
+    DEFAULT_ERROR_RATIO,
-    DEFAULT_MIN_DENIED,
+    DEFAULT_MIN_DENIED,
-    DEFAULT_MIN_ERRORS,
+    DEFAULT_MIN_ERRORS,
-    DEFAULT_TOP,
+    DEFAULT_TOP,
-    NPM_LOG_PATH,
+    NPM_LOG_PATH,
-    NPM_REPORT_DIR,
+    NPM_REPORT_DIR,
-    REPORT_DIR,
+    REPORT_DIR,
-    REPORT_RETENTION_DAYS,
+    REPORT_RETENTION_DAYS,
-    REPORT_RETENTION_MAX_FILES,
+    REPORT_RETENTION_MAX_FILES,
-    SUBNET_THRESHOLD,
+    SUBNET_THRESHOLD,
-)
+)
-from ..firewall import MikrotikFirewall, write_blocklist_files
+from ..firewall import MikrotikFirewall, write_blocklist_files
-from ..utils.retention import prune_report_dirs
+from ..utils.retention import prune_report_dirs
-from ..copyparty import CopypartyAnalyzer
+from ..copyparty import CopypartyAnalyzer
-from ..npm import NpmAnalyzer
+from ..npm import NpmAnalyzer
-from ..utils.ip_utils import is_whitelisted
+from ..utils.ip_utils import is_whitelisted
-from ..mikrotik import split_cidr_versions
+from ..mikrotik import split_cidr_versions
-from ..utils.whois import consolidate_by_subnet
+from ..utils.whois import consolidate_by_subnet
-
+
-
+
-@dataclass
+def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
-class ScanParams:
+                          desired_v6: list[str],
-    """Configuration for a consolidated scan.
+                          added_by: str | None = None) -> dict | None:
-
+    """Best-effort grow-only sync of block targets into the SQLite ledger.
-    Shared by the CLI entry point and the web interface so both run the
+
-    exact same pipeline.
+    Uses ``record_blocklist`` (add-only) so the ledger is the authoritative
-    """
+    cumulative record: once an IP is ever blocked it stays active even after it
-
+    leaves the scan window, matching the router's cumulative model. Returns the
-    copyparty_log_path: str = COPYPARTY_LOG_PATH
+    added delta or None if the DB is unavailable.
-    copyparty_report_dir: str = COPYPARTY_REPORT_DIR
+    """
-    min_denied: int = DEFAULT_MIN_DENIED
+    try:
-    npm_log_path: str = NPM_LOG_PATH
+        from ..db import get_connection
-    npm_report_dir: str = NPM_REPORT_DIR
+        from ..db import repositories as repo
-    min_errors: int = DEFAULT_MIN_ERRORS
+        conn = get_connection()
-    error_ratio: float = DEFAULT_ERROR_RATIO
+        result = repo.record_blocklist(
-    report_dir: str = REPORT_DIR
+            conn, list(desired_v4) + list(desired_v6), added_by=added_by)
-    top: int = DEFAULT_TOP
+        # Retention policy: expire stale single-host entries after 30 days
-    quiet: bool = False
+        # while keeping CIDR ranges/subnets intact.
-    full: bool = False
+        pruned = repo.prune_old_single_ip_blocks(conn, older_than_days=30)
-    update_firewall: bool = False
+        result["pruned_single_ips"] = pruned
-    dry_run: bool = False
+        return result
-
+    except Exception as exc:  # never break the scan for a DB issue
-
+        print(f"WARN: could not sync blocklist to db ({exc})")
-@dataclass
+        return None
-class ScanResult:
+
-    """Structured outcome of a consolidated scan."""
+
-
+def _db_complete_blocklist(desired_v4: list[str], desired_v6: list[str]):
-    ok: bool
+    """Complete cumulative blocklist from the DB ledger (B001).
-    cp_exists: bool = False
+
-    npm_exists: bool = False
+    Returns ``(complete_v4, complete_v6)`` as **sets** (same contract as
-    cp_candidates: int = 0
+    ``cumulative_blocklist``) derived from the active blocklist ledger plus
-    npm_candidates: int = 0
+    this scan's newly desired targets (in case the DB write for this scan has
-    blocked_ips: int = 0
+    not run yet). Falls back to the git-history reconstruction when the DB is
-    blocked_targets: int = 0
+    unavailable, so nothing regresses.
-    blocked_ipv4: list[str] = field(default_factory=list)
+    """
-    blocked_ipv6: list[str] = field(default_factory=list)
+    try:
-    blocklist_v4_path: str | None = None
+        from ..db import get_connection
-    blocklist_v6_path: str | None = None
+        from ..db import repositories as repo
-    firewall_connected: bool = False
+        conn = get_connection()
-    firewall_changes: dict | None = None
+        active = [e["cidr"] for e in repo.active_blocklist(conn)]
-    blocklist_report_path: str | None = None
+    except Exception:
-    errors: list[str] = field(default_factory=list)
+        active = []
-
+    union = set(active) | set(desired_v4) | set(desired_v6)
-
+    v4, v6 = split_cidr_versions(union)
-def run_scan(params: ScanParams) -> ScanResult:
+    return set(v4), set(v6)
-    """Run the full consolidated scan pipeline and return structured results.
+
-
+
-    Steps: connect to MikroTik (optional) -> snapshot state -> run copyparty
+@dataclass
-    + NPM analyzers -> consolidate candidates -> write blocklists -> compare
+class ScanParams:
-    and apply firewall changes (optional).
+    """Configuration for a consolidated scan.
-    """
+
-    errors: list[str] = []
+    Shared by the CLI entry point and the web interface so both run the
-
+    exact same pipeline.
-    # --- Step 1-3: connect to MikroTik RouterOS and capture current state ---
+    """
-    # If the router is unreachable we fall back to scan-only mode: still
+
-    # generate reports and blocklists, but skip the firewall sync.
+    copyparty_log_path: str = COPYPARTY_LOG_PATH
-    firewall: MikrotikFirewall | None = None
+    copyparty_report_dir: str = COPYPARTY_REPORT_DIR
-    if params.update_firewall:
+    min_denied: int = DEFAULT_MIN_DENIED
-        firewall = MikrotikFirewall()
+    npm_log_path: str = NPM_LOG_PATH
-        try:
+    npm_report_dir: str = NPM_REPORT_DIR
-            firewall.connect()
+    min_errors: int = DEFAULT_MIN_ERRORS
-            # Read current firewall rules + address lists and snapshot them to
+    error_ratio: float = DEFAULT_ERROR_RATIO
-            # reports/mikrotik/data/ and the git-diffable data/mikrotik/.
+    report_dir: str = REPORT_DIR
-            firewall.save_state_report(params.report_dir)
+    top: int = DEFAULT_TOP
-        except Exception as e:
+    quiet: bool = False
-            message = f"could not read MikroTik router ({firewall.router_ip}): {e}"
+    full: bool = False
-            errors.append(message)
+    update_firewall: bool = False
-            print(f"\nWARNING: {message}")
+    dry_run: bool = False
-            firewall.disconnect()
+
-            firewall = None
+
-
+@dataclass
-    # --- Step 4: run copyparty + NPM analyzers -> blocklist candidates ------
+class ScanResult:
-    cp_analyzer = CopypartyAnalyzer(
+    """Structured outcome of a consolidated scan."""
-        log_path=params.copyparty_log_path,
+
-        report_dir=params.copyparty_report_dir,
+    ok: bool
-        top=params.top,
+    cp_exists: bool = False
-        quiet=params.quiet,
+    npm_exists: bool = False
-        full=params.full,
+    cp_candidates: int = 0
-        min_denied=params.min_denied,
+    npm_candidates: int = 0
-    )
+    blocked_ips: int = 0
-
+    blocked_targets: int = 0
-    npm_analyzer = NpmAnalyzer(
+    blocked_ipv4: list[str] = field(default_factory=list)
-        log_path=params.npm_log_path,
+    blocked_ipv6: list[str] = field(default_factory=list)
-        report_dir=params.npm_report_dir,
+    blocklist_v4_path: str | None = None
-        top=params.top,
+    blocklist_v6_path: str | None = None
-        quiet=params.quiet,
+    firewall_connected: bool = False
-        full=params.full,
+    firewall_changes: dict | None = None
-        min_errors=params.min_errors,
+    blocklist_report_path: str | None = None
-        error_ratio=params.error_ratio,
+    errors: list[str] = field(default_factory=list)
-    )
+
-
+
-    print("=== Running copyparty scan ===")
+def run_scan(params: ScanParams) -> ScanResult:
-    cp_exists = True
+    """Run the full consolidated scan pipeline and return structured results.
-    try:
+
-        cp_analyzer.run()
+    Steps: connect to MikroTik (optional) -> snapshot state -> run copyparty
-    except Exception as e:
+    + NPM analyzers -> consolidate candidates -> write blocklists -> compare
-        errors.append(f"copyparty scan failed: {e}")
+    and apply firewall changes (optional).
-        print(f"Error running copyparty scan: {e}")
+    """
-        cp_exists = False
+    errors: list[str] = []
-
+
-    print("\n=== Running NPM scan ===")
+    # --- Step 0: open the SQLite store for this consolidated run -------------
-    npm_exists = True
+    db_scan_id: int | None = None
-    try:
+    try:
-        npm_analyzer.run()
+        from ..db import get_connection
-    except Exception as e:
+        from ..db import repositories as repo
-        errors.append(f"NPM scan failed: {e}")
+        from ..utils.whitelists import sync_whitelists_to_db
-        print(f"Error running NPM scan: {e}")
+        conn = get_connection()
-        npm_exists = False
+        sync_whitelists_to_db()
-
+        params_map = {
-    if not cp_exists and not npm_exists:
+            "full": params.full, "dry_run": params.dry_run,
-        message = "Both scans failed. Unable to generate consolidated blocklist."
+            "update_firewall": params.update_firewall,
-        errors.append(message)
+            "quiet": params.quiet, "top": params.top,
-        print(f"\n{message}")
+        }
-        if firewall is not None:
+        db_scan_id = repo.start_scan(conn, "ids", params=params_map,
-            firewall.disconnect()
+                                     full=params.full, dry_run=params.dry_run)
-        return ScanResult(ok=False, errors=errors)
+    except Exception as exc:
-
+        print(f"WARN: sqlite store unavailable ({exc})")
-    # Consolidate block candidates
+
-    consolidated_candidates: set[str] = set()
+    # --- Step 1-3: connect to MikroTik RouterOS and capture current state ---
-    if cp_exists:
+    # If the router is unreachable we fall back to scan-only mode: still
-        consolidated_candidates.update(cp_analyzer.block_candidates())
+    # generate reports and blocklists, but skip the firewall sync.
-    if npm_exists:
+    firewall: MikrotikFirewall | None = None
-        consolidated_candidates.update(npm_analyzer.block_candidates())
+    if params.update_firewall:
-
+        firewall = MikrotikFirewall()
-    # Exclude whitelisted IPs
+        try:
-    final_blocked_ips = {
+            firewall.connect()
-        ip for ip in consolidated_candidates if not is_whitelisted(ip)}
+            # Read current firewall rules + address lists and snapshot them to
-
+            # reports/mikrotik/data/ and the git-diffable data/mikrotik/.
-    # Group into subnets via WHOIS if multiple malicious IPs in same subnet
+            firewall.save_state_report(params.report_dir)
-    print("\nGrouping blocked IPs into subnets using WHOIS...")
+        except Exception as e:
-    final_blocked_targets = consolidate_by_subnet(
+            message = f"could not read MikroTik router ({firewall.router_ip}): {e}"
-        final_blocked_ips, threshold=SUBNET_THRESHOLD)
+            errors.append(message)
-    blocked_ipv4, blocked_ipv6 = split_cidr_versions(final_blocked_targets)
+            print(f"\nWARNING: {message}")
-
+            firewall.disconnect()
-    cp_candidates = len(cp_analyzer.block_candidates()) if cp_exists else 0
+            firewall = None
-    npm_candidates = len(npm_analyzer.block_candidates()) if npm_exists else 0
+
-
+    # --- Step 4: run copyparty + NPM analyzers -> blocklist candidates ------
-    print("\n=== Consolidated Summary ===")
+    cp_analyzer = CopypartyAnalyzer(
-    print(f"copyparty block candidates: {cp_candidates}")
+        log_path=params.copyparty_log_path,
-    print(f"NPM block candidates:       {npm_candidates}")
+        report_dir=params.copyparty_report_dir,
-    print(f"Total unified blocked IPs:  {len(final_blocked_ips)}")
+        top=params.top,
-    print(f"Total target blocks/subnets: {len(final_blocked_targets)}")
+        quiet=params.quiet,
-
+        full=params.full,
-    # --- Step 6: write consolidated blocklists to reports/app/data ----------
+        min_denied=params.min_denied,
-    # Capture the previous blocklist state BEFORE the files are overwritten so
+    )
-    # the change report can compute the complete-set delta.
+
-    app_data_dir = os.path.join(params.report_dir, "app", "data")
+    npm_analyzer = NpmAnalyzer(
-    blocklist_v4_path = os.path.join(app_data_dir, "blocklist_ipv4.txt")
+        log_path=params.npm_log_path,
-    blocklist_v6_path = os.path.join(app_data_dir, "blocklist_ipv6.txt")
+        report_dir=params.npm_report_dir,
-    prev_v4 = current_file_lines(blocklist_v4_path)
+        top=params.top,
-    prev_v6 = current_file_lines(blocklist_v6_path)
+        quiet=params.quiet,
-    history_v4 = committed_lines(blocklist_v4_path)
+        full=params.full,
-    history_v6 = committed_lines(blocklist_v6_path)
+        min_errors=params.min_errors,
-
+        error_ratio=params.error_ratio,
-    write_blocklist_files(blocked_ipv4, blocked_ipv6, params.report_dir)
+    )
-    print(f"blocklist files:\n  {blocklist_v4_path}\n  {blocklist_v6_path}")
+
-
+    print("=== Running copyparty scan ===")
-    # --- Steps 5+7: compare current state with candidates and apply ---------
+    cp_exists = True
-    firewall_changes: dict | None = None
+    try:
-    blocklist_report_path: str | None = None
+        cp_analyzer.run(db_scan_id=db_scan_id)
-    if firewall is not None:
+    except Exception as e:
-        try:
+        errors.append(f"copyparty scan failed: {e}")
-            firewall_changes = firewall.sync_blocklist(
+        print(f"Error running copyparty scan: {e}")
-                blocked_ipv4, blocked_ipv6, dry_run=params.dry_run,
+        cp_exists = False
-                report_dir=params.report_dir)
+
-        finally:
+    print("\n=== Running NPM scan ===")
-            firewall.disconnect()
+    npm_exists = True
-        action = "Would" if params.dry_run else ""
+    try:
-        print(f"{action} add {len(firewall_changes['added_ipv4'])} IPv4 and "
+        npm_analyzer.run(db_scan_id=db_scan_id)
-              f"{len(firewall_changes['added_ipv6'])} IPv6 addresses; "
+    except Exception as e:
-              f"remove {len(firewall_changes['removed_ipv4'])} IPv4 and "
+        errors.append(f"NPM scan failed: {e}")
-              f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.")
+        print(f"Error running NPM scan: {e}")
-
+        npm_exists = False
-        # Timestamped change report — only when the router was reachable.
+
-        complete_v4, complete_v6 = cumulative_blocklist(
+    if not cp_exists and not npm_exists:
-            blocklist_v4_path, blocklist_v6_path,
+        message = "Both scans failed. Unable to generate consolidated blocklist."
-            blocked_ipv4, blocked_ipv6)
+        errors.append(message)
-        blocklist_report_path = write_blocklist_report(
+        print(f"\n{message}")
-            params.report_dir,
+        if firewall is not None:
-            history_v4=history_v4, history_v6=history_v6,
+            firewall.disconnect()
-            prev_v4=prev_v4, prev_v6=prev_v6,
+        if db_scan_id is not None:
-            complete_v4=complete_v4, complete_v6=complete_v6,
+            try:
-            firewall_changes=firewall_changes,
+                from ..db import get_connection
-            dry_run=params.dry_run,
+                from ..db import repositories as repo
-            errors=errors)
+                repo.finish_scan(get_connection(), db_scan_id, status="error",
-        print(f"blocklist report: {blocklist_report_path}")
+                                 error="; ".join(errors))
-
+            except Exception:
-    # --- Step 8: prune old reports (retention) ------------------------------
+                pass
-    # Each report subdirectory keeps at most REPORT_RETENTION_MAX_FILES files,
+        return ScanResult(ok=False, errors=errors)
-    # all newer than REPORT_RETENTION_DAYS days. The blocklist files in
+
-    # ``app/data`` are always protected. ``--dry-run`` only previews.
+    # Consolidate block candidates
-    pruned = prune_report_dirs(
+    consolidated_candidates: set[str] = set()
-        params.report_dir, params.copyparty_report_dir, params.npm_report_dir,
+    if cp_exists:
-        dry_run=params.dry_run)
+        consolidated_candidates.update(cp_analyzer.block_candidates())
-    total_pruned = sum(len(paths) for paths in pruned.values())
+    if npm_exists:
-    action = "Would prune" if params.dry_run else "Pruned"
+        consolidated_candidates.update(npm_analyzer.block_candidates())
-    print(f"{action} {total_pruned} report file(s) "
+
-          f"(keep {REPORT_RETENTION_DAYS}d / "
+    # Exclude whitelisted IPs
-          f"max {REPORT_RETENTION_MAX_FILES} per dir)")
+    final_blocked_ips = {
-
+        ip for ip in consolidated_candidates if not is_whitelisted(ip)}
-    return ScanResult(
+
-        ok=True,
+    # Group into subnets via WHOIS if multiple malicious IPs in same subnet
-        cp_exists=cp_exists,
+    print("\nGrouping blocked IPs into subnets using WHOIS...")
-        npm_exists=npm_exists,
+    final_blocked_targets = consolidate_by_subnet(
-        cp_candidates=cp_candidates,
+        final_blocked_ips, threshold=SUBNET_THRESHOLD)
-        npm_candidates=npm_candidates,
+    blocked_ipv4, blocked_ipv6 = split_cidr_versions(final_blocked_targets)
-        blocked_ips=len(final_blocked_ips),
+
-        blocked_targets=len(final_blocked_targets),
+    cp_candidates = len(cp_analyzer.block_candidates()) if cp_exists else 0
-        blocked_ipv4=blocked_ipv4,
+    npm_candidates = len(npm_analyzer.block_candidates()) if npm_exists else 0
-        blocked_ipv6=blocked_ipv6,
+
-        blocklist_v4_path=blocklist_v4_path,
+    print("\n=== Consolidated Summary ===")
-        blocklist_v6_path=blocklist_v6_path,
+    print(f"copyparty block candidates: {cp_candidates}")
-        firewall_connected=firewall is not None,
+    print(f"NPM block candidates:       {npm_candidates}")
-        firewall_changes=firewall_changes,
+    print(f"Total unified blocked IPs:  {len(final_blocked_ips)}")
-        blocklist_report_path=blocklist_report_path,
+    print(f"Total target blocks/subnets: {len(final_blocked_targets)}")
-        errors=errors,
+
-    )
+    # --- Step 6: write consolidated blocklists to reports/app/data ----------
-
+    # Capture the previous blocklist state BEFORE the files are overwritten so
-
+    # the change report can compute the complete-set delta.
-def _build_parser() -> argparse.ArgumentParser:
+    app_data_dir = os.path.join(params.report_dir, "app", "data")
-    parser = argparse.ArgumentParser(
+    blocklist_v4_path = os.path.join(app_data_dir, "blocklist_ipv4.txt")
-        description="Consolidated IDS Scanner: run copyparty + NPM analyzers and generate unified blocklist."
+    blocklist_v6_path = os.path.join(app_data_dir, "blocklist_ipv6.txt")
-    )
+    prev_v4 = current_file_lines(blocklist_v4_path)
-
+    prev_v6 = current_file_lines(blocklist_v6_path)
-    # copyparty arguments
+    history_v4 = committed_lines(blocklist_v4_path)
-    parser.add_argument("--copyparty-log-path", default=COPYPARTY_LOG_PATH,
+    history_v6 = committed_lines(blocklist_v6_path)
-                        help=f"copyparty log directory (default: {COPYPARTY_LOG_PATH})")
+
-    parser.add_argument("--copyparty-report-dir", default=COPYPARTY_REPORT_DIR,
+    write_blocklist_files(blocked_ipv4, blocked_ipv6, params.report_dir)
-                        help=f"copyparty report directory (default: {COPYPARTY_REPORT_DIR})")
+    print(f"blocklist files:\n  {blocklist_v4_path}\n  {blocklist_v6_path}")
-    parser.add_argument("--min-denied", type=int, default=DEFAULT_MIN_DENIED,
+
-                        help=f"minimum copyparty denied hits before flagging (default: {DEFAULT_MIN_DENIED})")
+    # --- Steps 5+7: compare current state with candidates and apply ---------
-
+    firewall_changes: dict | None = None
-    # NPM arguments
+    blocklist_report_path: str | None = None
-    parser.add_argument("--npm-log-path", default=NPM_LOG_PATH,
+    if firewall is not None:
-                        help=f"NPM log directory (default: {NPM_LOG_PATH})")
+        try:
-    parser.add_argument("--npm-report-dir", default=NPM_REPORT_DIR,
+            firewall_changes = firewall.sync_blocklist(
-                        help=f"NPM report directory (default: {NPM_REPORT_DIR})")
+                blocked_ipv4, blocked_ipv6, dry_run=params.dry_run,
-    parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
+                report_dir=params.report_dir)
-                        help=f"minimum NPM error responses to flag (default: {DEFAULT_MIN_ERRORS})")
+        finally:
-    parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
+            firewall.disconnect()
-                        help=f"minimum NPM error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
+        action = "Would" if params.dry_run else ""
-
+        print(f"{action} add {len(firewall_changes['added_ipv4'])} IPv4 and "
-    # Shared arguments
+              f"{len(firewall_changes['added_ipv6'])} IPv6 addresses; "
-    parser.add_argument("--report-dir", default=REPORT_DIR,
+              f"remove {len(firewall_changes['removed_ipv4'])} IPv4 and "
-                        help=f"consolidated reports directory (default: {REPORT_DIR})")
+              f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.")
-    parser.add_argument("--top", type=int, default=DEFAULT_TOP,
+
-                        help=f"rows in the top-IP tables (default: {DEFAULT_TOP})")
+        # Timestamped change report — only when the router was reachable.
-    parser.add_argument("--quiet", action="store_true",
+        # Source of truth for the cumulative set: the SQLite |ledger| first
-                        help="write files, suppress console reports")
+        # (B001), falling back to the git-history reconstruction only if the
-    parser.add_argument("--full", action="store_true",
+        # DB is unavailable.
-                        help="rescan all log files from the start (ignore incremental state)")
+        complete_v4, complete_v6 = _db_complete_blocklist(
-    parser.add_argument("--update-firewall", action="store_true",
+            blocked_ipv4, blocked_ipv6)
-                        help="synchronize the MikroTik Blocked list after scanning")
+        if not complete_v4 and not complete_v6:
-    parser.add_argument("--dry-run", action="store_true",
+            complete_v4, complete_v6 = cumulative_blocklist(
-                        help="preview firewall changes without applying them")
+                blocklist_v4_path, blocklist_v6_path,
-    return parser
+                blocked_ipv4, blocked_ipv6)
-
+        blocklist_report_path = write_blocklist_report(
-
+            params.report_dir,
-def main(argv: list[str] | None = None) -> None:
+            history_v4=history_v4, history_v6=history_v6,
-    args = _build_parser().parse_args(argv)
+            prev_v4=prev_v4, prev_v6=prev_v6,
-    params = ScanParams(
+            complete_v4=complete_v4, complete_v6=complete_v6,
-        copyparty_log_path=args.copyparty_log_path,
+            firewall_changes=firewall_changes,
-        copyparty_report_dir=args.copyparty_report_dir,
+            dry_run=params.dry_run,
-        min_denied=args.min_denied,
+            errors=errors,
-        npm_log_path=args.npm_log_path,
+            scan_id=db_scan_id)
-        npm_report_dir=args.npm_report_dir,
+        print(f"blocklist report: {blocklist_report_path}")
-        min_errors=args.min_errors,
+
-        error_ratio=args.error_ratio,
+    # --- Step 8: prune old reports (retention) ------------------------------
-        report_dir=args.report_dir,
+    # Each report subdirectory keeps at most REPORT_RETENTION_MAX_FILES files,
-        top=args.top,
+    # all newer than REPORT_RETENTION_DAYS days. The blocklist files in
-        quiet=args.quiet,
+    # ``app/data`` are always protected. ``--dry-run`` only previews.
-        full=args.full,
+    pruned = prune_report_dirs(
-        update_firewall=args.update_firewall,
+        params.report_dir, params.copyparty_report_dir, params.npm_report_dir,
-        dry_run=args.dry_run,
+        dry_run=params.dry_run)
-    )
+    total_pruned = sum(len(paths) for paths in pruned.values())
-    run_scan(params)
+    action = "Would prune" if params.dry_run else "Pruned"
-
+    print(f"{action} {total_pruned} report file(s) "
-
+          f"(keep {REPORT_RETENTION_DAYS}d / "
-if __name__ == "__main__":
+          f"max {REPORT_RETENTION_MAX_FILES} per dir)")
-    main()
+
+    # --- Step 9: persist consolidated results to SQLite ---------------------
+    # Sync the blocklist ledger and close out the scan record regardless of
+    # whether the router sync was attempted.
+    _sync_blocklist_to_db(
+        firewall is not None, blocked_ipv4, blocked_ipv6,
+        added_by=("scan" if db_scan_id is None else f"scan:{db_scan_id}"))
+    if db_scan_id is not None:
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            conn = get_connection()
+            result_payload = {
+                "ok": True,
+                "cp_candidates": cp_candidates,
+                "npm_candidates": npm_candidates,
+                "blocked_ips": len(final_blocked_ips),
+                "blocked_targets": len(final_blocked_targets),
+                "firewall_connected": firewall is not None,
+            }
+            repo.finish_scan(
+                conn, db_scan_id, status="done",
+                result=result_payload, log_files=0,
+                request_count=(cp_analyzer and len(cp_analyzer.requests) or 0)
+                + (npm_analyzer and len(npm_analyzer.requests) or 0),
+                error="; ".join(errors) or None,
+            )
+        except Exception as exc:
+            print(f"WARN: could not finalize scan in db ({exc})")
+
+    return ScanResult(
+        ok=True,
+        cp_exists=cp_exists,
+        npm_exists=npm_exists,
+        cp_candidates=cp_candidates,
+        npm_candidates=npm_candidates,
+        blocked_ips=len(final_blocked_ips),
+        blocked_targets=len(final_blocked_targets),
+        blocked_ipv4=blocked_ipv4,
+        blocked_ipv6=blocked_ipv6,
+        blocklist_v4_path=blocklist_v4_path,
+        blocklist_v6_path=blocklist_v6_path,
+        firewall_connected=firewall is not None,
+        firewall_changes=firewall_changes,
+        blocklist_report_path=blocklist_report_path,
+        errors=errors,
+    )
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        description="Consolidated IDS Scanner: run copyparty + NPM analyzers and generate unified blocklist."
+    )
+
+    # copyparty arguments
+    parser.add_argument("--copyparty-log-path", default=COPYPARTY_LOG_PATH,
+                        help=f"copyparty log directory (default: {COPYPARTY_LOG_PATH})")
+    parser.add_argument("--copyparty-report-dir", default=COPYPARTY_REPORT_DIR,
+                        help=f"copyparty report directory (default: {COPYPARTY_REPORT_DIR})")
+    parser.add_argument("--min-denied", type=int, default=DEFAULT_MIN_DENIED,
+                        help=f"minimum copyparty denied hits before flagging (default: {DEFAULT_MIN_DENIED})")
+
+    # NPM arguments
+    parser.add_argument("--npm-log-path", default=NPM_LOG_PATH,
+                        help=f"NPM log directory (default: {NPM_LOG_PATH})")
+    parser.add_argument("--npm-report-dir", default=NPM_REPORT_DIR,
+                        help=f"NPM report directory (default: {NPM_REPORT_DIR})")
+    parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
+                        help=f"minimum NPM error responses to flag (default: {DEFAULT_MIN_ERRORS})")
+    parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
+                        help=f"minimum NPM error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
+
+    # Shared arguments
+    parser.add_argument("--report-dir", default=REPORT_DIR,
+                        help=f"consolidated reports directory (default: {REPORT_DIR})")
+    parser.add_argument("--top", type=int, default=DEFAULT_TOP,
+                        help=f"rows in the top-IP tables (default: {DEFAULT_TOP})")
+    parser.add_argument("--quiet", action="store_true",
+                        help="write files, suppress console reports")
+    parser.add_argument("--full", action="store_true",
+                        help="rescan all log files from the start (ignore incremental state)")
+    parser.add_argument("--update-firewall", action="store_true",
+                        help="synchronize the MikroTik Blocked list after scanning")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="preview firewall changes without applying them")
+    return parser
+
+
+def main(argv: list[str] | None = None) -> None:
+    args = _build_parser().parse_args(argv)
+    params = ScanParams(
+        copyparty_log_path=args.copyparty_log_path,
+        copyparty_report_dir=args.copyparty_report_dir,
+        min_denied=args.min_denied,
+        npm_log_path=args.npm_log_path,
+        npm_report_dir=args.npm_report_dir,
+        min_errors=args.min_errors,
+        error_ratio=args.error_ratio,
+        report_dir=args.report_dir,
+        top=args.top,
+        quiet=args.quiet,
+        full=args.full,
+        update_firewall=args.update_firewall,
+        dry_run=args.dry_run,
+    )
+    run_scan(params)
+
+
+if __name__ == "__main__":
+    main()

app/templates/analysis.html

@@ -0,0 +1,191 @@
+{% extends "base.html" %} {% block title %}Log Analysis — Intrusion Detection{%
+endblock %} {% block content %}
+<h1>Log Analysis</h1>
+<p class="muted">
+  Detailed analysis of persisted log data from the SQLite store. Select an IP
+  to see its full request history across all log sources.
+</p>
+
+<div class="panel">
+  <div class="panel-head">
+    <h2>Top IPs</h2>
+    <span class="muted" id="top-disclaimer"></span>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>IP</th>
+        <th class="num">Requests</th>
+        <th class="num">Errors</th>
+        <th class="num">Malicious</th>
+        <th></th>
+      </tr>
+    </thead>
+    <tbody id="top-ips"></tbody>
+  </table>
+</div>
+
+<div class="panel">
+  <div class="panel-head">
+    <h2>Cross-source IPs</h2>
+    <span class="muted">Seen in more than one log source</span>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>IP</th>
+        <th class="num">Sources</th>
+        <th>Source list</th>
+        <th class="num">Total</th>
+      </tr>
+    </thead>
+    <tbody id="cross-source"></tbody>
+  </table>
+</div>
+
+<div class="panel" id="ip-panel" hidden>
+  <div class="panel-head">
+    <h2 id="ip-title">Request history</h2>
+    <a class="btn ghost" href="#" id="ip-close">✕ Close</a>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Time</th>
+        <th>Source</th>
+        <th>Method</th>
+        <th>Path</th>
+        <th class="num">Status</th>
+        <th>UA</th>
+      </tr>
+    </thead>
+    <tbody id="ip-timeline"></tbody>
+  </table>
+</div>
+{% endblock %} {% block scripts %}
+<script>
+  function showIp(ip) {
+    const panel = document.getElementById("ip-panel");
+    const body = document.getElementById("ip-timeline");
+    document.getElementById("ip-title").textContent = `Request history — ${ip}`;
+    panel.hidden = false;
+    body.innerHTML =
+      '<tr><td colspan="6" class="muted">Loading…</td></tr>';
+    api(`/api/analysis/ip/${encodeURIComponent(ip)}`)
+      .then((data) => {
+        body.innerHTML = "";
+        const tl = data.timeline || [];
+        if (!tl.length) {
+          body.innerHTML =
+            '<tr><td colspan="6" class="muted">No persisted requests for this IP.</td></tr>';
+          return;
+        }
+        for (const r of tl) {
+          const status = r.status ?? "—";
+          const row = el(
+            "tr",
+            el("td", {
+              class: "mono",
+              text: r.ts || "—",
+            }),
+            el("td", { text: r.source }),
+            el("td", { text: r.method || "—" }),
+            el("td", { class: "mono", text: r.path || "—" }),
+            el(
+              "td",
+              {
+                class: "num",
+                text: status,
+                style: status >= 400 ? "color:var(--danger)" : "",
+              },
+            ),
+            el("td", { class: "muted", text: r.ua || "—" }),
+          );
+          if (r.is_error) row.classList.add("row-error");
+          body.append(row);
+        }
+      })
+      .catch((e) => toast(e.message, "error"));
+    panel.scrollIntoView({ behavior: "smooth", block: "start" });
+  }
+
+  async function loadTop() {
+    const body = document.getElementById("top-ips");
+    body.innerHTML = "";
+    try {
+      const data = await api("/api/analysis/top-ips?limit=50");
+      body.innerHTML = "";
+      const ips = data.top_ips || [];
+      document.getElementById("top-disclaimer").textContent =
+        `showing top ${ips.length} IPs`;
+      if (!ips.length) {
+        body.innerHTML =
+          '<tr><td colspan="5" class="muted">No persisted requests yet — run a scan to populate the SQLite store.</td></tr>';
+        return;
+      }
+      for (const r of ips) {
+        body.append(
+          el(
+            "tr",
+            el("td", { class: "mono", text: r.ip }),
+            el("td", { class: "num", text: r.total }),
+            el("td", { class: "num", text: r.errors }),
+            el("td", { class: "num", text: r.malicious_hits }),
+            el(
+              "td",
+              el(
+                "button",
+                { class: "btn ghost", onclick: () => showIp(r.ip) },
+                "Detail",
+              ),
+            ),
+          ),
+        );
+      }
+    } catch (e) {
+      body.innerHTML =
+        '<tr><td colspan="5" class="muted b">SQLite analysis unavailable.</td></tr>';
+      toast(e.message, "error");
+    }
+  }
+
+  async function loadCrossSource() {
+    const body = document.getElementById("cross-source");
+    body.innerHTML = "";
+    try {
+      const data = await api("/api/analysis/cross-source?limit=100");
+      const ips = data.ips || [];
+      if (!ips.length) {
+        body.innerHTML =
+          '<tr><td colspan="4" class="muted">No IPs seen across multiple sources yet.</td></tr>';
+        return;
+      }
+      for (const r of ips) {
+        body.append(
+          el(
+            "tr",
+            el("td", { class: "mono", text: r.ip }),
+            el("td", { class: "num", text: r.sources }),
+            el("td", { text: r.source_list }),
+            el("td", { class: "num", text: r.total }),
+          ),
+        );
+      }
+    } catch (e) {
+      body.innerHTML =
+        '<tr><td colspan="4" class="muted b">SQLite analysis unavailable.</td></tr>';
+      toast(e.message, "error");
+    }
+  }
+
+  document
+    .getElementById("ip-close")
+    .addEventListener("click", (e) => {
+      e.preventDefault();
+      document.getElementById("ip-panel").hidden = true;
+    });
+
+  loadTop();
+  loadCrossSource();
+</script>
+{% endblock %}

app/templates/base.html

@@ -40,6 +40,11 @@
           class="{{ 'active' if request.path == '/graph' }}"
           >Graph</a
         >
+        <a
+          href="/analysis"
+          class="{{ 'active' if request.path == '/analysis' }}"
+          >Analysis</a
+        >
       </nav>
     </header>
     <main class="container">{% block content %}{% endblock %}</main>

app/templates/dashboard.html

@@ -57,6 +57,13 @@ endblock %} {% block content %}
           s.firewall_snapshot && s.firewall_snapshot.exists ? "ok" : "muted",
       },
     ];
+    if (s.analysis && s.analysis.available) {
+      cards.push({
+        label: "SQLite stores",
+        value: `${s.analysis.request_count ?? 0} req / ${s.analysis.scan_count ?? 0} scans`,
+        tone: "ok",
+      });
+    }
     const box = document.getElementById("status-cards");
     box.innerHTML = "";
     for (const c of cards) {

app/templates/firewall.html

@@ -23,6 +23,11 @@ endblock %} {% block content %}
   <div class="panel-head"><h2>Firewall rules</h2></div>
   <div id="firewall-rules"></div>
 </div>
+
+<div class="panel">
+  <div class="panel-head"><h2>Snapshot history (SQLite)</h2></div>
+  <div id="fw-snapshots"></div>
+</div>
 {% endblock %} {% block scripts %}
 <script>
   async function loadState() {
@@ -35,6 +40,45 @@ endblock %} {% block content %}
     }
   }
 
+  async function loadSnapshots() {
+    const box = document.getElementById("fw-snapshots");
+    try {
+      const d = await api("/api/firewall/snapshots?limit=10");
+      const snaps = d.snapshots || [];
+      if (!snaps.length) {
+        box.innerHTML =
+          '<p class="muted">No SQLite snapshots yet — they are recorded on each <code>--update-firewall</code> run.</p>';
+        return;
+      }
+      box.innerHTML = "";
+      const rows = snaps.map((s) =>
+        el(
+          "tr",
+          el("td", { class: "mono", text: `#${s.id}` }),
+          el("td", { text: fmtTime(s.captured_at) }),
+          el("td", { text: s.kind }),
+          el("td", { text: s.trigger }),
+          el("td", { class: "num", text: s.addr_entries }),
+          el("td", { class: "num", text: s.rule_entries }),
+        ),
+      );
+      box.append(
+        el(
+          "table",
+          { class: "table" },
+          el(
+            "thead",
+            el("tr", {}, el("th", { text: "ID" }), el("th", { text: "Captured" }), el("th", { text: "Kind" }), el("th", { text: "Trigger" }), el("th", { text: "Addr" }), el("th", { text: "Rules" })),
+          ),
+          el("tbody", {}, ...rows),
+        ),
+      );
+    } catch (e) {
+      box.innerHTML=
+        '<p class="muted">Snapshot history unavailable (SQLite).</p>';
+    }
+  }
+
   function renderAddressLists(addr) {
     const box = document.getElementById("address-lists");
     box.innerHTML = "";
@@ -172,5 +216,6 @@ endblock %} {% block content %}
     .getElementById("apply-btn")
     .addEventListener("click", () => sync(false));
   loadState();
+  loadSnapshots();
 </script>
 {% endblock %}

app/templates/graph.html

@@ -1,227 +1,210 @@
-{% extends "base.html" %} {% block title %}Graph — Intrusion Detection{%
+{% extends "base.html" %} {% block title %}Graph — Intrusion Detection{% endblock %} {% block content %}
-endblock %} {% block content %}
+<h1>IPv4 address space map</h1>
-<h1>IPv4 address space map</h1>
+<p class="muted">
-<p class="muted">
+  One pixel per /24 network, arranged as 16×16 tiles of /8 (first octet). Blocked targets are red, whitelisted ranges
-  One pixel per /24 network, arranged as 16×16 tiles of /8 (first octet).
+  are green.
-  Blocked targets are red, whitelisted ranges are green.
+</p>
-</p>
+
-
+<div class="panel">
-<div class="panel">
+  <div class="panel-head">
-  <div class="panel-head">
+    <div class="graph-controls">
-    <div class="graph-controls">
+      <label class="check" style="margin-top: 0">
-      <label class="check" style="margin-top: 0">
+        <input type="checkbox" id="show-whitelisted" checked />
-        <input type="checkbox" id="show-whitelisted" checked />
+        Show whitelisted
-        Show whitelisted
+      </label>
-      </label>
+      <span class="legend">
-      <span class="legend">
+        <span class="sw red"></span> Blocked <span class="sw green"></span> Whitelisted
-        <span class="sw red"></span> Blocked
+        <span class="sw plain"></span> Unlisted
-        <span class="sw green"></span> Whitelisted
+      </span>
-        <span class="sw plain"></span> Unlisted
+    </div>
-      </span>
+    <button class="btn ghost" id="graph-refresh">↻ Refresh</button>
-    </div>
+  </div>
-    <button class="btn ghost" id="graph-refresh">↻ Refresh</button>
+  <div class="graph-wrap" id="graph-wrap">
-  </div>
+    <canvas id="graph-canvas" width="4096" height="4096"></canvas>
-  <div class="graph-wrap" id="graph-wrap">
+    <div class="graph-overlay" id="graph-overlay"></div>
-    <canvas id="graph-canvas" width="4096" height="4096"></canvas>
+    <div class="graph-tooltip hidden" id="graph-tooltip"></div>
-    <div class="graph-overlay" id="graph-overlay"></div>
+  </div>
-    <div class="graph-tooltip hidden" id="graph-tooltip"></div>
+  <p class="muted mono" id="graph-stats">Loading…</p>
-  </div>
+</div>
-  <p class="muted mono" id="graph-stats">Loading…</p>
+{% endblock %} {% block scripts %}
-</div>
+<script>
-{% endblock %} {% block scripts %}
+  // Layout mirrors plot.py: 16x16 grid of /8 tiles, each 256x256 px (one pixel
-<script>
+  // per /24). Blocked ranges override whitelisted ranges.
-  // Layout mirrors plot.py: 16x16 grid of /8 tiles, each 256x256 px (one pixel
+  const TILE = 16;
-  // per /24). Blocked ranges override whitelisted ranges.
+  const SIZE = 256;
-  const TILE = 16;
+  const TOTAL = TILE * SIZE; // 4096
-  const SIZE = 256;
+
-  const TOTAL = TILE * SIZE; // 4096
+  const V_UNLISTED = 0;
-
+  const V_WHITELISTED = 1;
-  const V_UNLISTED = 0;
+  const V_BLOCKED = 2;
-  const V_WHITELISTED = 1;
+  const COLOR_WHITELISTED = "#43a047";
-  const V_BLOCKED = 2;
+  const COLOR_BLOCKED = "#e53935";
-  const COLOR_WHITELISTED = "#43a047";
+
-  const COLOR_BLOCKED = "#e53935";
+  const canvas = document.getElementById("graph-canvas");
-
+  const ctx = canvas.getContext("2d");
-  const canvas = document.getElementById("graph-canvas");
+  const overlay = document.getElementById("graph-overlay");
-  const ctx = canvas.getContext("2d");
+  const tooltip = document.getElementById("graph-tooltip");
-  const overlay = document.getElementById("graph-overlay");
+  const stats = document.getElementById("graph-stats");
-  const tooltip = document.getElementById("graph-tooltip");
+
-  const stats = document.getElementById("graph-stats");
+  // /24 index (0..2^24-1) -> value (whitelisted/blocked)
-
+  const marked = new Map();
-  // /24 index (0..2^24-1) -> value (whitelisted/blocked)
+
-  const marked = new Map();
+  function parseCIDR(s) {
-
+    const m = String(s)
-  function parseCIDR(s) {
+      .trim()
-    const m = String(s)
+      .match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})(?:\/(\d{1,2}))?$/);
-      .trim()
+    if (!m) return null;
-      .match(/^(\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})(?:\/(\d{1,2}))?$/);
+    const o = [m[1], m[2], m[3], m[4]].map(Number);
-    if (!m) return null;
+    if (o.some((n) => n > 255)) return null;
-    const o = [m[1], m[2], m[3], m[4]].map(Number);
+    const prefix = m[5] === undefined ? 32 : Number(m[5]);
-    if (o.some((n) => n > 255)) return null;
+    if (prefix < 0 || prefix > 32) return null;
-    const prefix = m[5] === undefined ? 32 : Number(m[5]);
+    const ip = ((o[0] << 24) | (o[1] << 16) | (o[2] << 8) | o[3]) >>> 0;
-    if (prefix < 0 || prefix > 32) return null;
+    const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0;
-    const ip = ((o[0] << 24) | (o[1] << 16) | (o[2] << 8) | o[3]) >>> 0;
+    const network = (ip & mask) >>> 0;
-    const mask = prefix === 0 ? 0 : (0xffffffff << (32 - prefix)) >>> 0;
+    const broadcast = (ip | (~mask >>> 0)) >>> 0;
-    const network = (ip & mask) >>> 0;
+    return {
-    const broadcast = (ip | (~mask >>> 0)) >>> 0;
+      first24: network >>> 8,
-    return {
+      last24: broadcast >>> 8,
-      first24: network >>> 8,
+      label: `${o[0]}.${o[1]}.${o[2]}.${o[3]}/${prefix}`,
-      last24: broadcast >>> 8,
+    };
-      label: `${o[0]}.${o[1]}.${o[2]}.${o[3]}/${prefix}`,
+  }
-    };
+
-  }
+  function idxToPos(idx) {
-
+    const a = (idx >>> 16) & 0xff;
-  function idxToPos(idx) {
+    const b = (idx >>> 8) & 0xff;
-    const a = (idx >>> 16) & 0xff;
+    const c = idx & 0xff;
-    const b = (idx >>> 8) & 0xff;
+    return {
-    const c = idx & 0xff;
+      x: (a % TILE) * SIZE + b,
-    return {
+      y: Math.floor(a / TILE) * SIZE + c,
-      x: (a % TILE) * SIZE + b,
+    };
-      y: Math.floor(a / TILE) * SIZE + c,
+  }
-    };
+
-  }
+  function markRange(range, value) {
-
+    for (let idx = range.first24; idx <= range.last24; idx++) {
-  function markRange(range, value) {
+      const prev = marked.get(idx);
-    for (let idx = range.first24; idx <= range.last24; idx++) {
+      if (prev === undefined || value > prev) marked.set(idx, value);
-      const prev = marked.get(idx);
+    }
-      if (prev === undefined || value > prev) marked.set(idx, value);
+  }
-    }
+
-  }
+  function drawOverlay() {
-
+    overlay.innerHTML = "";
-  function drawOverlay() {
+    // /8 tile borders
-    overlay.innerHTML = "";
+    for (let i = 0; i <= TILE; i++) {
-    // /8 tile borders
+      const pct = (i / TILE) * 100;
-    for (let i = 0; i <= TILE; i++) {
+      const v = el("div", { class: "g-line g-v", style: `left:${pct}%` });
-      const pct = (i / TILE) * 100;
+      const h = el("div", { class: "g-line g-h", style: `top:${pct}%` });
-      const v = el("div", { class: "g-line g-v", style: `left:${pct}%` });
+      overlay.append(v, h);
-      const h = el("div", { class: "g-line g-h", style: `top:${pct}%` });
+    }
-      overlay.append(v, h);
+    // first-octet labels, one per /8 tile
-    }
+    for (let a = 0; a < 256; a++) {
-    // first-octet labels, one per /8 tile
+      const col = a % TILE;
-    for (let a = 0; a < 256; a++) {
+      const row = Math.floor(a / TILE);
-      const col = a % TILE;
+      overlay.append(
-      const row = Math.floor(a / TILE);
+        el("div", {
-      overlay.append(
+          class: "g-label",
-        el("div", {
+          style: `left:${(col / TILE) * 100}%;top:${(row / TILE) * 100}%`,
-          class: "g-label",
+          text: `${a}.0.0.0/8`,
-          style: `left:${(col / TILE) * 100}%;top:${(row / TILE) * 100}%`,
+        }),
-          text: `${a}.0.0.0/8`,
+      );
-        }),
+    }
-      );
+  }
-    }
+
-  }
+  function render() {
-
+    const showWl = document.getElementById("show-whitelisted").checked;
-  function render() {
+    ctx.fillStyle = "#0c0c0c";
-    const showWl = document.getElementById("show-whitelisted").checked;
+    ctx.fillRect(0, 0, TOTAL, TOTAL);
-    ctx.fillStyle = "#ffffff";
+    for (const [idx, value] of marked) {
-    ctx.fillRect(0, 0, TOTAL, TOTAL);
+      if (!showWl && value === V_WHITELISTED) continue;
-    for (const [idx, value] of marked) {
+      const p = idxToPos(idx);
-      if (!showWl && value === V_WHITELISTED) continue;
+      ctx.fillStyle = value === V_BLOCKED ? COLOR_BLOCKED : COLOR_WHITELISTED;
-      const p = idxToPos(idx);
+      ctx.fillRect(p.x, p.y, 1, 1);
-      ctx.fillStyle = value === V_BLOCKED ? COLOR_BLOCKED : COLOR_WHITELISTED;
+    }
-      ctx.fillRect(p.x, p.y, 1, 1);
+  }
-    }
+
-  }
+  // --- hover tooltip ------------------------------------------------------
-
+
-  // --- hover tooltip ------------------------------------------------------
+  function matchesIn(ranges, idx) {
-
+    const out = [];
-  function matchesIn(ranges, idx) {
+    for (const r of ranges) {
-    const out = [];
+      if (idx >= r.first24 && idx <= r.last24) out.push(r.label);
-    for (const r of ranges) {
+      if (out.length >= 12) break;
-      if (idx >= r.first24 && idx <= r.last24) out.push(r.label);
+    }
-      if (out.length >= 12) break;
+    return out;
-    }
+  }
-    return out;
+
-  }
+  function showTooltip(e, x, y) {
-
+    const tileCol = Math.floor(x / SIZE);
-  function showTooltip(e, x, y) {
+    const tileRow = Math.floor(y / SIZE);
-    const tileCol = Math.floor(x / SIZE);
+    const a = tileRow * TILE + tileCol;
-    const tileRow = Math.floor(y / SIZE);
+    const b = x % SIZE;
-    const a = tileRow * TILE + tileCol;
+    const c = y % SIZE;
-    const b = x % SIZE;
+    const idx = (a << 16) | (b << 8) | c;
-    const c = y % SIZE;
+    const value = marked.get(idx) ?? V_UNLISTED;
-    const idx = (a << 16) | (b << 8) | c;
+    const status = value === V_BLOCKED ? "blocked" : value === V_WHITELISTED ? "whitelisted" : "unlisted";
-    const value = marked.get(idx) ?? V_UNLISTED;
+    const entries = matchesIn(blockedRanges, idx).concat(matchesIn(whitelistedRanges, idx));
-    const status =
+    tooltip.innerHTML = "";
-      value === V_BLOCKED
+    tooltip.append(
-        ? "blocked"
+      el("div", { class: "tt-head mono", text: `${a}.${b}.${c}.0/24` }),
-        : value === V_WHITELISTED
+      el("div", { class: `tt-status ${status}`, text: status }),
-          ? "whitelisted"
+    );
-          : "unlisted";
+    if (entries.length) {
-    const entries = matchesIn(blockedRanges, idx).concat(
+      tooltip.append(el("div", { class: "tt-list mono", text: entries.join("\n") }));
-      matchesIn(whitelistedRanges, idx),
+    } else {
-    );
+      tooltip.append(el("div", { class: "tt-list muted", text: "no matching entry" }));
-    tooltip.innerHTML = "";
+    }
-    tooltip.append(
+    tooltip.classList.remove("hidden");
-      el("div", { class: "tt-head mono", text: `${a}.${b}.${c}.0/24` }),
+    const rect = canvas.getBoundingClientRect();
-      el("div", { class: `tt-status ${status}`, text: status }),
+    let left = e.clientX - rect.left + 14;
-    );
+    let top = e.clientY - rect.top + 14;
-    if (entries.length) {
+    if (left + tooltip.offsetWidth > rect.width) left = e.clientX - rect.left - tooltip.offsetWidth - 14;
-      tooltip.append(
+    if (top + tooltip.offsetHeight > rect.height) top = e.clientY - rect.top - tooltip.offsetHeight - 14;
-        el("div", { class: "tt-list mono", text: entries.join("\n") }),
+    tooltip.style.left = `${Math.max(0, left)}px`;
-      );
+    tooltip.style.top = `${Math.max(0, top)}px`;
-    } else {
+  }
-      tooltip.append(
+
-        el("div", { class: "tt-list muted", text: "no matching entry" }),
+  canvas.addEventListener("mousemove", (e) => {
-      );
+    const rect = canvas.getBoundingClientRect();
-    }
+    const lx = ((e.clientX - rect.left) * TOTAL) / rect.width;
-    tooltip.classList.remove("hidden");
+    const ly = ((e.clientY - rect.top) * TOTAL) / rect.height;
-    const rect = canvas.getBoundingClientRect();
+    const x = Math.floor(lx);
-    let left = e.clientX - rect.left + 14;
+    const y = Math.floor(ly);
-    let top = e.clientY - rect.top + 14;
+    if (x < 0 || x >= TOTAL || y < 0 || y >= TOTAL) {
-    if (left + tooltip.offsetWidth > rect.width)
+      tooltip.classList.add("hidden");
-      left = e.clientX - rect.left - tooltip.offsetWidth - 14;
+      return;
-    if (top + tooltip.offsetHeight > rect.height)
+    }
-      top = e.clientY - rect.top - tooltip.offsetHeight - 14;
+    showTooltip(e, x, y);
-    tooltip.style.left = `${Math.max(0, left)}px`;
+  });
-    tooltip.style.top = `${Math.max(0, top)}px`;
+
-  }
+  canvas.addEventListener("mouseleave", () => {
-
+    tooltip.classList.add("hidden");
-  canvas.addEventListener("mousemove", (e) => {
+  });
-    const rect = canvas.getBoundingClientRect();
+
-    const lx = ((e.clientX - rect.left) * TOTAL) / rect.width;
+  // --- load ---------------------------------------------------------------
-    const ly = ((e.clientY - rect.top) * TOTAL) / rect.height;
+
-    const x = Math.floor(lx);
+  let blockedRanges = [];
-    const y = Math.floor(ly);
+  let whitelistedRanges = [];
-    if (x < 0 || x >= TOTAL || y < 0 || y >= TOTAL) {
+
-      tooltip.classList.add("hidden");
+  async function loadGraph() {
-      return;
+    try {
-    }
+      const d = await api("/api/graph/ipv4");
-    showTooltip(e, x, y);
+      blockedRanges = (d.blocked || []).map(parseCIDR).filter(Boolean);
-  });
+      whitelistedRanges = (d.whitelisted || []).map(parseCIDR).filter(Boolean);
-
+      marked.clear();
-  canvas.addEventListener("mouseleave", () => {
+      for (const r of whitelistedRanges) markRange(r, V_WHITELISTED);
-    tooltip.classList.add("hidden");
+      for (const r of blockedRanges) markRange(r, V_BLOCKED);
-  });
+      render();
-
+      drawOverlay();
-  // --- load ---------------------------------------------------------------
+      const c = d.counts || {};
-
+      stats.textContent = `blocked targets: ${c.blocked ?? blockedRanges.length} · whitelisted ranges: ${c.whitelisted ?? whitelistedRanges.length} · marked /24 pixels: ${marked.size}`;
-  let blockedRanges = [];
+    } catch (err) {
-  let whitelistedRanges = [];
+      stats.textContent = `failed to load graph data: ${err.message}`;
-
+      toast(err.message, "error");
-  async function loadGraph() {
+    }
-    try {
+  }
-      const d = await api("/api/graph/ipv4");
+
-      blockedRanges = (d.blocked || []).map(parseCIDR).filter(Boolean);
+  document.getElementById("show-whitelisted").addEventListener("change", render);
-      whitelistedRanges = (d.whitelisted || []).map(parseCIDR).filter(Boolean);
+  document.getElementById("graph-refresh").addEventListener("click", loadGraph);
-      marked.clear();
+
-      for (const r of whitelistedRanges) markRange(r, V_WHITELISTED);
+  loadGraph();
-      for (const r of blockedRanges) markRange(r, V_BLOCKED);
+</script>
-      render();
+{% endblock %}
-      drawOverlay();
-      const c = d.counts || {};
-      stats.textContent = `blocked targets: ${c.blocked ?? blockedRanges.length} · whitelisted ranges: ${c.whitelisted ?? whitelistedRanges.length} · marked /24 pixels: ${marked.size}`;
-    } catch (err) {
-      stats.textContent = `failed to load graph data: ${err.message}`;
-      toast(err.message, "error");
-    }
-  }
-
-  document
-    .getElementById("show-whitelisted")
-    .addEventListener("change", render);
-  document.getElementById("graph-refresh").addEventListener("click", loadGraph);
-
-  loadGraph();
-</script>
-{% endblock %}

app/templates/reports.html

@@ -1,76 +1,124 @@
-{% extends "base.html" %} {% block title %}Reports — Intrusion Detection{%
+{% extends "base.html" %} {% block title %}Reports — Intrusion Detection{% endblock %} {% block content %}
-endblock %} {% block content %}
+<h1>Reports</h1>
-<h1>Reports</h1>
+<p class="muted">Browse the timestamped reports written by each scanner.</p>
-<p class="muted">Browse the timestamped reports written by each scanner.</p>
+
-
+<div class="panel">
-<div class="panel">
+  <div class="panel-actions" style="margin-top: 0">
-  <div class="panel-actions" style="margin-top: 0">
+    <label style="flex-direction: row; align-items: center; gap: 0.5rem"
-    <label style="flex-direction: row; align-items: center; gap: 0.5rem"
+      >Source
-      >Source
+      <select id="source">
-      <select id="source">
+        <option value="copyparty">copyparty</option>
-        <option value="copyparty">copyparty</option>
+        <option value="npm">npm</option>
-        <option value="npm">npm</option>
+        <option value="app">app</option>
-        <option value="app">app</option>
+        <option value="mikrotik">mikrotik</option>
-        <option value="mikrotik">mikrotik</option>
+      </select>
-      </select>
+    </label>
-    </label>
+    <button class="btn" id="refresh-btn">Refresh</button>
-    <button class="btn" id="refresh-btn">Refresh</button>
+  </div>
-  </div>
+  <div class="list" id="files"></div>
-  <div class="list" id="files"></div>
+</div>
-</div>
+
-
+<div class="panel">
-<div class="panel">
+  <div class="panel-head"><h2 id="content-title">Report content</h2></div>
-  <div class="panel-head"><h2 id="content-title">Report content</h2></div>
+  <pre class="content" id="content">Select a report on the left to view it.</pre>
-  <pre class="content" id="content">
+</div>
-Select a report on the left to view it.</pre
+{% endblock %} {% block scripts %}
-  >
+<script>
-</div>
+  let nextCursor = null;
-{% endblock %} {% block scripts %}
+
-<script>
+  function addReportLink(list, label, onClick) {
-  async function loadFiles() {
+    const link = document.createElement("a");
-    const source = document.getElementById("source").value;
+    link.href = "#";
-    const list = document.getElementById("files");
+    link.textContent = label;
-    list.innerHTML = '<div class="muted">Loading…</div>';
+    link.addEventListener("click", (event) => {
-    try {
+      event.preventDefault();
-      const data = await api(`/api/reports/${source}`);
+      onClick();
-      list.innerHTML = "";
+    });
-      const files = data.files || [];
+    list.append(link);
-      if (!files.length) {
+  }
-        list.innerHTML = '<div class="muted">No reports in this source.</div>';
+
-        return;
+  async function loadFiles(append = false) {
-      }
+    const source = document.getElementById("source").value;
-      for (const name of files) {
+    const list = document.getElementById("files");
-        list.append(
+    if (!append) {
-          el("a", {
+      nextCursor = null;
-            href: "#",
+      list.innerHTML = '<div class="muted">Loading…</div>';
-            text: name,
+    } else {
-            onclick: (ev) => {
+      document.getElementById("load-more")?.remove();
-              ev.preventDefault();
+    }
-              openReport(name);
+
-            },
+    let importedPaths = new Set();
-          }),
+    try {
-        );
+      const query = new URLSearchParams({ source, limit: "50" });
-      }
+      if (append && nextCursor) query.set("cursor", nextCursor);
-    } catch (e) {
+      const data = await api(`/api/reports?${query}`);
-      list.innerHTML = `<div class="muted">${escapeHtml(e.message)}</div>`;
+      if (!append) list.innerHTML = "";
-    }
+      for (const artifact of data.items || []) {
-  }
+        importedPaths.add(artifact.original_path);
-
+        addReportLink(list, `${artifact.generated_at} | ${artifact.report_type} | ${artifact.original_path}`, () =>
-  async function openReport(name) {
+          openArtifact(artifact.id),
-    const source = document.getElementById("source").value;
+        );
-    document.getElementById("content-title").textContent = name;
+      }
-    const pre = document.getElementById("content");
+      nextCursor = data.next_cursor;
-    pre.textContent = "Loading…";
+    } catch (e) {
-    try {
+      if (append) {
-      const data = await api(`/api/reports/${source}/${name}`);
+        list.append(el("div", { className: "muted", text: e.message }));
-      pre.textContent = data.content;
+        return;
-    } catch (e) {
+      }
-      pre.textContent = `Error: ${e.message}`;
+    }
-    }
+
-  }
+    if (!append) {
-
+      try {
-  document.getElementById("refresh-btn").addEventListener("click", loadFiles);
+        const data = await api(`/api/reports/${source}`);
-  document.getElementById("source").addEventListener("change", loadFiles);
+        for (const name of data.files || []) {
-  loadFiles();
+          if (importedPaths.has(`${source}/${name}`)) continue;
-</script>
+          addReportLink(list, `${name} | legacy file`, () => openReport(name));
-{% endblock %}
+        }
+      } catch (e) {
+        if (!list.childElementCount) list.append(el("div", { className: "muted", text: e.message }));
+      }
+    }
+
+    if (nextCursor) {
+      const button = document.createElement("button");
+      button.id = "load-more";
+      button.className = "btn";
+      button.textContent = "Load more";
+      button.addEventListener("click", () => loadFiles(true));
+      list.append(button);
+    }
+    if (!list.childElementCount) list.innerHTML = '<div class="muted">No reports in this source.</div>';
+  }
+
+  async function openArtifact(id) {
+    document.getElementById("content-title").textContent = "Database report";
+    const pre = document.getElementById("content");
+    pre.textContent = "Loading…";
+    try {
+      const data = await api(`/api/report-artifacts/${id}`);
+      document.getElementById("content-title").textContent = data.original_path;
+      const payload = data.payload || {};
+      pre.textContent = payload.format === "text" ? payload.content || "" : JSON.stringify(payload, null, 2);
+    } catch (e) {
+      pre.textContent = `Error: ${e.message}`;
+    }
+  }
+
+  async function openReport(name) {
+    const source = document.getElementById("source").value;
+    document.getElementById("content-title").textContent = name;
+    const pre = document.getElementById("content");
+    pre.textContent = "Loading…";
+    try {
+      const data = await api(`/api/reports/${source}/${name}`);
+      pre.textContent = data.content;
+    } catch (e) {
+      pre.textContent = `Error: ${e.message}`;
+    }
+  }
+
+  document.getElementById("refresh-btn").addEventListener("click", () => loadFiles());
+  document.getElementById("source").addEventListener("change", () => loadFiles());
+  loadFiles();
+</script>
+{% endblock %}

app/utils/blocklist_history.py

@@ -20,6 +20,7 @@ import re
 import subprocess
 from datetime import datetime
 from typing import Any, Iterable
+import hashlib
 
 from .files import write_json
 from .reporting import timestamp_name
@@ -151,6 +152,7 @@ def write_blocklist_report(report_dir: str,
                            firewall_changes: dict | None = None,
                            dry_run: bool = False,
                            errors: Iterable[str] = (),
+                           scan_id: int | None = None,
                            ) -> str:
     """Write the timestamped blocklist report to ``<report_dir>/app/``.
 
@@ -183,4 +185,18 @@ def write_blocklist_report(report_dir: str,
     os.makedirs(app_dir, exist_ok=True)
     path = os.path.join(app_dir, f"{timestamp_name()}.json")
     write_json(path, payload)
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        with open(path, "rb") as file:
+            content = file.read()
+        repo.insert_report_artifact(
+            get_connection(), source="app", report_type="blocklist_change",
+            generated_at=payload["generated_at"], payload=payload,
+            original_path=f"app/{os.path.basename(path)}",
+            content_sha256=hashlib.sha256(content).hexdigest(),
+            scan_id=scan_id,
+        )
+    except Exception as exc:
+        print(f"WARN: blocklist report DB write skipped ({exc})")
     return path

app/utils/constants.py

@@ -53,6 +53,11 @@ MIKROTIK_DATA_DIR = os.path.join(DATA_DIR, "mikrotik")
 
 WHOIS_CACHE_PATH = os.path.join(DATA_DIR, "whois_cache.json")
 
+# Local SQLite database (see .github/instructions/DATABASE.md). The default
+# lives under data/ and holds log records, scan metadata, blocklist/whitelist
+# and firewall history. Override with IDS_DB_PATH (used by app/db).
+DATA_DB_PATH = os.environ.get("IDS_DB_PATH", os.path.join(DATA_DIR, "ids.db"))
+
 # Incremental-scan state: absolute log path -> last processed byte offset
 SCAN_STATE_PATH = os.path.join(DATA_DIR, "scan_state.json")
 

app/utils/whitelists.py

@@ -275,3 +275,20 @@ def refresh_whitelists(force: bool = False) -> bool:
     update_whitelists()
     _refreshed = True
     return True
+
+
+def sync_whitelists_to_db() -> None:
+    """Persist the current in-memory ``WHITELISTS`` into the SQLite store.
+
+    Best-effort (never raises): keeps the DB whitelist tables consistent with
+    the in-memory/provider state so lookups and analysis can rely on them.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        conn = get_connection()
+        for provider, cidrs in WHITELISTS.items():
+            repo.set_whitelist(conn, provider, cidrs,
+                               source="builtin/refresh")
+    except Exception as exc:  # pragma: no cover - defensive
+        print(f"WARN: could not sync whitelists to db ({exc})")

app/web/__init__.py

@@ -131,6 +131,41 @@ def _sync_firewall(dry_run: bool):
     return _with_router(_sync)
 
 
+def _analysis_helpers():
+    """SQLite-backed log-analysis query helpers (None when DB unavailable).
+
+    Functions take the shared DB connection each call so routes stay stateless
+    across the background scan thread.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+    except Exception:
+        return None
+
+    def top_ips(since: str | None = None, limit: int = 50) -> list:
+        return repo.top_ips(get_connection(), since=since, limit=limit)
+
+    def ip_timeline(ip: str) -> list:
+        return repo.ip_timeline(get_connection(), ip=ip)
+
+    def cross_source(limit: int = 100) -> list:
+        return repo.correlate_across_sources(get_connection(), limit=limit)
+
+    def conn():
+        return get_connection()
+
+    def request_count(conn) -> int:
+        return repo.count_requests(conn)
+
+    def scan_count(conn) -> int:
+        return repo.count_scans(conn)
+
+    return {"top_ips": top_ips, "ip_timeline": ip_timeline,
+            "cross_source": cross_source, "conn": conn,
+            "request_count": request_count, "scan_count": scan_count}
+
+
 # ---------------------------------------------------------------------------
 # app factory
 # ---------------------------------------------------------------------------
@@ -155,6 +190,7 @@ def create_app() -> Flask:
         "_params_from_request": _params_from_request,
         "_with_router": _with_router,
         "_sync_firewall": _sync_firewall,
+        "_analysis": _analysis_helpers(),
     }
     register_routes(app, REPORT_SOURCES, SCAN_KINDS, web_scan, helpers)
     return app

app/web/routes.py

@@ -3,9 +3,13 @@
 
 from __future__ import annotations
 
+import csv
+import io
+import json
 import os
 
-from flask import Flask, abort, jsonify, render_template, request
+from flask import Flask, Response, abort, jsonify, render_template, request
+from werkzeug.utils import secure_filename
 
 from ..utils.constants import (
     APP_REPORT_DATA_DIR,
@@ -32,6 +36,40 @@ def register_routes(app: Flask,
     _with_router = helpers["_with_router"]
     _sync_firewall = helpers["_sync_firewall"]
 
+    # --- log analysis (SQLite-backed) --------------------------------------
+
+    _analysis = helpers.get("_analysis")
+
+    @app.get("/analysis")
+    def analysis_page():
+        return render_template("analysis.html")
+
+    @app.get("/api/analysis/top-ips")
+    def analysis_top_ips():
+        if _analysis is None:
+            return jsonify({"error": "analysis unavailable"}), 503
+        since = (request.args.get("since") or "").strip() or None
+        limit = request.args.get("limit", 50, type=int)
+        return jsonify({"top_ips": _analysis["top_ips"](since=since, limit=limit)})
+
+    @app.get("/api/analysis/ip/<ip>")
+    def analysis_ip(ip):
+        if _analysis is None:
+            return jsonify({"error": "analysis unavailable"}), 503
+        return jsonify({"ip": ip,
+                        "timeline": _analysis["ip_timeline"](ip)})
+
+    @app.get("/api/analysis/cross-source")
+    def analysis_cross_source():
+        if _analysis is None:
+            return jsonify({"error": "analysis unavailable"}), 503
+        limit = request.args.get("limit", 100, type=int)
+        return jsonify({"ips": _analysis["cross_source"](limit=limit)})
+
+    @app.get("/api/analysis/manifest")
+    def analysis_manifest():
+        return jsonify({"ok": _analysis is not None})
+
     # --- pages -------------------------------------------------------------
 
     @app.get("/")
@@ -68,6 +106,17 @@ def register_routes(app: Flask,
     def status():
         v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
         v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
+        analysis = {}
+        if _analysis is not None:
+            try:
+                conn = _analysis["conn"]()
+                analysis = {
+                    "available": True,
+                    "request_count": _analysis["request_count"](conn),
+                    "scan_count": _analysis["scan_count"](conn),
+                }
+            except Exception:
+                analysis = {"available": False}
         return jsonify({
             "host": WEB_HOST,
             "port": WEB_PORT,
@@ -79,6 +128,7 @@ def register_routes(app: Flask,
                 "exists": os.path.isdir(MIKROTIK_DATA_DIR),
                 "path": MIKROTIK_DATA_DIR,
             },
+            "analysis": analysis,
         })
 
     # --- scans -------------------------------------------------------------
@@ -108,6 +158,84 @@ def register_routes(app: Flask,
 
     # --- reports -----------------------------------------------------------
 
+    @app.get("/api/reports")
+    def list_report_artifacts():
+        source = (request.args.get("source") or "").strip() or None
+        if source is not None and source not in report_sources:
+            return jsonify({"error": "unknown report source"}), 400
+        limit = request.args.get("limit", default=50, type=int)
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            result = repo.list_report_artifacts(
+                get_connection(), source=source,
+                report_type=(request.args.get("type") or "").strip() or None,
+                since=(request.args.get("since") or "").strip() or None,
+                until=(request.args.get("until") or "").strip() or None,
+                limit=limit,
+                cursor=(request.args.get("cursor") or "").strip() or None,
+            )
+            return jsonify(result)
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+        except Exception as exc:
+            return jsonify({"error": f"report database unavailable: {exc}"}), 503
+
+    @app.get("/api/report-artifacts/<int:artifact_id>")
+    def get_report_artifact(artifact_id: int):
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            artifact = repo.get_report_artifact(get_connection(), artifact_id)
+        except Exception as exc:
+            return jsonify({"error": f"report database unavailable: {exc}"}), 503
+        if artifact is None:
+            abort(404, "report artifact not found")
+        return jsonify(artifact)
+
+    @app.get("/api/report-artifacts/<int:artifact_id>/export")
+    def export_report_artifact(artifact_id: int):
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            artifact = repo.get_report_artifact(get_connection(), artifact_id)
+        except Exception as exc:
+            return jsonify({"error": f"report database unavailable: {exc}"}), 503
+        if artifact is None:
+            abort(404, "report artifact not found")
+
+        payload = artifact["payload"]
+        export_format = (request.args.get("format") or "json").lower()
+        filename = secure_filename(os.path.basename(
+            artifact["original_path"])) or f"report-{artifact_id}"
+        if export_format == "json":
+            content = json.dumps(payload, ensure_ascii=False, indent=2)
+            mimetype = "application/json"
+            extension = ".json"
+        elif export_format == "txt" and payload.get("format") == "text":
+            content = payload.get("content", "")
+            mimetype = "text/plain; charset=utf-8"
+            extension = ".txt"
+        elif export_format == "csv" and payload.get("format") == "csv":
+            output = io.StringIO(newline="")
+            writer = csv.DictWriter(
+                output, fieldnames=payload.get("columns", []),
+                extrasaction="ignore")
+            writer.writeheader()
+            writer.writerows(payload.get("rows", []))
+            content = output.getvalue()
+            mimetype = "text/csv; charset=utf-8"
+            extension = ".csv"
+        else:
+            return jsonify({"error": "unsupported export format"}), 406
+
+        download_name = (filename if filename.lower().endswith(extension)
+                         else f"{filename}{extension}")
+        response = Response(content, mimetype=mimetype)
+        response.headers["Content-Disposition"] = (
+            f'attachment; filename="{download_name}"')
+        return response
+
     @app.get("/api/reports/<source>")
     def list_reports(source):
         base = report_sources.get(source)
@@ -219,6 +347,19 @@ def register_routes(app: Flask,
             "snapshot_dir": MIKROTIK_DATA_DIR,
         })
 
+    @app.get("/api/firewall/snapshots")
+    def firewall_snapshots():
+        """Recent firewall snapshots persisted in the SQLite store (F001)."""
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            conn = get_connection()
+            limit = request.args.get("limit", 10, type=int)
+            return jsonify({"snapshots": repo.recent_firewall_snapshots(
+                conn, limit=limit)})
+        except Exception as exc:
+            return jsonify({"error": str(exc), "snapshots": []}), 503
+
     @app.post("/api/firewall/sync")
     def firewall_sync():
         return _sync_firewall(dry_run=True)

app/web/web_scan.py

@@ -258,23 +258,71 @@ def latest_file_scan() -> dict[str, Any] | None:
     }
 
 
+def _db_scans(limit: int = 10) -> list[dict[str, Any]]:
+    """Recent scan records from the SQLite ``scans`` table (S001).
+
+    Converts DB rows into the web job-dict shape so the dashboard can surface
+    scans that were persisted by ``run_scan`` / analyzers even if they were
+    never run through the in-memory job store (e.g. CLI scans or restarts).
+    Returns [] when the DB is unavailable.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        conn = get_connection()
+        rows = repo.latest_scans(conn, limit=limit)
+    except Exception:
+        return []
+    out = []
+    for r in rows:
+        try:
+            result_json = r.get("result_json")
+            result = json.loads(result_json) if result_json else {}
+        except (TypeError, ValueError):
+            result = {}
+        out.append({
+            "job_id": f"db-scan-{r['id']}",
+            "kind": r.get("kind") or "consolidated",
+            "state": r.get("status") or "done",
+            "started": r.get("started_at"),
+            "finished": r.get("finished_at"),
+            "error": r.get("error"),
+            "busy": False,
+            "details": {},
+            "result": result or None,
+        })
+    return out
+
+
 def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
     """Return the most recently started jobs, newest first.
 
-    Appends a file-derived "latest scan" entry when the newest report files on
+    Merges three sources, newest-first:
-    disk reflect a scan newer than any known job (e.g. after a restart or a
+      1. the in-memory job store (currently running / last-run jobs),
-    CLI run), so the dashboard always shows the latest completed scan.
+      2. the SQLite ``scans`` table (persisted by run_scan/analyzers — S001),
+      3. a file-derived "latest scan" fallback for pre-DB runs.
+    Any DB-backed scan not already represented in memory is appended so the
+    dashboard shows scans from CLI runs and across restarts.
     """
     load_persisted()
     with _jobs_lock:
         jobs = sorted(_jobs.values(),
                       key=lambda j: j.started or "", reverse=True)
     entries = [j.to_dict() for j in jobs]
+
+    # Add DB scan records that aren't already in the in-memory store. Key on
+    # (kind, started_at) to avoid duplicating a job that run_scan also logged.
+    known = {(e.get("kind"), e.get("started")) for e in entries}
+    for db_row in _db_scans(limit=limit * 2):
+        if (db_row.get("kind"), db_row.get("started")) not in known:
+            entries.append(db_row)
+
     file_entry = latest_file_scan()
     if file_entry is not None:
-        newest = jobs[0] if jobs else None
+        newest = (entries or [None])[0]
-        newest_ts = _iso_key(
+        newest_key = (
-            (newest.finished or newest.started) if newest else None)
+            (newest.get("finished") or newest.get("started")) if newest else None)
+        newest_ts = _iso_key(newest_key)
         # Only surface the file-derived entry when it is genuinely newer than
         # every known job (tolerance avoids duplicating the run that produced
         # it, where collected_at lands a moment after the job finished).

deploy/README.md

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Deployment support for the intrusion-detection IDS.
+#
+# This directory is intentionally small and uses no external framework:
+#   deploy.sh                  conflict-free `git pull` update wrapper (D004)
+#   ids.service               systemd unit template (D005)
+#   restart-service.sh         supervisor-agnostic restart shim (D005)
+#   check_runtime_git.sh       proves a scan never dirties tracked runtime files (D001)
+#
+# See .github/instructions/DEPLOYMENT.md for the full plan.
+
+set -euo pipefail

deploy/check_runtime_git.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# D001 — runtime write-path guard.
+#
+# Proves the deployment invariant: after running a scan, `git status` shows
+# NO tracked file modified by the scan — every runtime write lands in a
+# gitignored location (data/, reports/, *.db). This is what makes `git pull`
+# conflict-free on a server where scans keep running.
+#
+# If this fails, a scan wrote into a git-tracked path and `git pull` on the
+# server can conflict. Investigate the printed files before deploying.
+#
+# Usage:
+#   deploy/check_runtime_git.sh [--live]
+#     --live   additionally run a real scan first (requires log sources) and
+#              then assert the tree is still clean. Without --live it only
+#              verifies that whatever runtime files already exist are untracked.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [[ "${1:-}" == "--live" ]]; then
+  echo "== Running a real scan to exercise every write path =="
+  python app.py scan --quiet --dry-run || true
+fi
+
+echo "== Checking that runtime data is gitignored / untracked =="
+# Paths a scan writes to. Any of these being tracked OR showing as (non-ignored)
+# untracked means the invariant is broken and a git pull could conflict.
+RUNTIME_PATHS=(
+  "app/data"
+  "app/reports"
+  "app/data/ids.db"
+  "app/data/ids.db-wal"
+  "app/data/ids.db-shm"
+)
+
+fail=0
+# 1) No runtime path may be TRACKED, except the single allowed read-only
+#    PaloAlto seed. `git ls-files` reflects the index (including staged
+#    changes), so unborn commits don't cause false failures.
+allowed="app/data/paloalto/scanner-ips.txt"
+tracked=$(git ls-files app/data app/reports)
+problem=$(printf '%s\n' "$tracked" | grep -xvF "$allowed" || true)
+if [[ -n "$problem" ]]; then
+  echo "FAIL: unexpected tracked files under runtime dirs:" >&2
+  echo "$problem" >&2
+  fail=1
+fi
+
+# 2) Nothing may show as untracked-but-not-ignored under the runtime dirs, i.e.
+#    a scan wrote a file git would happily add (and later conflict on).
+for p in "${RUNTIME_PATHS[@]}"; do
+  dirty=$(git status --porcelain --untracked-files=all -- "$p" 2>/dev/null || true)
+  if [[ -n "$dirty" ]]; then
+    echo "FAIL: runtime path has changes git cares about:" >&2
+    echo "$dirty" >&2
+    fail=1
+  fi
+done
+
+if [[ "$fail" -eq 0 ]]; then
+  echo "OK: no tracked runtime data; git pull will not conflict with scan writes."
+else
+  echo "BROKEN: a scan wrote into git-tracked paths — fix before deploying." >&2
+  exit 1
+fi

deploy/deploy.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#
+# D004 — conflict-free deployment update wrapper.
+#
+# Updates a deployed instance with `git pull` WITHOUT ever hitting a merge
+# conflict from files a scan wrote since the last pull. Guarantees:
+#
+#   1. Refuses to update over uncommitted tracked changes (clean-tree guard).
+#   2. Fast-forwards only (--ff-only): never creates merge commits and never
+#      silently conflicts.
+#   3. Runtime data (data/, reports/, *.db) is gitignored, so a running scan
+#      leaves no dirty tracked paths and never blocks the pull.
+#   4. Reinstalls dependencies only if requirements.txt changed.
+#   5. Applies DB migrations idempotently.
+#   6. Restarts the app under its supervisor.
+#
+# Usage:
+#   sudo deploy/deploy.sh            # update + restart (default env: production)
+#   ENV=production deploy/deploy.sh  # use deploy/ids@production.service
+#   deploy/deploy.sh --no-restart    # update deps+migrate but don't restart
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+ENV="${ENV:-production}"
+RESTART="${1:-restart}"
+ROOT="$(pwd)"
+PY="${PY:-python}"
+
+echo "== Intrusion Detection deploy (env=$ENV) =="
+
+# 1) Clean-tree guard. Runtime data is gitignored, so this only fails if a
+#    human edited tracked source without committing.
+if ! git diff --quiet HEAD || ! git diff --cached --quiet; then
+  echo "REFUSING: working tree is not clean." >&2
+  echo "Commit or stash tracked changes, then re-run." >&2
+  echo "  git status --short" >&2
+  exit 1
+fi
+
+# 2) Pre-flight: prove runtime isolation holds (fail fast before touching repo).
+if "$ROOT/deploy/check_runtime_git.sh"; then
+  echo "== runtime isolation ok =="
+else
+  echo "ABORT: runtime isolation check failed." >&2
+  exit 1
+fi
+
+# 3) Fast-forward only.
+echo "== fetching upstream =="
+git fetch origin
+echo "== fast-forward pull =="
+git pull --ff-only
+
+# 4) Dependencies (only if the requirement set changed since the last pull).
+if git diff --quiet HEAD@{1} HEAD -- requirements.txt; then
+  echo "== requirements.txt unchanged; skipping pip =="
+else
+  echo "== requirements.txt changed; installing deps =="
+  command -v "$PY" >/dev/null && "$PY" -m pip install -r requirements.txt \
+    || pip install -r requirements.txt
+fi
+
+# 5) Apply DB migrations (idempotent).
+echo "== applying DB migrations =="
+"$PY" -m app.db --migrate
+
+# 6) Restart under the supervisor.
+if [[ "$RESTART" == "--no-restart" ]]; then
+  echo "== --no-restart: update complete; restart manually =="
+else
+  echo "== restarting service =="
+  "$ROOT/deploy/restart-service.sh" "$ENV"
+fi
+
+echo "== deploy complete =="

deploy/ids.service

@@ -0,0 +1,41 @@
+# systemd unit template (D005) — intrusion-detection web UI.
+#
+# Install:
+#   - Replace /path/to/intrusion-detection and the User/Group.
+#   - Copy to /etc/systemd/system/ids.service
+#   - sudo systemctl daemon-reload && sudo systemctl enable --now ids
+#
+# The app binds to 127.0.0.1:<WEB_PORT>; it must NOT be exposed to the LAN.
+#
+# Restart policy: automatically restart on crash. `deploy/deploy.sh` restarts
+# this unit after a successful `git pull`. Scans run in a background thread, so
+# one (serialized) scan at a time; deploy waits for the worker to be idle
+# (see restart-service.sh) before restarting.
+
+[Unit]
+Description=Intrusion Detection web UI + IDS scanner
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+# Run as an unprivileged user; the app only needs read access to the log dirs
+# it consumes (COPYPARTY_LOG_PATH / NPM_LOG_PATH) and write access to its own
+# data/ + reports/ + ids.db.
+User=ids
+Group=ids
+WorkingDirectory=/path/to/intrusion-detection
+EnvironmentFile=-/etc/ids.env
+ExecStart=/path/to/intrusion-detection/.venv/bin/python /path/to/intrusion-detection/app.py
+Restart=on-failure
+RestartSec=5
+
+# Hardening
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=full
+ProtectHome=no
+ReadWritePaths=/path/to/intrusion-detection/data /path/to/intrusion-detection/reports /var/lib/ids
+
+[Install]
+WantedBy=multi-user.target

deploy/restart-service.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# D005 — supervisor-agnostic restart shim.
+#
+# Restarts the intrusion-detection service after a deploy, preferring systemd
+# (Linux), with a fallback message for other environments (Windows dev).
+#
+# It first waits for any in-flight background scan to finish so a checkout
+# change never interleaves with a live scan (belt-and-suspenders; the runtime
+# isolation guarantee already means scans only write gitignored files).
+#
+# Usage:
+#   deploy/restart-service.sh [production]
+#   ENV=production deploy/restart-service.sh
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+ENV="${1:-${ENV:-production}}"
+UNIT="ids@${ENV}.service"
+
+# Wait for an in-flight scan to finish (~0..120s). We probe the web UI status
+# endpoint if it is reachable; a failure to reach it just means no web UI is up.
+web_host="${WEB_HOST:-127.0.0.1}"
+web_port="${WEB_PORT:-54666}"
+status_url="http://${web_host}:${web_port}/api/scan/status"
+for _ in $(seq 1 60); do
+  running=$(curl -fsS --max-time 2 "$status_url" 2>/dev/null \
+    | python3 -c 'import sys,json;print(json.load(sys.stdin).get("running",False))' 2>/dev/null \
+    || echo false)
+  [[ "$running" == "false" ]] && break
+  echo "  waiting for running scan to finish..."
+  sleep 2
+done
+
+if command -v systemctl >/dev/null 2>&1; then
+  # Installed unit is <unit>.service.template; the enabled name includes env.
+  if systemctl list-unit-files "${UNIT}" >/dev/null 2>&1; then
+    echo "== systemctl restart ${UNIT} =="
+    systemctl restart "${UNIT}"
+    systemctl --no-pager status "${UNIT}" --lines=0 || true
+  elif systemctl list-unit-files "ids.service" >/dev/null 2>&1; then
+    echo "== systemctl restart ids.service =="
+    systemctl restart ids.service
+    systemctl --no-pager status ids.service --lines=0 || true
+  else
+    echo "No ids systemd unit found; restart manually." >&2
+    exit 2
+  fi
+else
+  echo "No systemd available (Windows dev?). Restart the app manually." >&2
+  echo "  python app.py   # in a terminal, or via your dev runner" >&2
+  exit 2
+fi

tests/test_deployment.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Deployment invariant tests (D001).
+
+Proves that running a scan never creates or modifies git-tracked runtime data,
+which is what makes ``git pull`` conflict-free on a deployed server.
+
+Run with:
+    python -m unittest discover -s tests -p 'test_*.py'
+or (if pytest is installed):
+    python -m pytest tests/
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import unittest
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# The paloalto scanner-ips.txt seed is deliberately tracked (read-only); every
+# other file under the runtime dirs must remain untracked.
+ALLOWED_TRACKED_RUNTIME = {os.path.join("app", "data", "paloalto",
+                                        "scanner-ips.txt")}
+
+
+def _git(*args: str) -> str:
+    return subprocess.run(["git", *args], cwd=ROOT, check=True,
+                          capture_output=True, text=True).stdout
+
+
+class RuntimeGitTest(unittest.TestCase):
+    def test_no_runtime_data_tracked(self):
+        """No runtime file (other than the read-only paloalto seed) is tracked."""
+        tracked = _git("ls-files", "app/data", "app/reports").splitlines()
+        unexpected = [f for f in tracked
+                      if f not in ALLOWED_TRACKED_RUNTIME]
+        self.assertEqual(
+            unexpected, [],
+            "git-tracked runtime files would make `git pull` conflict; "
+            "add them to .gitignore: %s" % unexpected)
+
+    def test_runtime_write_paths_ignored(self):
+        """Every runtime write path is ignored by git (won't dirty the tree)."""
+        stats = subprocess.run(
+            ["git", "check-ignore", "-v",
+             "app/data/scan_state.json", "app/data/scan_jobs.json",
+             "app/data/whois_cache.json", "app/data/ids.db",
+             "app/data/ids.db-wal", "app/reports/app/data/blocklist_ipv4.txt",
+             ], cwd=ROOT, capture_output=True, text=True).stdout
+        self.assertIn("scan_state.json", stats)
+        self.assertIn("ids.db", stats)
+        self.assertIn("blocklist_ipv4.txt", stats)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_firewall_sync.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Tests for evidence-first firewall sync behavior."""
+
+from __future__ import annotations
+
+import unittest
+from unittest.mock import patch
+
+from app.firewall import MikrotikFirewall
+
+
+class _FakeFirewall(MikrotikFirewall):
+    def __init__(self):
+        super().__init__(router_ip="0.0.0.0", username="x", password="x")
+        self.calls: list[str] = []
+
+    def get_address_list_ipv4(self, name: str) -> set[str]:
+        self.calls.append(f"v4:{name}")
+        if name == "Blocked":
+            return {"9.9.9.9/32"}
+        if name == "Init7":
+            return {"77.74.80.0/21"}
+        return set()
+
+    def get_address_list_ipv6(self, name: str) -> set[str]:
+        self.calls.append(f"v6:{name}")
+        if name == "Blocked":
+            return set()
+        if name == "Init7":
+            return {"2001:1620::/30"}
+        return set()
+
+
+class FirewallSyncEvidenceTest(unittest.TestCase):
+    def test_sync_loads_router_status_before_computing_changes(self):
+        fw = _FakeFirewall()
+        # type: ignore[assignment]
+        fw.save_state_report = lambda *args, **kwargs: "noop"
+
+        with patch("app.firewall.cumulative_blocklist",
+                   return_value=({"8.8.8.8/32"}, set())):
+            changes = fw.sync_blocklist(["8.8.8.8/32"], [], dry_run=True)
+
+        self.assertIn("v4:Blocked", fw.calls)
+        self.assertIn("v6:Blocked", fw.calls)
+        self.assertIn("v4:Init7", fw.calls)
+        self.assertIn("v6:Init7", fw.calls)
+        self.assertEqual(changes["added_ipv4"], ["8.8.8.8/32"])
+        self.assertEqual(changes["removed_ipv4"], ["9.9.9.9/32"])
+
+    def test_sync_evaluates_against_loaded_current_status(self):
+        fw = _FakeFirewall()
+        # type: ignore[assignment]
+        fw.save_state_report = lambda *args, **kwargs: "noop"
+
+        with patch("app.firewall.cumulative_blocklist",
+                   return_value=({"9.9.9.9/32"}, set())):
+            changes = fw.sync_blocklist(["9.9.9.9/32"], [], dry_run=True)
+
+        self.assertEqual(changes["added_ipv4"], [])
+        self.assertEqual(changes["removed_ipv4"], [])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_report_backfill.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Tests for legacy report artifact backfill."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from app.db import connect
+from app.db.backfill import backfill_report_artifacts
+from app.db import repositories as repo
+
+
+class ReportBackfillTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp_dir.name)
+        self.conn = connect(str(self.root / "test.db"))
+        self.roots = {
+            "copyparty": str(self.root / "copyparty"),
+            "npm": str(self.root / "npm"),
+            "app": str(self.root / "app"),
+            "mikrotik": str(self.root / "mikrotik"),
+        }
+        for path in self.roots.values():
+            Path(path).mkdir(parents=True)
+
+    def tearDown(self):
+        self.conn.close()
+        self.temp_dir.cleanup()
+
+    def test_dry_run_and_import_are_idempotent(self):
+        Path(self.roots["copyparty"], "2026-11-06-11-30-10.txt").write_text(
+            "copyparty summary\n", encoding="utf-8")
+        Path(self.roots["npm"], "2026-11-06.csv").write_text(
+            "ip,count\n192.0.2.1,3\n", encoding="utf-8")
+        Path(self.roots["app"], "2026-11-06.json").write_text(
+            '{"summary":{"blocked":1}}', encoding="utf-8")
+        Path(self.roots["app"], "data").mkdir()
+        Path(self.roots["app"], "data", "requests.csv").write_text(
+            "ip\n192.0.2.2\n", encoding="utf-8")
+
+        preview = backfill_report_artifacts(
+            self.conn, dry_run=True, roots=self.roots)["reports"]
+        self.assertEqual(preview["would_import"], 3)
+        self.assertEqual(preview["skipped_canonical"], [
+            {"source": "app", "path": "data/requests.csv"}])
+        self.assertEqual(preview["by_source"]["copyparty"]["would_import"], 1)
+        self.assertEqual(preview["by_format"]["csv"]["would_import"], 1)
+        self.assertEqual(repo.list_report_artifacts(self.conn)["items"], [])
+
+        imported = backfill_report_artifacts(
+            self.conn, roots=self.roots)["reports"]
+        self.assertEqual(imported["imported"], 3)
+        self.assertEqual(imported["unmapped"], [])
+
+        repeated = backfill_report_artifacts(
+            self.conn, roots=self.roots)["reports"]
+        self.assertEqual(repeated["imported"], 0)
+        self.assertEqual(repeated["already_imported"], 3)
+
+        items = repo.list_report_artifacts(
+            self.conn, source="copyparty")["items"]
+        self.assertEqual(items[0]["original_path"],
+                         "copyparty/2026-11-06-11-30-10.txt")
+        self.assertEqual(items[0]["generated_at"],
+                         "2026-11-06T11:30:10+00:00")
+
+    def test_malformed_json_is_preserved_with_parse_error(self):
+        Path(self.roots["app"], "2026-11-06.json").write_text(
+            '{"incomplete":', encoding="utf-8")
+
+        result = backfill_report_artifacts(
+            self.conn, roots=self.roots)["reports"]
+        self.assertEqual(result["imported"], 1)
+        artifact = repo.list_report_artifacts(
+            self.conn, source="app")["items"][0]
+        detail = repo.get_report_artifact(self.conn, artifact["id"])
+        self.assertEqual(detail["payload"]["content"], '{"incomplete":')
+        self.assertIn("parse_error", detail["payload"])
+
+    def test_artifacts_survive_sqlite_backup_and_restore(self):
+        artifact_id = repo.insert_report_artifact(
+            self.conn, source="app", report_type="summary",
+            generated_at="2026-11-06T00:00:00+00:00",
+            payload={"summary": {"count": 4}},
+            original_path="app/2026-11-06.json", content_sha256="c" * 64,
+        )
+        restored = connect(str(self.root / "restored.db"))
+        try:
+            self.conn.backup(restored)
+            artifact = repo.get_report_artifact(restored, artifact_id)
+            self.assertEqual(artifact["payload"], {"summary": {"count": 4}})
+        finally:
+            restored.close()
+
+    def test_oversized_file_is_reported_without_deleting_source(self):
+        source = Path(self.roots["npm"], "large.txt")
+        source.write_text("keep", encoding="utf-8")
+        from unittest.mock import patch
+
+        with patch("app.db.backfill.MAX_REPORT_ARTIFACT_BYTES", 1):
+            result = backfill_report_artifacts(
+                self.conn, roots=self.roots)["reports"]
+
+        self.assertEqual(len(result["unmapped"]), 1)
+        self.assertTrue(source.exists())
+        self.assertEqual(result["unmapped"][0]["path"], "large.txt")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_report_persistence.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Tests for dual-writing generated reports to SQLite."""
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import app.scan
+from app.analyzer import Analyzer
+from app.db import connect
+from app.db import repositories as repo
+from app.utils.blocklist_history import write_blocklist_report
+
+
+class ReportPersistenceTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp_dir.name)
+        self.conn = connect(str(self.root / "test.db"))
+        self.connection_patch = patch("app.db.get_connection",
+                                      return_value=self.conn)
+        self.connection_patch.start()
+
+    def tearDown(self):
+        self.connection_patch.stop()
+        self.conn.close()
+        self.temp_dir.cleanup()
+
+    def test_analyzer_summary_is_stored_with_file_provenance(self):
+        report_dir = self.root / "copyparty"
+        report_dir.mkdir()
+        path = report_dir / "2026-11-06-11-30-10.txt"
+        path.write_text("summary body", encoding="utf-8")
+        analyzer = Analyzer("logs", str(report_dir), quiet=True)
+
+        analyzer.persist_report_artifact(str(path), scan_id=14)
+
+        item = repo.list_report_artifacts(
+            self.conn, source="ids")["items"][0]
+        detail = repo.get_report_artifact(self.conn, item["id"])
+        self.assertEqual(item["original_path"], "ids/2026-11-06-11-30-10.txt")
+        self.assertEqual(detail["scan_id"], 14)
+        self.assertEqual(detail["payload"]["content"], "summary body")
+
+    def test_blocklist_report_is_stored_as_structured_artifact(self):
+        path = write_blocklist_report(
+            str(self.root / "reports"), history_v4=set(), history_v6=set(),
+            prev_v4=set(), prev_v6=set(),
+            complete_v4={"192.0.2.1/32"}, complete_v6=set(),
+            firewall_changes={"added_ipv4": ["192.0.2.1/32"]},
+            dry_run=True, scan_id=22,
+        )
+
+        items = repo.list_report_artifacts(
+            self.conn, source="app", report_type="blocklist_change")["items"]
+        self.assertEqual(len(items), 1)
+        detail = repo.get_report_artifact(self.conn, items[0]["id"])
+        self.assertEqual(detail["scan_id"], 22)
+        self.assertEqual(detail["payload"]["blocked_ipv4"], ["192.0.2.1/32"])
+        self.assertTrue(Path(path).exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_report_routes.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Tests for database-backed report API routes."""
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from app.db import connect
+from app.db import repositories as repo
+from app.web import create_app
+
+
+class ReportRoutesTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.conn = connect(str(Path(self.temp_dir.name) / "test.db"))
+        self.app = create_app()
+        self.app.config["TESTING"] = True
+        self.client = self.app.test_client()
+        self.connection_patch = patch("app.db.get_connection",
+                                      return_value=self.conn)
+        self.connection_patch.start()
+        self.artifact_id = repo.insert_report_artifact(
+            self.conn, source="copyparty", report_type="text_report",
+            generated_at="2026-11-06T10:00:00+00:00",
+            payload={"format": "text", "content": "summary"},
+            original_path="copyparty/2026-11-06.txt",
+            content_sha256="b" * 64,
+        )
+
+    def tearDown(self):
+        self.connection_patch.stop()
+        self.conn.close()
+        self.temp_dir.cleanup()
+
+    def test_list_detail_and_text_export(self):
+        listed = self.client.get("/api/reports?source=copyparty&limit=1")
+        self.assertEqual(listed.status_code, 200)
+        self.assertEqual(listed.json["items"][0]["id"], self.artifact_id)
+
+        detail = self.client.get(
+            f"/api/report-artifacts/{self.artifact_id}")
+        self.assertEqual(detail.status_code, 200)
+        self.assertEqual(detail.json["payload"]["content"], "summary")
+
+        exported = self.client.get(
+            f"/api/report-artifacts/{self.artifact_id}/export?format=txt")
+        self.assertEqual(exported.status_code, 200)
+        self.assertEqual(exported.data.decode(), "summary")
+        self.assertIn("attachment", exported.headers["Content-Disposition"])
+        self.assertNotIn(".txt.txt", exported.headers["Content-Disposition"])
+
+    def test_invalid_source_cursor_and_missing_id(self):
+        self.assertEqual(
+            self.client.get("/api/reports?source=unknown").status_code, 400)
+        self.assertEqual(
+            self.client.get("/api/reports?cursor=invalid").status_code, 400)
+        self.assertEqual(
+            self.client.get("/api/report-artifacts/9999").status_code, 404)
+
+    def test_legacy_file_routes_remain_registered(self):
+        response = self.client.get("/api/reports/copyparty")
+        self.assertEqual(response.status_code, 200)
+        self.assertIn("files", response.json)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_repositories.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+"""Tests for the SQLite repositories + schema migrations (T001).
+
+Run with:
+    python -m unittest discover -s tests -p 'test_*.py'
+or (if pytest is installed):
+    python -m pytest tests/
+"""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+import tempfile
+import unittest
+from datetime import datetime, timedelta, timezone
+
+from app.db import connect
+from app.db import repositories as repo
+from app.db.schema import SCHEMA_VERSION, apply_migrations
+
+
+def _new_conn() -> sqlite3.Connection:
+    fd, path = tempfile.mkstemp(suffix=".db")
+    os.close(fd)
+    os.unlink(path)
+    return connect(path)
+
+
+class SchemaTest(unittest.TestCase):
+    def test_migrations_apply_and_are_idempotent(self):
+        conn = _new_conn()
+        self.assertEqual(conn.execute("PRAGMA user_version").fetchone()[0],
+                         SCHEMA_VERSION)
+        # Re-applying should be a no-op (no exception, version unchanged).
+        apply_migrations(conn)
+        self.assertEqual(conn.execute("PRAGMA user_version").fetchone()[0],
+                         SCHEMA_VERSION)
+        conn.close()
+
+    def test_core_tables_exist(self):
+        conn = _new_conn()
+        tables = {r["name"] for r in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'")}
+        for t in ("requests", "ip_stats", "scans", "scans_sources",
+                  "blocklist_entries", "blocklist_events",
+                  "blocklist_reasons", "whitelist_entries",
+                  "whitelist_sources", "firewall_snapshots",
+                  "firewall_address_list_entries", "firewall_rule_entries",
+                  "whois_info", "log_sources", "report_artifacts"):
+            self.assertIn(t, tables, f"missing table {t}")
+        conn.close()
+
+
+class ScanAndRequestTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_scan_lifecycle(self):
+        sid = repo.start_scan(self.conn, "ids", params={"full": True})
+        self.assertEqual(repo.get_scan(self.conn, sid)["status"], "running")
+        repo.finish_scan(self.conn, sid, status="done",
+                         result={"ok": True}, request_count=3)
+        s = repo.get_scan(self.conn, sid)
+        self.assertEqual(s["status"], "done")
+        self.assertEqual(s["request_count"], 3)
+        self.assertEqual([x["id"] for x in repo.latest_scans(self.conn)],
+                         [sid])
+
+    def test_insert_and_query_requests(self):
+        sid = repo.start_scan(self.conn, "ids")
+        rows = [
+            {"source": "npm", "scan_id": sid, "log_file": "a.log",
+             "ts": "2026-09-01 10:00:00", "ip": "1.2.3.4", "version": 4,
+             "method": "GET", "path": "/.env", "status": 404,
+             "host": "h", "user": None, "ua": "curl", "referer": None,
+             "denied": False, "banned": False, "is_error": True, "extra": None},
+            {"source": "copyparty", "scan_id": sid, "log_file": "f.txt",
+             "ts": "2026-09-01 10:00:01", "ip": "1.2.3.4", "version": 4,
+             "method": "GET", "path": "/admin", "status": 403,
+             "host": None, "user": None, "ua": None, "referer": None,
+             "denied": True, "banned": False, "is_error": True, "extra": None},
+        ]
+        self.assertEqual(repo.insert_requests(self.conn, rows), 2)
+        self.assertEqual(repo.count_requests(self.conn), 2)
+        # cross-source
+        cs = repo.correlate_across_sources(self.conn)
+        self.assertEqual(cs[0]["ip"], "1.2.3.4")
+        self.assertEqual(cs[0]["sources"], 2)
+        # timeline
+        tl = repo.ip_timeline(self.conn, "1.2.3.4")
+        self.assertEqual(len(tl), 2)
+        # top ips
+        top = repo.top_ips(self.conn)
+        self.assertEqual(top[0]["total"], 2)
+        repo.finish_scan(self.conn, sid, status="done")
+
+
+class BlocklistTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_grow_only_record(self):
+        r1 = repo.record_blocklist(self.conn, ["5.5.5.5/32", "6.6.6.6/32"],
+                                   added_by="scan:1")
+        self.assertEqual(r1["active"], 2)
+        # a later scan only sees 5.5.5.5; 6.6.6.6 stays (cumulative).
+        r2 = repo.record_blocklist(
+            self.conn, ["5.5.5.5/32"], added_by="scan:2")
+        active = [e["cidr"] for e in repo.active_blocklist(self.conn)]
+        self.assertIn("6.6.6.6/32", active)
+        self.assertEqual(r2["active"], 2)
+
+    def test_unblock(self):
+        repo.record_blocklist(self.conn, ["7.7.7.7/32"])
+        self.assertTrue(repo.unblock_entry(self.conn, "7.7.7.7/32"))
+        self.assertNotIn("7.7.7.7/32",
+                         [e["cidr"] for e in repo.active_blocklist(self.conn)])
+        self.assertFalse(repo.unblock_entry(self.conn, "7.7.7.7/32"))
+
+    def test_prune_old_single_ip_blocks_keeps_ranges(self):
+        old = (datetime.now(timezone.utc) - timedelta(days=31)).isoformat(
+            timespec="seconds")
+        with repo.transaction(self.conn):
+            repo._block_entry_inner(self.conn, "1.2.3.4/32", last_seen=old)
+            repo._block_entry_inner(
+                self.conn, "2001:db8::1/128", last_seen=old)
+            repo._block_entry_inner(self.conn, "1.2.3.0/24", last_seen=old)
+            repo._block_entry_inner(self.conn, "2001:db8::/64", last_seen=old)
+
+        removed = repo.prune_old_single_ip_blocks(
+            self.conn, older_than_days=30)
+        self.assertCountEqual(removed, ["1.2.3.4/32", "2001:db8::1/128"])
+
+        active = {e["cidr"] for e in repo.active_blocklist(self.conn)}
+        self.assertIn("1.2.3.0/24", active)
+        self.assertIn("2001:db8::/64", active)
+        self.assertNotIn("1.2.3.4/32", active)
+        self.assertNotIn("2001:db8::1/128", active)
+
+    def test_prune_old_single_ip_blocks_respects_recency(self):
+        fresh = (datetime.now(timezone.utc) - timedelta(days=5)).isoformat(
+            timespec="seconds")
+        with repo.transaction(self.conn):
+            repo._block_entry_inner(self.conn, "9.9.9.9/32", last_seen=fresh)
+
+        removed = repo.prune_old_single_ip_blocks(
+            self.conn, older_than_days=30)
+        self.assertEqual(removed, [])
+        self.assertIn("9.9.9.9/32",
+                      {e["cidr"] for e in repo.active_blocklist(self.conn)})
+
+
+class WhitelistTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_whitelist_lookup(self):
+        repo.set_whitelist(self.conn, "Cloudflare", ["173.245.48.0/20"],
+                           source="test")
+        self.assertTrue(repo.is_whitelisted_db(self.conn, "173.245.48.5"))
+        self.assertFalse(repo.is_whitelisted_db(self.conn, "8.8.8.8"))
+        # provider replace deactivates old ranges
+        repo.set_whitelist(self.conn, "Cloudflare", ["104.16.0.0/13"])
+        self.assertFalse(repo.is_whitelisted_db(self.conn, "173.245.48.5"))
+
+
+class FirewallTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_firewall_snapshot(self):
+        sid = repo.save_firewall_snapshot(
+            self.conn,
+            address_lists_v4=[{"list": "Blocked", "address": "5.5.5.5/32"}],
+            address_lists_v6=[{"list": "Init7", "address": "2001:1620::/30"}],
+            rules_v4=[{"chain": "input", "action": "drop", "comment": "x"}],
+            rules_v6=[])
+        self.assertEqual(sid, 1)
+        self.assertEqual(self.conn.execute(
+            "SELECT COUNT(*) FROM firewall_address_list_entries").fetchone()[0], 2)
+        self.assertEqual(self.conn.execute(
+            "SELECT COUNT(*) FROM firewall_rule_entries").fetchone()[0], 1)
+
+
+class ReportArtifactTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_artifact_insert_is_idempotent_and_detail_decodes_payload(self):
+        kwargs = {
+            "source": "copyparty", "report_type": "analyzer_summary",
+            "generated_at": "2026-11-01T10:00:00+00:00",
+            "payload": {"summary": {"requests": 12}},
+            "original_path": "reports/copyparty/2026-11-01.txt",
+            "content_sha256": "a" * 64,
+        }
+        first_id = repo.insert_report_artifact(self.conn, **kwargs)
+        duplicate_id = repo.insert_report_artifact(self.conn, **kwargs)
+
+        self.assertEqual(first_id, duplicate_id)
+        self.assertEqual(self.conn.execute(
+            "SELECT COUNT(*) FROM report_artifacts").fetchone()[0], 1)
+        self.assertEqual(repo.get_report_artifact(self.conn, first_id)["payload"],
+                         kwargs["payload"])
+
+    def test_artifact_cursor_orders_stably_and_filters_source(self):
+        for day, source in (("01", "npm"), ("02", "copyparty"),
+                            ("03", "npm")):
+            repo.insert_report_artifact(
+                self.conn, source=source, report_type="summary",
+                generated_at=f"2026-11-{day}T00:00:00+00:00",
+                payload={"day": day}, original_path=f"reports/{day}.txt",
+                content_sha256=day * 64,
+            )
+
+        first = repo.list_report_artifacts(self.conn, source="npm", limit=1)
+        self.assertEqual(len(first["items"]), 1)
+        self.assertIsNotNone(first["next_cursor"])
+        second = repo.list_report_artifacts(
+            self.conn, source="npm", limit=1, cursor=first["next_cursor"])
+        self.assertEqual(len(second["items"]), 1)
+        self.assertIsNone(second["next_cursor"])
+        self.assertGreater(first["items"][0]["generated_at"],
+                           second["items"][0]["generated_at"])
+
+    def test_artifact_cursor_rejects_invalid_token(self):
+        with self.assertRaises(ValueError):
+            repo.list_report_artifacts(self.conn, cursor="invalid")
+
+
+if __name__ == "__main__":
+    unittest.main()