feat(reports): store and browse report artifacts

B001 change caused 'unsupported operand type(s) for -: list and set' during a
full consolidated scan: _db_complete_blocklist returned split_cidr_versions()
output, which is tuple[list, list], but write_blocklist_report()/blocklist_delta()
expect set operands. Return set(v4), set(v6) to match cumulative_blocklist's
contract. Reproduced the exact report path with a populated ledger and
verified it no longer crashes.

.editorconfig

@@ -0,0 +1,63 @@
+# https://editorconfig.org
+root = true
+
+# -------------------------------------------------
+# Common settings – apply to every file
+# -------------------------------------------------
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+# -------------------------------------------------
+# Python (*.py)
+# -------------------------------------------------
+[*.py]
+indent_style = space
+indent_size = 4
+max_line_length = 88          # matches Black formatter default line length
+# Optional: enforce a newline at the end of the file (already set globally)
+
+# -------------------------------------------------
+# JavaScript & TypeScript (*.js, *.jsx, *.ts, *.tsx)
+# -------------------------------------------------
+[*.{js,jsx,ts,tsx}]
+indent_style = space
+indent_size = 2
+max_line_length = 120
+trim_trailing_whitespace = true
+
+# -------------------------------------------------
+# JSON (*.json, *.jsonc)
+# -------------------------------------------------
+[*.json]
+indent_style = space
+indent_size = 2
+max_line_length = off         # JSON is often compact; do not enforce length
+trim_trailing_whitespace = false   # JSON parsers ignore trailing spaces
+
+# -------------------------------------------------
+# HTML (*.html, *.htm)
+# -------------------------------------------------
+[*.{html,htm}]
+indent_style = space
+indent_size = 2
+max_line_length = 120
+trim_trailing_whitespace = true
+
+# -------------------------------------------------
+# CSS & SCSS (*.css, *.scss)
+# -------------------------------------------------
+[*.{css,scss}]
+indent_style = space
+indent_size = 2
+max_line_length = 120
+trim_trailing_whitespace = true
+
+# -------------------------------------------------
+# Markdown (*.md, *.markdown)
+# -------------------------------------------------
+[*.{md,markdown}]
+max_line_length = off        # Markdown lines can be long for readability
+trim_trailing_whitespace = false   # Preserve intentional spaces (e.g., line breaks)

.env.example

@@ -1,76 +1,90 @@
-# ---------------------------------------------------------------------------
-# Intrusion Detection System — environment template
-# ---------------------------------------------------------------------------
-# Copy this file to `.env` for local development and fill in your values:
-#
-#     copy .env.example .env
-#
-# Precedence: real system environment variables win over values in `.env`,
-# so production deployments can set these via the OS and skip `.env`.
-# `.env` is git-ignored — never commit real secrets to it.
-
-# --- Web server (localhost only) -------------------------------------------------
-
-# The Flask web UI binds to localhost so it is only reachable from this machine. 
-# Do not expose WEB_HOST beyond 127.0.0.1!
-WEB_HOST=127.0.0.1
-# Please choose a very high port number (e.g. 54666) to avoid conflicts with other services.
-WEB_PORT=54666
-
-# --- Log sources -----------------------------------------------------------
-
-# Directory containing copyparty access logs (YYYY-MM-DD.txt files).
-COPYPARTY_LOG_PATH=P:\copyparty
-
-# Directory containing Nginx Proxy Manager proxy logs (.log / .gz files).
-NPM_LOG_PATH=Y:\npm\logs
-
-# --- MikroTik RouterOS connection -----------------------------------------
-
-# IP/hostname of the router to sync the blocklist to.
-ROUTER_IP=192.168.88.1
-
-# SSH user for the router.
-ROUTER_USER=admin
-
-# SSH password for the router (required when --update-firewall is used).
-ROUTER_PASSWORD=change-me
-
-# Address list on the router that holds blocked IPs/subnets.
-BLOCKLIST_NAME=Blocked
-
-# Comma-separated address lists that are treated as whitelisted (never blocked).
-WHITELIST_NAMES=Cloudflare,Openfactory
-
-# --- WHOIS subnet consolidation --------------------------------------------
-
-WHOIS_SERVER=whois.radb.net
-WHOIS_PORT=43
-WHOIS_TIMEOUT=3.0
-
-# Minimum number of same-subnet IPs before blocking the whole subnet.
-SUBNET_THRESHOLD=3
-
-# --- Scan / flagging defaults ----------------------------------------------
-
-# Rows in the top-IP report tables.
-DEFAULT_TOP=30
-
-# copyparty: minimum denied hits before flagging an IP.
-DEFAULT_MIN_DENIED=1
-
-# npm: minimum error responses before flagging an IP.
-DEFAULT_MIN_ERRORS=5
-
-# npm: minimum error ratio before flagging an IP.
-DEFAULT_ERROR_RATIO=0.5
-
-# Rows in the "top error paths" report section.
-DEFAULT_TOP_PATHS=20
-
-# --- Whitelist sources -----------------------------------------------------
-
-OPENFACTORY_URL=https://openfactory.net/geofeed.csv
-CLOUDFLARE_URL_V4=https://www.cloudflare.com/ips-v4
-CLOUDFLARE_URL_V6=https://www.cloudflare.com/ips-v6
+# ---------------------------------------------------------------------------
+# Intrusion Detection System — environment template
+# ---------------------------------------------------------------------------
+# Copy this file to `.env` for local development and fill in your values:
+#
+#     copy .env.example .env
+#
+# Precedence: real system environment variables win over values in `.env`,
+# so production deployments can set these via the OS and skip `.env`.
+# `.env` is git-ignored — never commit real secrets to it.
+
+# --- Web server (localhost only) -------------------------------------------------
+
+# The Flask web UI binds to localhost so it is only reachable from this machine. 
+# Do not expose WEB_HOST beyond 127.0.0.1!
+WEB_HOST=127.0.0.1
+# Please choose a very high port number (e.g. 54666) to avoid conflicts with other services.
+WEB_PORT=54666
+
+# --- Log sources -----------------------------------------------------------
+
+# Directory containing copyparty access logs (YYYY-MM-DD.txt files).
+COPYPARTY_LOG_PATH=P:\copyparty
+
+# Directory containing Nginx Proxy Manager proxy logs (.log / .gz files).
+NPM_LOG_PATH=Y:\npm\logs
+
+# --- MikroTik RouterOS connection -----------------------------------------
+
+# IP/hostname of the router to sync the blocklist to.
+ROUTER_IP=192.168.88.1
+
+# SSH user for the router.
+ROUTER_USER=admin
+
+# SSH password for the router (required when --update-firewall is used).
+ROUTER_PASSWORD=change-me
+
+# Address list on the router that holds blocked IPs/subnets.
+BLOCKLIST_NAME=Blocked
+
+# Comma-separated address lists that are treated as whitelisted (never blocked).
+WHITELIST_NAMES=Cloudflare,Init7
+
+# --- WHOIS subnet consolidation --------------------------------------------
+
+WHOIS_SERVER=whois.radb.net
+WHOIS_PORT=43
+WHOIS_TIMEOUT=3.0
+
+# Minimum number of same-subnet IPs before blocking the whole subnet.
+SUBNET_THRESHOLD=3
+
+# --- Report retention ------------------------------------------------------
+
+# Each subdirectory under /reports/ keeps at most REPORT_RETENTION_MAX_FILES
+# files, all newer than REPORT_RETENTION_DAYS days. Older timestamped reports
+# are pruned automatically after each consolidated scan. The blocklist files
+# in reports/app/data/ are always protected.
+REPORT_RETENTION_DAYS=30
+REPORT_RETENTION_MAX_FILES=10
+
+# --- SQLite store ---------------------------------------------------------
+
+# Local SQLite database canonical store (logs, scans, blocklist/whitelist and
+# firewall history). Created automatically on first run. Default: data/ids.db
+#IDS_DB_PATH=/absolute/path/to/ids.db
+
+# --- Scan / flagging defaults ----------------------------------------------
+
+# Rows in the top-IP report tables.
+DEFAULT_TOP=30
+
+# copyparty: minimum denied hits before flagging an IP.
+DEFAULT_MIN_DENIED=1
+
+# npm: minimum error responses before flagging an IP.
+DEFAULT_MIN_ERRORS=5
+
+# npm: minimum error ratio before flagging an IP.
+DEFAULT_ERROR_RATIO=0.5
+
+# Rows in the "top error paths" report section.
+DEFAULT_TOP_PATHS=20
+
+# --- Whitelist sources -----------------------------------------------------
+
+CLOUDFLARE_URL_V4=https://www.cloudflare.com/ips-v4
+CLOUDFLARE_URL_V6=https://www.cloudflare.com/ips-v6
 PALOALTO_URL=https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity

.gitignore

@@ -1,16 +1,31 @@
 # instructions for agents
 .github/instructions/
+.agents
+skills-lock.json
 
 # environment variables
 .env
 
 # python cache
 __pycache__/
+.mypy_cache/
 
-# incremental-scan state (regenerated each run)
-data/scan_state.json
+# python virtual environment
+.venv/
 
-# reports data
-reports/copyparty/
-reports/mikrotik/
-reports/npm/
+# app runtime data (feeds, caches, scan state - regenerated each run).
+# All of this must stay UNTRACKED so ``git pull`` never conflicts with files a
+# scan has written since the last pull (see .github/instructions/DEPLOYMENT.md).
+# The single allowed tracked seed is the read-only PaloAlto scanner-ips.txt
+# (never rewritten at runtime); the Cloudflare ips-*.txt are runtime-refreshed
+# downloads, so they are NOT tracked and are re-downloaded if absent.
+app/data/*
+!app/data/paloalto/scanner-ips.txt
+
+# SQLite database (canonical store) + WAL/SHM sidecars
+app/data/*.db
+app/data/*.db-wal
+app/data/*.db-shm
+
+# generated reports
+app/reports/

README.md

@@ -5,21 +5,16 @@ Local-development intrusion detection / log analysis for two log sources:
 | Script                | Source                                                          | Detection rule                                                             |
 | --------------------- | --------------------------------------------------------------- | -------------------------------------------------------------------------- |
 | `app.py`              | unified entry point                                             | runs both analyzers + optional firewall sync                               |
-| `app/scan.py`         | consolidated scanner                                            | runs both analyzers, merges blocklists, syncs firewall                     |
+| `app/scan/`           | consolidated scanner                                            | runs both analyzers, merges blocklists, syncs firewall                     |
 | `app/copyparty.py`    | [copyparty](https://github.com/9001/copyparty) server logs      | denied-only IPs + copyparty auto-bans                                      |
 | `app/npm.py`          | [Nginx Proxy Manager](https://nginxproxymanager.com) proxy logs | high error-rate IPs (scanners)                                             |
-| `app/utils.py`        | re-export layer                                                 | thin barrel — imports from sub-modules below                               |
-| `app/models.py`       | data model                                                      | `Request`, `IPStats`, `aggregate()`                                        |
-| `app/ip_utils.py`     | IP helpers                                                      | `parse_ip`, `whitelist_of`, `is_whitelisted`, `sort_ips`, `split_versions` |
-| `app/flags.py`        | detection rules                                                 | `flag_scanners`, `flag_denied_only`, `flag_malicious_patterns`, `by_total` |
+| `app/utils/`          | shared utilities                                                | constants, I/O helpers, IP utils, models, reporting, whitelists, WHOIS     |
+| `app/web/`            | Flask web UI                                                    | app factory, routes, background scan runner                                |
 | `app/analyzer.py`     | pipeline                                                        | `Analyzer` — discover → load → analyze → report → write                    |
-| `app/whitelists.py`   | whitelist module                                                | provider CIDR constants, local/remote load, CIDR extraction                |
+| `app/flags.py`        | detection rules                                                 | `flag_scanners`, `flag_denied_only`, `flag_malicious_patterns`, `by_total` |
 | `app/bad_patterns.py` | pattern detection                                               | malicious URL/file regex classification                                    |
-| `app/whois.py`        | WHOIS query                                                     | `query_whois_subnet`, `consolidate_by_subnet`                              |
 | `app/mikrotik.py`     | RouterOS export                                                 | `mikrotik_script`, `write_mikrotik`, `split_cidr_versions`                 |
 | `app/firewall.py`     | RouterOS SSH client                                             | `MikrotikFirewall` — connect, sync blocklist, audit state                  |
-| `app/files.py`        | I/O helpers                                                     | `read_lines`, `find_logs`, `write_csv`, `write_json`, etc.                 |
-| `app/reporting.py`    | text rendering                                                  | `Column`, `render_table`, `render_kv`, `timestamp_name`                    |
 
 Both per-source scripts parse logs, aggregate per-IP behaviour, split
 IPv4/IPv6, skip whitelisted provider ranges, detect malicious path patterns,
@@ -40,34 +35,45 @@ intrusion-detection/
 ├── app.py               unified entry point
 ├── app/                 application package
 │   ├── __init__.py
-│   ├── constants.py     all config constants (paths, env vars, thresholds)
-│   ├── scan.py          consolidated scanner
-│   ├── copyparty.py     copyparty log adapter (~150 lines)
-│   ├── npm.py           NPM proxy log adapter (~150 lines)
-│   ├── firewall.py      MikroTik RouterOS SSH client
-│   ├── utils.py         thin re-export barrel (~90 lines)
-│   ├── models.py        Request, IPStats, aggregate()
-│   ├── ip_utils.py      IP parsing, whitelist lookups, sorting
-│   ├── flags.py         flag_scanners, flag_denied_only, flag_malicious_patterns
 │   ├── analyzer.py      Analyzer pipeline base class
-│   ├── whitelists.py    provider CIDR constants, local/remote load
 │   ├── bad_patterns.py  malicious URL/file regex patterns
-│   ├── whois.py         query_whois_subnet, consolidate_by_subnet
+│   ├── copyparty.py     copyparty log adapter (~150 lines)
+│   ├── firewall.py      MikroTik RouterOS SSH client
+│   ├── flags.py         flag_scanners, flag_denied_only, flag_malicious_patterns
 │   ├── mikrotik.py      RouterOS .rsc script generation
-│   ├── files.py         read_lines, find_logs, write_csv/json/lines
-│   ├── reporting.py     Column, render_table, render_kv, timestamp_name
-│   ├── web.py           Flask web UI (app factory, JSON API, pages)
-│   ├── web_scan.py      background scan runner + in-memory job store
-│   ├── templates/       Jinja2 page templates (dashboard, scan, reports, …)
-│   └── static/          style.css + app.js (dark-theme UI)
+│   ├── npm.py           NPM proxy log adapter (~150 lines)
+│   ├── web/             Flask web UI
+│   │   ├── __init__.py  app factory + helpers + serve()
+│   │   ├── routes.py    all HTTP route definitions
+│   │   └── web_scan.py  background scan runner + in-memory job store
+│   ├── scan/            scan pipeline
+│   │   ├── __init__.py  consolidated scanner (run_scan, ScanParams, ScanResult)
+│   │   └── scan_state.py incremental log scan state (load/save)
+│   ├── db/              SQLite store (see .github/instructions/DATABASE.md)
+│   │   ├── __init__.py     connection factory + migrations runner
+│   │   ├── schema.py       DDL for all tables (PRAGMA user_version migrations)
+│   │   └── repositories.py  queries: logs, scans, blocklist, whitelist, firewall
+│   ├── utils/           shared utilities
+│   │   ├── constants.py     all config constants (paths, env vars, thresholds)
+│   │   ├── files.py         read_lines, find_logs, write_csv/json/lines
+│   │   ├── ip_utils.py      IP parsing, whitelist lookups, sorting
+│   │   ├── models.py        Request, IPStats, aggregate()
+│   │   ├── reporting.py     Column, render_table, render_kv, timestamp_name
+│   │   ├── utils.py         shared CLI arg parser
+│   │   ├── whitelists.py    provider CIDR constants, local/remote load
+│   │   ├── whois.py         query_whois_subnet, consolidate_by_subnet
+│   │   ├── blocklist_history.py git-backed cumulative blocklist
+│   │   └── retention.py     report file retention & pruning
+│   ├── static/          style.css + app.js (dark-theme UI)
+│   └── templates/       Jinja2 page templates (dashboard, scan, reports, …)
 ├── data/                whitelist source files (cached locally)
-│   ├── openfactory/
-│   │   └── geofeed.csv
+│   ├── init7/           (ranges built into app/utils/whitelists.py — no geofeed)
 │   ├── cloudflare/
 │   │   ├── ips-v4.txt
 │   │   └── ips-v6.txt
-│   └── paloalto/
-│       └── scanner-ips.txt
+│   ├── paloalto/
+│   │   └── scanner-ips.txt
+│   └── ids.db           local SQLite store (log/scan/blocklist/whitelist history; auto-created)
 ├── reports/             generated reports (per-run + per-source)
 │   ├── copyparty/
 │   ├── npm/
@@ -80,22 +86,24 @@ matcher, a line parser, a flagging rule, and column definitions.
 
 Core responsibilities by module:
 
-| Module                | Owns                                                                             |
-| --------------------- | -------------------------------------------------------------------------------- |
-| `app/constants.py`    | all config constants — paths, env vars, network/scan thresholds                  |
-| `app/models.py`       | `Request`, `IPStats`, `aggregate()`                                              |
-| `app/ip_utils.py`     | `parse_ip`, `whitelist_of`, `is_whitelisted`, `sort_ips`, `split_versions`       |
-| `app/flags.py`        | `flag_scanners`, `flag_denied_only`, `flag_malicious_patterns`, `by_total`       |
-| `app/analyzer.py`     | `Analyzer` — discover → load → analyze → report → write                          |
-| `app/whitelists.py`   | provider CIDR constants, local-file / remote / built-in loading                  |
-| `app/bad_patterns.py` | malicious URL/file regex classification                                          |
-| `app/whois.py`        | `query_whois_subnet`, `consolidate_by_subnet`                                    |
-| `app/mikrotik.py`     | `mikrotik_script`, `write_mikrotik`, `split_cidr_versions`                       |
-| `app/firewall.py`     | `MikrotikFirewall` — SSH connect, sync blocklist, audit state                    |
-| `app/web.py`          | Flask app factory, JSON API routes, page rendering                               |
-| `app/web_scan.py`     | background scan runner, in-memory job store, single-scan busy lock               |
-| `app/files.py`        | `read_lines`, `find_logs`, `validate_dir`, `ensure_dirs`, `write_csv/json/lines` |
-| `app/reporting.py`    | `Column`, `render_table`, `render_kv`, `timestamp_name`                          |
+| Module                           | Owns                                                                             |
+| -------------------------------- | -------------------------------------------------------------------------------- |
+| `app/utils/constants.py`         | all config constants — paths, env vars, network/scan thresholds                  |
+| `app/utils/models.py`            | `Request`, `IPStats`, `aggregate()`                                              |
+| `app/utils/ip_utils.py`          | `parse_ip`, `whitelist_of`, `is_whitelisted`, `sort_ips`, `split_versions`       |
+| `app/flags.py`                   | `flag_scanners`, `flag_denied_only`, `flag_malicious_patterns`, `by_total`       |
+| `app/analyzer.py`                | `Analyzer` — discover → load → analyze → report → write                          |
+| `app/utils/whitelists.py`        | provider CIDR constants, local-file / remote / built-in loading                  |
+| `app/bad_patterns.py`            | malicious URL/file regex classification                                          |
+| `app/utils/whois.py`             | `query_whois_subnet`, `consolidate_by_subnet`                                    |
+| `app/mikrotik.py`                | `mikrotik_script`, `write_mikrotik`, `split_cidr_versions`                       |
+| `app/firewall.py`                | `MikrotikFirewall` — SSH connect, sync blocklist, audit state                    |
+| `app/web/`                       | Flask app factory, JSON API routes, page rendering, background scan runner       |
+| `app/scan/`                      | consolidated scan pipeline, incremental scan state                               |
+| `app/utils/files.py`             | `read_lines`, `find_logs`, `validate_dir`, `ensure_dirs`, `write_csv/json/lines` |
+| `app/utils/reporting.py`         | `Column`, `render_table`, `render_kv`, `timestamp_name`                          |
+| `app/utils/blocklist_history.py` | git-backed cumulative blocklist reconstruction                                   |
+| `app/utils/retention.py`         | report file retention & pruning                                                  |
 
 Console output and saved reports are the **same string** (`Analyzer.text_report()`).
 
@@ -131,13 +139,26 @@ production can set the same variables via the OS and skip `.env` entirely.
 | `ROUTER_USER`        | `admin`                  | MikroTik connection   |
 | `ROUTER_PASSWORD`    | _(set in .env)_          | MikroTik connection   |
 | `BLOCKLIST_NAME`     | `Blocked`                | MikroTik address list |
-| `WHITELIST_NAMES`    | `Cloudflare,Openfactory` | MikroTik whitelists   |
+| `WHITELIST_NAMES`    | `Cloudflare,Init7`       | MikroTik whitelists   |
 | `WEB_HOST`           | `127.0.0.1`              | web UI bind host      |
 | `WEB_PORT`           | `54666`                  | web UI port           |
 
 See `.env.example` for the complete list of supported variables (WHOIS
 tuning, scan thresholds, whitelist source URLs, and more).
 
+### Network context (2026-09-08)
+
+Connectivity provider switched Openfactory → Init7 on 2026-09-08:
+
+- WAN IPv4: `77.74.80.159/32` (init7 `77.74.80.0/24`, "Hybrid7-NET").
+- IPv6: delegated `/48` `2001:1620:549f::/48` (init7 allocation `2001:1620::/30`).
+- Upstream DNS: `77.109.128.2`, `213.144.129.20`.
+
+The `Openfactory` whitelist (old AS58299 geofeed) is obsolete and must be
+replaced with an `Init7` whitelist — see
+[`.github/instructions/TODO.md`](.github/instructions/TODO.md) for the
+switchover plan and the verified init7 CIDR ranges.
+
 ## Usage
 
 ```powershell
@@ -151,6 +172,11 @@ python app.py scan --quiet
 python app.py scan --update-firewall --dry-run
 python app.py scan --update-firewall
 
+# detailed log-analysis report from the SQLite store (needs scans run first)
+python app.py analyze                  # top IPs + cross-source + blocklist
+python app.py analyze --ip 1.2.3.4     # drill into one IP's request history
+python app.py analyze --json           # machine-readable output
+
 # individual scanners (still runnable directly)
 python -m app.copyparty
 python -m app.npm
@@ -184,6 +210,49 @@ python -m app.copyparty --top 100
 python app.py scan --quiet --update-firewall --dry-run
 ```
 
+## Blocklist behavior (cumulative / grow-only)
+
+The MikroTik `Blocked` address list is **cumulative**: it only grows over
+time and is never trimmed automatically.
+
+- An IP (or subnet) that the system has ever blocked stays blocked — even if
+  it stops appearing in recent logs (for example after it leaves the NPM
+  ~30-day log-retention window, or once WHOIS subnet consolidation groups it
+  into a larger range).
+- The **source of truth** for "all blocked IPs" is the **SQLite blocklist
+  ledger** (`app.db` → `blocklist_entries` / `blocklist_events`), not git
+  history. Each scan syncs the desired blocklist into the ledger; the router's
+  cumulative state is derived from it. `app/utils/blocklist_history.py` is a
+  read-only fallback only.
+- The sync only ever **adds** entries. The only automatic removals are router
+  entries that were never blocked by the system — i.e. absent from the ledger
+  (manual / rogue entries).
+- To explicitly unblock an IP, remove it manually via the web UI
+  (`/blocklist` → remove) or directly on the router; the automated sync will
+  not re-add entries that are not part of a scan.
+
+> **Deployment note:** runtime data (blocklists, reports, the DB) is **never
+> committed to git**. Keeping it untracked is what makes `git pull` updates
+> conflict-free — see [`DEPLOYMENT.md`](.github/instructions/DEPLOYMENT.md)
+> and the `deploy/` scripts.
+
+Each consolidated scan with `--update-firewall` also writes a timestamped
+change report to `reports/app/YYYY-MM-DD-hh-mm-ss.json`: the number and list
+of entries added / removed (complete-set delta), the full cumulative blocked
+list, and the router sync changes. It also persists a firewall snapshot into
+the `firewall_snapshots` SQLite tables for long-term audit (F001).
+
+### Report retention
+
+Generated files under `reports/` are pruned automatically after each
+consolidated scan: every subdirectory keeps at most
+`REPORT_RETENTION_MAX_FILES` files (default 10), all newer than
+`REPORT_RETENTION_DAYS` days (default 30). The blocklist files
+(`reports/app/data/blocklist_ipv4.txt` / `blocklist_ipv6.txt`) are always
+protected and never pruned. Tune via `REPORT_RETENTION_DAYS` /
+`REPORT_RETENTION_MAX_FILES`; `--dry-run` previews the cleanup without
+deleting.
+
 ## Web UI
 
 `python app.py` starts a **localhost-only** Flask web UI (default
@@ -201,6 +270,8 @@ Pages:
 | `/blocklist` | view the consolidated blocklist, add/remove entries   |
 | `/firewall`  | view address lists + rules, preview/apply a sync      |
 | `/whitelist` | view/manage provider whitelist CIDRs                  |
+| `/graph`      | visualize the 0.0.0.0/8 address space (blocked/whitelisted pixels) |
+| `/analysis`   | detailed persisted-log analysis (top IPs, drill-down, cross-source) |
 
 Scans run in a background thread (only one at a time) and report progress via
 the JSON API, so the UI never blocks. Firewall sync is **dry-run by default**
@@ -209,6 +280,35 @@ from the UI — applying changes is an explicit action. The main JSON endpoints
 `/api/firewall/*`, `/api/whitelist`) are also usable directly from a REST
 client.
 
+## Deployment & updating (`git pull`)
+
+The app runs scans continuously (web UI + cron + CLI), and all of it is
+intended to be updated on the server with a plain `git pull`. To make that
+**safe** — i.e. never conflict with a file a scan wrote since the last pull —
+runtime data is strictly **never committed to git**:
+
+- `app/data/*`, `app/reports/*` and the SQLite DB (`app/data/ids.db`) are
+gitignored. The only tracked file under `data/` is the read-only
+`app/data/paloalto/scanner-ips.txt` seed.
+- The Cloudflare `ips-*.txt` whitelist seeds are **not** tracked: they are
+runtime-refreshed downloads and are re-fetched if absent (built-in ranges are
+the fallback).
+- History that must survive — the cumulative blocklist and firewall audit —
+lives in the SQLite store, not in git revisions.
+
+Update a deployed instance with:
+
+```sh
+sudo deploy/deploy.sh
+```
+
+See [`DEPLOYMENT.md`](.github/instructions/DEPLOYMENT.md) for the full plan,
+and `deploy/` for the scripts (`deploy.sh`, `restart-service.sh`,
+`check_runtime_git.sh`, `ids.service` systemd template). The wrapper
+fast-forwards only (`git pull --ff-only`), refuses to run on an unclean tree,
+reinstalls deps if `requirements.txt` changed, applies DB migrations
+(`python -m app.db --migrate`), and restarts the `ids` unit.
+
 ## Output
 
 Each script writes to its own report directory. `app.py` also writes
@@ -259,9 +359,11 @@ reports/
 ```
 
 The current router state (address lists + firewall rules) is also mirrored to
-`data/mikrotik/` on every `--update-firewall` run. Those files are overwritten
-each run and are **git-visible**, so `git diff` shows how the firewall rules
-and address lists change over time.
+`data/mikrotik/` on every `--update-firewall` run as a scratch pointer to the
+latest snapshot. Those files are overwritten each run and are **not** tracked;
+the long-term audit trail is the SQLite `firewall_snapshots` tables plus the
+timestamped reports under `reports/mikrotik/`. `git pull` is therefore never
+blocked by firewall-state files.
 
 ## Detection
 

REPORTS.md

@@ -0,0 +1,142 @@
+# Report Inventory
+
+This inventory records the current report surface before moving report reads
+or writes into SQLite. Generated paths are rooted at `REPORT_DIR` (normally
+`reports/`). The SQLite tables listed below already exist in schema v1 unless
+noted otherwise.
+
+## Artifacts
+
+| Area | Producer and formats | Current role and consumers | Classification |
+| --- | --- | --- | --- |
+| `copyparty/` | `Analyzer.write_report()` creates timestamped `.txt`; `write_data()` creates per-IP CSV/JSON, denied-path CSV, flagged/banned IP TXT, `requests.csv`, `authed_users.json`, and `mikrotik-block.rsc`. | Reports API serves top-level text reports. `requests.csv` is imported by `app.db.backfill`; per-run data supports inspection and legacy workflows. Retention prunes generated files. | Request rows and per-scan IP stats are canonical in `requests` and `ip_stats`; rendered summaries and exports are derived. |
+| `npm/` | Same analyzer outputs, plus user-agent and host summary CSV files. | Same consumers as copyparty. `requests.csv` is a legacy backfill input; retention prunes generated files. | Request rows and per-scan IP stats are canonical in `requests` and `ip_stats`; summaries and exports are derived. |
+| `app/data/` | Consolidated scan writes `blocklist_ipv4.txt` and `blocklist_ipv6.txt`. | Blocklist UI/API reads these files; scan history reconstruction and firewall sync use them as current/compatibility inputs. Retention explicitly protects them. | Operational compatibility exports. Canonical blocked targets and events belong in `blocklist_entries` and `blocklist_events`; keep files until every consumer uses the DB and git-history contract is replaced. |
+| `app/` timestamped JSON | `write_blocklist_report()` emits scan/firewall change reports when the router is reachable. | Human-readable audit trail containing run delta, cumulative targets, router changes, dry-run state, and errors; retained under report retention. | Derived audit artifact. Preserve structured fields and source scan association in the DB before considering file retirement. |
+| `mikrotik/` | `MikrotikFirewall.save_state_report()` writes timestamped JSON snapshots and report data; latest state is also written to `data/mikrotik/*.json`. | Firewall UI/API displays current state and snapshot history. `app.db.backfill` currently imports only the latest `data/mikrotik` snapshot; runtime snapshot persistence uses `firewall_snapshots`, `firewall_address_list_entries`, and `firewall_rule_entries`. | Firewall rows and snapshots are canonical in the DB once persisted. Keep JSON exports during migration and retain any git-diff inputs required by deployment/audit. |
+
+## Consumers
+
+- `GET /api/reports/<source>` lists top-level files in the selected report
+  directory; `GET /api/reports/<source>/<path>` reads a file below it.
+- The reports page consumes those endpoints. Scan status and dashboard helpers
+  select the newest source and MikroTik report files for display.
+- `/api/blocklist` reads the consolidated IPv4/IPv6 text files. Scan history
+  reconstruction and firewall sync also consume those files alongside the
+  SQLite ledger and git history.
+- RouterOS `.rsc` exports and firewall synchronization consume generated
+  targets; these are operational outputs, not disposable presentation files.
+- `app.db.backfill` imports per-source `requests.csv`, blocklist history, and
+  the latest common MikroTik JSON snapshot. It does not currently import all
+  timestamped reports or all historical MikroTik snapshots.
+- Report retention removes old generated artifacts after consolidated scans,
+  subject to configured age/file-count limits; the consolidated blocklist
+  files are protected.
+
+## Restructuring Implications
+
+- Keep `requests`, `ip_stats`, blocklist tables, and firewall snapshot tables
+  as the queryable source of truth for facts they already represent. Avoid
+  copying those facts into a generic report blob.
+- A future report-artifact record should store metadata and structured fields
+  that have no canonical home yet. Rendered text/CSV/JSON should be generated
+  from those records or remain an explicit export.
+- Preserve source paths, generated timestamps, scan IDs, and hashes during
+  import so DB records can be traced back to original files.
+- Do not retire blocklist exports, RouterOS scripts, or source logs until
+  their consumers and audit requirements have explicit replacements.
+
+## Database and API Design
+
+### Canonical Data
+
+Continue treating normalized domain tables as canonical for facts they
+already represent:
+
+- `requests` for parsed request events.
+- `ip_stats` for per-IP/per-scan aggregates.
+- `scans` and `scans_sources` for run status, parameters, and source counts.
+- `blocklist_entries`, `blocklist_reasons`, and `blocklist_events` for block
+  decisions and history.
+- `firewall_snapshots`, `firewall_address_list_entries`, and
+  `firewall_rule_entries` for router state.
+
+Add a `report_artifacts` table only for report-specific summaries that cannot
+be reconstructed from those canonical rows. Proposed columns:
+
+| Column | Purpose |
+| --- | --- |
+| `id` | Integer primary key. |
+| `source` | `copyparty`, `npm`, `app`, or `mikrotik`. Keep as text because `log_sources` currently covers only request sources. |
+| `scan_id` | Nullable link to the generating scan; legacy imports may not have a known scan. |
+| `report_type` | Stable type such as `analyzer_summary`, `blocklist_change`, or `firewall_snapshot_summary`. |
+| `generated_at` | Original generation timestamp when known; otherwise import time plus an explicit unknown/original-time flag in metadata. |
+| `schema_version` | Version of the structured payload shape. |
+| `payload_json` | Structured report-specific summary, not a duplicate of requests or raw logs. |
+| `original_path` | Relative source path for provenance and legacy lookup. |
+| `content_sha256` | Hash of imported source content for integrity and idempotent import. |
+| `created_at` | Database import/creation timestamp. |
+
+Use a unique index on `(original_path, content_sha256)` for idempotent file
+imports. Keep payload schemas type-specific and versioned. Do not copy
+`requests.csv`, `ips.json`, firewall rows, or blocklists wholesale into
+`payload_json`; query their canonical tables and generate exports from those
+tables. If a rendered text report is retained as a database artifact, store
+its report-specific structured inputs and render text on demand rather than
+using opaque text as the only source of truth.
+
+### API Transition Contract
+
+- Add `GET /api/reports` with optional `source`, `type`, `since`, `until`,
+  `limit`, and opaque `cursor` filters. Return a stable list of artifact
+  metadata: `id`, `source`, `scan_id`, `type`, `generated_at`, and available
+  export formats. Enforce a bounded default and maximum page size.
+- Add `GET /api/report-artifacts/<artifact_id>` for the structured payload and
+  provenance metadata. Use a distinct path because `/api/reports/<source>`
+  already owns the one-segment route shape.
+- Add `GET /api/report-artifacts/<artifact_id>/export` for supported report types and formats
+  (initially JSON and, where useful, CSV/TXT). Generate exports from canonical
+  tables and versioned artifact payloads; do not accept filesystem paths from
+  a database-backed export request.
+- Preserve the current source/path routes as a read-only compatibility
+  fallback while files remain. The reports page should consume DB metadata
+  first, then offer legacy files when no imported artifact exists. Return
+  stable JSON errors for unknown IDs, unsupported formats, and unavailable
+  DB state.
+- Keep ordering deterministic (`generated_at DESC`, then `id DESC`), validate
+  filter values, and use cursor pagination for large histories. Do not expose
+  absolute server paths in list responses.
+
+### Sensitive and Large Content
+
+Do not include raw request lines or full request histories in report payloads;
+those belong in the dedicated log-storage design and require separate access
+and retention controls. Keep high-volume CSV exports generated on demand. Treat
+IP addresses, paths/query strings, user names, user agents, and firewall state
+as sensitive operational data: the existing web server must remain
+localhost-only, payloads should include only fields needed by the report, and
+new endpoints must not expose raw log data by default. Backups and exported
+files inherit the same access and retention requirements as the database.
+
+## Migration Implementation Status
+
+- Schema v2 adds `report_artifacts` with source, scan link, report type,
+  generation time, payload version, original relative path, and SHA-256 hash.
+- `python -m app.db.backfill --dry-run --reports-only` inventories supported
+  report files by source and format. The importer is idempotent, preserves
+  malformed JSON as text with a parse error, reports unreadable/oversized
+  files, and never modifies source files. It skips exports owned by canonical
+  tables (for example `requests.csv` and current blocklists).
+- New analyzer text summaries and consolidated blocklist change reports are
+  written to SQLite and their existing files. Firewall snapshots continue to
+  be persisted in the normalized firewall tables.
+- The Reports page lists database artifacts first, supports cursor pagination
+  and JSON/CSV/TXT downloads, and retains the existing file-backed routes as a
+  fallback for unmigrated files.
+- Validation covers idempotency, malformed/oversized inputs, source/format
+  counts, backup/restore, dual-write payloads, and API fallback. A workspace
+  dry-run found 76 files: 60 eligible, 16 canonical exports skipped, and none
+  unmappable at the time of the check.
+- No report files have been retired. Keep this gate closed until every listed
+  consumer has moved or has an explicit compatibility requirement, and a
+  production backup/restore and parity review has been approved.

app.py

@@ -1,55 +1,65 @@
-#!/usr/bin/env python3
-"""Intrusion Detection System — unified entry point.
-
-By default this starts the localhost-only Flask web UI. Pass ``scan`` to
-run the consolidated CLI scan instead.
-
-Usage:
-    python app.py                          # start the web UI (default)
-    python app.py --web-port 54666         # web UI with a custom port
-    python app.py scan [options]           # run the consolidated CLI scan
-"""
-
-from __future__ import annotations
-
-import argparse
-import sys
-
-from app.constants import WEB_HOST, WEB_PORT
-
-
-def _web_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(
-        prog="app.py",
-        description="Intrusion Detection System: web UI (default) or 'scan' subcommand.",
-    )
-    parser.add_argument("--web-host", default=WEB_HOST,
-                        help=f"web UI bind host (default: {WEB_HOST})")
-    parser.add_argument("--web-port", type=int, default=WEB_PORT,
-                        help=f"web UI port (default: {WEB_PORT})")
-    return parser
-
-
-def _run_web(host: str, port: int) -> None:
-    from app.web import serve
-    serve(host=host, port=port)
-
-
-def _run_scan(argv: list[str]) -> None:
-    from app.scan import main as scan_main
-    scan_main(argv)
-
-
-def main() -> None:
-    # `python app.py scan ...` runs the consolidated CLI scan.
-    if len(sys.argv) > 1 and sys.argv[1] == "scan":
-        _run_scan(sys.argv[2:])
-        return
-
-    # Default: start the web UI (parse_known_args tolerates stray flags).
-    args, _ = _web_parser().parse_known_args()
-    _run_web(args.web_host, args.web_port)
-
-
-if __name__ == "__main__":
-    main()
+#!/usr/bin/env python3
+"""Intrusion Detection System — unified entry point.
+
+By default this starts the localhost-only Flask web UI. Pass ``scan`` to
+run the consolidated CLI scan instead.
+
+Usage:
+    python app.py                          # start the web UI (default)
+    python app.py --web-port 54666         # web UI with a custom port
+    python app.py scan [options]           # run the consolidated CLI scan
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from app.utils.constants import WEB_HOST, WEB_PORT
+
+
+def _web_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(
+        prog="app.py",
+        description="Intrusion Detection System: web UI (default) or 'scan' subcommand.",
+    )
+    parser.add_argument("--web-host", default=WEB_HOST,
+                        help=f"web UI bind host (default: {WEB_HOST})")
+    parser.add_argument("--web-port", type=int, default=WEB_PORT,
+                        help=f"web UI port (default: {WEB_PORT})")
+    return parser
+
+
+def _run_web(host: str, port: int) -> None:
+    from app.web import serve
+    serve(host=host, port=port)
+
+
+def _run_scan(argv: list[str]) -> None:
+    from app.scan import main as scan_main
+    scan_main(argv)
+
+
+def _run_analyze(argv: list[str]) -> None:
+    from app.analyze import main as analyze_main
+    analyze_main(argv)
+
+
+def main() -> None:
+    # `python app.py scan ...` runs the consolidated CLI scan.
+    if len(sys.argv) > 1 and sys.argv[1] == "scan":
+        _run_scan(sys.argv[2:])
+        return
+
+    # `python app.py analyze ...` prints a detailed report from the SQLite store.
+    if len(sys.argv) > 1 and sys.argv[1] == "analyze":
+        _run_analyze(sys.argv[2:])
+        return
+
+    # Default: start the web UI (parse_known_args tolerates stray flags).
+    args, _ = _web_parser().parse_known_args()
+    _run_web(args.web_host, args.web_port)
+
+
+if __name__ == "__main__":
+    main()

app/__init__.py

@@ -1 +1,9 @@
-# Intrusion Detection System — app package
+# Intrusion Detection System — app package
+#
+# Package structure:
+#   app/          core domain (analyzers, firewall, detection rules)
+#   app/web/      Flask web UI (app factory, routes, background scan runner)
+#   app/scan/     consolidated scan pipeline + incremental scan state
+#   app/utils/    shared utilities (constants, I/O, models, whitelists, etc.)
+#   app/static/   static assets (CSS, JS)
+#   app/templates/ Jinja2 templates

app/analyze.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""CLI detailed log-analysis report (L002).
+
+``python app.py analyze [--ip IP] [--since ISO] [--top N] [--json]`` reads the
+persisted SQLite store (request history, per-IP aggregates, blocklist ledger)
+and prints a detailed, interactive analysis. It complements the web UI's
+``/analysis`` page for use on a headless server or in scripts.
+
+No scan is run here: this only queries what ``scan`` has already persisted.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from datetime import datetime
+
+
+def _fmt_ts(epoch: float | None, default: str = "-") -> str:
+    if not epoch:
+        return default
+    try:
+        return datetime.fromtimestamp(epoch).isoformat(timespec="seconds")
+    except (ValueError, OverflowError, OSError):
+        return default
+
+
+def _build_parser() -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(
+        prog="python app.py analyze",
+        description="Detailed log-analysis report from the SQLite store.")
+    p.add_argument("--ip", metavar="IP", default=None,
+                   help="focus the report on a single IP (drill-down)")
+    p.add_argument("--since", metavar="ISO", default=None,
+                   help="only analyse requests at/after this ISO timestamp")
+    p.add_argument("--top", type=int, default=20,
+                   help="rows in the top-IP / threat tables (default: 20)")
+    p.add_argument("--json", action="store_true",
+                   help="emit raw JSON instead of a human-readable report")
+    return p
+
+
+def _report(conn, opts) -> dict:
+    from .db import repositories as repo
+
+    data: dict = {}
+    data["scans"] = repo.latest_scans(conn, limit=10)
+    data["totals"] = {
+        "requests": repo.count_requests(conn),
+        "scans": repo.count_scans(conn),
+    }
+
+    if opts.ip:
+        data["ip"] = {
+            "ip": opts.ip,
+            "timeline": repo.ip_timeline(conn, ip=opts.ip),
+            "active_blocklist": [e for e in repo.active_blocklist(conn)
+                                 if e["cidr"].startswith(opts.ip)],
+        }
+    else:
+        data["top_ips"] = repo.top_ips(conn, since=opts.since, limit=opts.top)
+        data["cross_source"] = repo.correlate_across_sources(
+            conn, limit=opts.top)
+        data["blocklist"] = repo.active_blocklist(conn)
+        data["recent_errors"] = repo.query_requests(
+            conn, since=opts.since, limit=opts.top)
+    return data
+
+
+def _render_text(data: dict, opts) -> str:
+    lines = ["Detailed log analysis", "=" * 40, ""]
+    totals = data["totals"]
+    lines.append(f"SQLite store: {totals['requests']} requests across "
+                 f"{totals['scans']} scan(s)")
+    if data["scans"]:
+        lines.append("Latest scans:")
+        for s in data["scans"]:
+            lines.append(f"  #{s['id']:<4} {s['kind']:<8} {s['status']:<7} "
+                         f"{s.get('started') or s.get('started_at') or '-'}")
+    lines.append("")
+
+    if "ip" in data:
+        ip = data["ip"]
+        lines.append(f"== Request history for {ip['ip']} ==")
+        bl = ip["active_blocklist"]
+        if bl:
+            lines.append(f"  (source: active blocklist entries "
+                         f"{', '.join(e['cidr'] for e in bl)})")
+        lines.append(f"{'time':<24} {'src':<9} {'method':<8} {'status':<6} path")
+        for r in ip["timeline"][:opts.top]:
+            lines.append(
+                f"{r.get('ts') or '-':<24} {str(r.get('source')):<9} "
+                f"{str(r.get('method') or '-'):<8} "
+                f"{str(r.get('status') or '-'):<6} {r.get('path') or '-'}")
+    else:
+        lines.append("== Top IPs ==")
+        lines.append(
+            f"{'ip':<18} {'ver':<3} {'reqs':<6} {'err':<5} {'mal':<5} "
+            f"{'first':<22} last")
+        for r in data["top_ips"]:
+            lines.append(
+                f"{str(r['ip']):<18} {r.get('version') or '-':<3} "
+                f"{r['total']:<6} {r['errors']:<5} {r['malicious_hits']:<5} "
+                f"{_fmt_ts(r['first_epoch']):<22} {_fmt_ts(r['last_epoch'])}")
+        lines.append("")
+
+        lines.append("== Cross-source IPs ==")
+        if data["cross_source"]:
+            for r in data["cross_source"]:
+                lines.append(
+                    f"  {r['ip']:<18} sources={r['sources']} "
+                    f"({r['source_list']}) total={r['total']}")
+        else:
+            lines.append("  (none — no IP seen in more than one log source)")
+
+        lines.append("")
+        lines.append("== Active blocklist (ledger) ==")
+        if data["blocklist"]:
+            for e in data["blocklist"]:
+                lines.append(f"  {e['cidr']:<22} added={e['added_at']}")
+        else:
+            lines.append("  (empty)")
+    return "\n".join(lines)
+
+
+def main(argv: list[str] | None = None) -> None:
+    opts = _build_parser().parse_args(argv)
+    try:
+        from .db import get_connection
+        conn = get_connection()
+        data = _report(conn, opts)
+    except Exception as exc:
+        print(f"error: could not read SQLite store ({exc}); "
+              f"run a scan first so data exists.", file=sys.stderr)
+        sys.exit(1)
+
+    if opts.json:
+        print(json.dumps(data, indent=2, sort_keys=True, default=str))
+    else:
+        print(_render_text(data, opts))
+
+
+if __name__ == "__main__":
+    main()

app/analyzer.py

@@ -1,290 +1,378 @@
-#!/usr/bin/env python3
-"""Analysis pipeline for the intrusion-detection system.
-
-`Analyzer` is the log-source-agnostic base class.  Subclasses implement
-`parse_line()` and `flag()`.
-"""
-
-from __future__ import annotations
-
-import os
-import re
-from collections import Counter
-from datetime import datetime
-from typing import Any, Sequence
-
-from .files import ensure_dirs, find_logs, read_lines, read_lines_from, validate_dir, write_csv, write_json, write_lines
-from .flags import by_total, flag_malicious_patterns
-from .ip_utils import is_whitelisted, split_versions
-from .mikrotik import write_mikrotik
-from .models import IPStats, Request, aggregate
-from .reporting import Column, columns_header, columns_row, render_kv, render_table, timestamp_name
-from .constants import DEFAULT_TOP, DEFAULT_TOP_PATHS
-from . import scan_state
-from .whitelists import refresh_whitelists
-
-
-class Analyzer:
-    """Log-source-agnostic analysis pipeline.
-
-    Subclasses must set the class attributes below and implement
-    `parse_line()` and `flag()`.
-    """
-
-    name = "ids"                       # short id, used in file names / tags
-    title = "IDS log analysis report"
-    log_pattern: re.Pattern | None = None
-    log_suffixes: tuple[str, ...] = ()
-    exclude_prefixes: tuple[str, ...] = ()
-    ip_columns: Sequence[Column] = ()
-    request_columns: Sequence[Column] = ()
-    paths_file = "error_paths.csv"
-    paths_label = "error paths"
-    flag_files = ("flagged_ips.txt", "flagged_ipv6.txt")
-    flag_label = "Flagged IPs"
-    flag_note = ""
-
-    def __init__(self, log_path: str, report_dir: str, top: int = DEFAULT_TOP,
-                 quiet: bool = False, write_rsc: bool = True,
-                 full: bool = False):
-        self.log_path = log_path
-        self.report_dir = report_dir
-        self.data_dir = os.path.join(report_dir, "data")
-        self.top = top
-        self.quiet = quiet
-        self.write_rsc = write_rsc
-        self.full = full
-        self.files: list[str] = []
-        self.requests: list[Request] = []
-        self.stats: dict[str, IPStats] = {}
-        self.flagged: dict[str, IPStats] = {}
-        self.per_file: dict[str, int] = {}
-
-    # --- hooks -------------------------------------------------------------
-
-    def parse_line(self, line: str) -> Request | None:
-        raise NotImplementedError
-
-    def flag(self, stats: dict[str, IPStats]) -> dict[str, IPStats]:
-        raise NotImplementedError
-
-    def flag_comment(self, s: IPStats) -> str:
-        return (f"errors={s.errors} total={s.total} ratio={s.error_ratio} "
-                f"first={s.first} last={s.last}")
-
-    def block_candidates(self) -> set[str]:
-        malicious = flag_malicious_patterns(self.stats)
-        return set(self.flagged) | set(malicious)
-
-    def extra_summary_rows(self) -> list[tuple[str, Any]]:
-        return []
-
-    def write_extra(self, data_dir: str) -> list[str]:
-        return []
-
-    # --- pipeline ----------------------------------------------------------
-
-    def discover(self) -> list[str]:
-        self.files = find_logs(self.log_path, self.log_pattern,
-                               self.log_suffixes, self.exclude_prefixes)
-        if not self.files:
-            raise FileNotFoundError(f"no log files found in: {self.log_path}")
-        return self.files
-
-    def load(self) -> list[Request]:
-        """Read log entries, resuming from the last processed offset.
-
-        Non-gzip logs are read incrementally (only new bytes since the last
-        scan) unless ``full`` is set; gzip archives are always fully read
-        because they cannot be seeked.
-        """
-        state: dict[str, int] = {}
-        if not self.full:
-            state = scan_state.load_state()
-            # Prune entries whose log file no longer exists (safe across
-            # analyzers: only removes dead keys, keeps live ones).
-            state = {k: v for k, v in state.items() if os.path.exists(k)}
-        for name in self.files:
-            full_path = os.path.join(self.log_path, name)
-            count = 0
-            if self.full or full_path.endswith(".gz"):
-                for line in read_lines(full_path):
-                    req = self.parse_line(line)
-                    if req is not None:
-                        self.requests.append(req)
-                        count += 1
-            else:
-                try:
-                    size = os.path.getsize(full_path)
-                except OSError:
-                    size = 0
-                recorded = state.get(full_path, 0)
-                # Reset to the start if the file shrank or was truncated.
-                start = recorded if 0 < recorded <= size else 0
-                last_resume = start
-                for line, resume in read_lines_from(full_path, start):
-                    req = self.parse_line(line)
-                    if req is not None:
-                        self.requests.append(req)
-                        count += 1
-                    last_resume = resume
-                state[full_path] = last_resume
-            self.per_file[name] = count
-        if not self.full:
-            scan_state.save_state(state)
-        return self.requests
-
-    def analyze(self) -> dict[str, IPStats]:
-        self.stats = aggregate(self.requests)
-        self.flagged = self.flag(self.stats)
-        return self.stats
-
-    # --- derived data ------------------------------------------------------
-
-    def error_paths(self) -> Counter:
-        return Counter(r.path for r in self.requests if r.is_error and r.path)
-
-    def flagged_split(self) -> tuple[list[IPStats], list[IPStats]]:
-        v4 = [s for s in self.flagged.values() if s.version == 4]
-        v6 = [s for s in self.flagged.values() if s.version == 6]
-        def key(s): return (-s.errors, s.ip)
-        return sorted(v4, key=key), sorted(v6, key=key)
-
-    def summary_rows(self) -> list[tuple[str, Any]]:
-        v4, v6 = split_versions(self.stats)
-        rows = [
-            ("log path", self.log_path),
-            ("log files", len(self.files)),
-            ("total requests", len(self.requests)),
-            ("unique IPs",
-             f"{len(self.stats)} (IPv4 {len(v4)} / IPv6 {len(v6)})"),
-            ("error requests", sum(1 for r in self.requests if r.is_error)),
-            (f"unique {self.paths_label}", len(self.error_paths())),
-        ]
-        rows += self.extra_summary_rows()
-        rows += [
-            (self.flag_label.lower(), len(self.flagged)),
-            ("whitelisted IPs seen",
-             sum(1 for ip in self.stats if is_whitelisted(ip))),
-            ("block candidates", len(self.block_candidates())),
-        ]
-        return rows
-
-    # --- reporting ---------------------------------------------------------
-
-    def text_report(self) -> str:
-        out = [self.title,
-               f"generated: {datetime.now().isoformat(timespec='seconds')}",
-               "",
-               render_kv(self.summary_rows()), ""]
-
-        out.append(f"== top {self.top} IPs ==")
-        rows = [columns_row(self.ip_columns, s)
-                for s in by_total(self.stats)[:self.top]]
-        out.append(render_table(columns_header(self.ip_columns), rows))
-        if len(self.stats) > self.top:
-            out.append(f"... {len(self.stats) - self.top} more IPs omitted")
-        out.append("")
-
-        out.append(f"== top {DEFAULT_TOP_PATHS} {self.paths_label} ==")
-        for path, n in self.error_paths().most_common(DEFAULT_TOP_PATHS):
-            out.append(f"  {n:>5}  {path}")
-        out.append("")
-
-        v4, v6 = self.flagged_split()
-        out.append(f"== {self.flag_label} ({len(self.flagged)}) ==")
-        if self.flag_note:
-            out.append(self.flag_note)
-        for label, group in (("IPv4", v4), ("IPv6", v6)):
-            if not group:
-                continue
-            out.append(f"-- {label} ({len(group)}) --")
-            rows = [columns_row(self.ip_columns, s) for s in group]
-            out.append(render_table(columns_header(self.ip_columns), rows))
-        out.append("")
-        return "\n".join(out)
-
-    # --- writing -----------------------------------------------------------
-
-    def write_data(self) -> list[str]:
-        ensure_dirs(self.report_dir, self.data_dir)
-        written: list[str] = []
-        d = self.data_dir
-        header = columns_header(self.ip_columns)
-        ordered = by_total(self.stats)
-
-        for version, fname in ((4, "ips_ipv4.csv"), (6, "ips_ipv6.csv")):
-            rows = [columns_row(self.ip_columns, s)
-                    for s in ordered if s.version == version]
-            written.append(write_csv(os.path.join(d, fname), header, rows))
-
-        written.append(write_json(
-            os.path.join(d, "ips.json"),
-            {s.ip: s.to_dict() for s in ordered}))
-
-        written.append(write_csv(
-            os.path.join(d, self.paths_file), ["path", "count"],
-            self.error_paths().most_common()))
-
-        v4, v6 = self.flagged_split()
-        for group, fname in zip((v4, v6), self.flag_files):
-            written.append(write_lines(
-                os.path.join(d, fname),
-                [f"{s.ip}  # {self.flag_comment(s)}" for s in group]))
-
-        blocked_v4, blocked_v6 = split_versions(
-            ip for ip in self.block_candidates() if not is_whitelisted(ip))
-        written.append(write_lines(
-            os.path.join(d, "blocked_ipv4.txt"), blocked_v4))
-        written.append(write_lines(
-            os.path.join(d, "blocked_ipv6.txt"), blocked_v6))
-
-        written.append(write_csv(
-            os.path.join(d, "requests.csv"),
-            columns_header(self.request_columns),
-            (columns_row(self.request_columns, r) for r in self.requests)))
-
-        written += self.write_extra(d)
-        return written
-
-    def write_report(self) -> str:
-        ensure_dirs(self.report_dir)
-        path = os.path.join(self.report_dir, f"{timestamp_name()}.txt")
-        with open(path, "w", encoding="utf-8") as f:
-            f.write(self.text_report())
-        return path
-
-    def write_rsc_file(self) -> tuple[str, int, int]:
-        ensure_dirs(self.data_dir)
-        return write_mikrotik(
-            os.path.join(self.data_dir, "mikrotik-block.rsc"),
-            f"{self.name}-ids", self.block_candidates())
-
-    # --- entrypoint --------------------------------------------------------
-
-    def run(self) -> dict[str, Any]:
-        validate_dir(self.log_path)
-        ensure_dirs(self.report_dir, self.data_dir)
-        refresh_whitelists()
-        self.discover()
-        self.load()
-        self.analyze()
-
-        if not self.quiet:
-            print(self.text_report())
-
-        self.write_data()
-        report_path = self.write_report()
-        print(f"data files:     {self.data_dir}")
-        print(f"summary report: {report_path}")
-
-        result: dict[str, Any] = {"report": report_path,
-                                  "data_dir": self.data_dir}
-        if self.write_rsc:
-            rsc, n_blocked, n_whitelisted = self.write_rsc_file()
-            print(f"mikrotik script: {rsc}")
-            print(f"  {n_blocked} IPs blocked, "
-                  f"{n_whitelisted} CIDR ranges whitelisted")
-            result.update(rsc=rsc, blocked=n_blocked,
-                          whitelisted=n_whitelisted)
-        return result
+#!/usr/bin/env python3
+"""Analysis pipeline for the intrusion-detection system.
+
+`Analyzer` is the log-source-agnostic base class.  Subclasses implement
+`parse_line()` and `flag()`.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+from collections import Counter
+from datetime import datetime
+from typing import Any, Sequence
+
+from .utils.files import ensure_dirs, find_logs, read_lines, read_lines_from, validate_dir, write_csv, write_json, write_lines
+from .flags import by_total, flag_malicious_patterns
+from .utils.ip_utils import is_whitelisted, split_versions
+from .mikrotik import write_mikrotik
+from .utils.models import IPStats, Request, aggregate
+from .utils.reporting import Column, columns_header, columns_row, render_kv, render_table, timestamp_name
+from .utils.constants import DEFAULT_TOP, DEFAULT_TOP_PATHS
+from .scan import scan_state
+from .utils.whitelists import refresh_whitelists
+
+
+class Analyzer:
+    """Log-source-agnostic analysis pipeline.
+
+    Subclasses must set the class attributes below and implement
+    `parse_line()` and `flag()`.
+    """
+
+    name = "ids"                       # short id, used in file names / tags
+    title = "IDS log analysis report"
+    log_pattern: re.Pattern | None = None
+    log_suffixes: tuple[str, ...] = ()
+    exclude_prefixes: tuple[str, ...] = ()
+    ip_columns: Sequence[Column] = ()
+    request_columns: Sequence[Column] = ()
+    paths_file = "error_paths.csv"
+    paths_label = "error paths"
+    flag_files = ("flagged_ips.txt", "flagged_ipv6.txt")
+    flag_label = "Flagged IPs"
+    flag_note = ""
+
+    def __init__(self, log_path: str, report_dir: str, top: int = DEFAULT_TOP,
+                 quiet: bool = False, write_rsc: bool = True,
+                 full: bool = False):
+        self.log_path = log_path
+        self.report_dir = report_dir
+        self.data_dir = os.path.join(report_dir, "data")
+        self.top = top
+        self.quiet = quiet
+        self.write_rsc = write_rsc
+        self.full = full
+        self.files: list[str] = []
+        self.requests: list[Request] = []
+        self.stats: dict[str, IPStats] = {}
+        self.flagged: dict[str, IPStats] = {}
+        self.per_file: dict[str, int] = {}
+
+    # --- sqlite persistence ------------------------------------------------
+
+    def persist_to_db(self, scan_id: int | None = None) -> None:
+        """Ingest this analyzer's requests + per-IP stats into SQLite.
+
+        Lazily imported to avoid cycles. When ``scan_id`` is None a standalone
+        scan record is created for this analyzer so individual
+        (non-consolidated) runs are still tracked. Never raises: a DB failure
+        logs a warning but does not break the scan.
+        """
+        if not self.requests and not self.stats:
+            return
+        try:
+            from .db import get_connection
+            from .db import repositories as repo
+            from .utils.ip_utils import ip_version as _ip_ver
+            conn = get_connection()
+            if scan_id is None:
+                scan_id = repo.start_scan(conn, self.name,
+                                          params={"full": self.full})
+            source = self.name
+            rows: list[dict] = []
+            for r in self.requests:
+                rows.append({
+                    "source": source, "scan_id": scan_id,
+                    "log_file": ((r.extra or {}).get("log_file")
+                                  or (self.files[0] if self.files else None)),
+                    "ts": r.ts, "ip": r.ip,
+                    "version": _ip_ver(r.ip) or 0,
+                    "method": r.method, "path": r.path,
+                    "status": r.status,
+                    "upstream_status": r.upstream_status,
+                    "host": r.host, "user": r.user, "ua": r.ua,
+                    "referer": r.referer, "denied": r.denied,
+                    "banned": r.banned, "is_error": r.is_error,
+                    "extra": r.extra,
+                })
+            repo.insert_requests(conn, rows)
+            repo.insert_ip_stats(conn, scan_id,
+                                 list(self.stats.values()),
+                                 set(self.block_candidates()))
+            repo.set_scan_source_counts(
+                conn, scan_id, source, len(self.requests), len(self.files))
+            repo.finish_scan(conn, scan_id, status="done",
+                             log_files=len(self.files),
+                             request_count=len(self.requests))
+        except Exception as exc:  # never break the scan because of DB issues
+            if not self.quiet:
+                print(f"WARN: sqlite ingest skipped ({exc})")
+    # --- hooks -------------------------------------------------------------
+
+    def parse_line(self, line: str) -> Request | None:
+        raise NotImplementedError
+
+    def attr_source(self, req: Request, log_file: str) -> None:
+        """Record the originating log file onto a parsed request."""
+        if req.extra is None:
+            req.extra = {}
+        req.extra["log_file"] = req.extra.get("log_file", log_file)
+
+    def flag(self, stats: dict[str, IPStats]) -> dict[str, IPStats]:
+        raise NotImplementedError
+
+    def flag_comment(self, s: IPStats) -> str:
+        return (f"errors={s.errors} total={s.total} ratio={s.error_ratio} "
+                f"first={s.first} last={s.last}")
+
+    def block_candidates(self) -> set[str]:
+        malicious = flag_malicious_patterns(self.stats)
+        return set(self.flagged) | set(malicious)
+
+    def extra_summary_rows(self) -> list[tuple[str, Any]]:
+        return []
+
+    def write_extra(self, data_dir: str) -> list[str]:
+        return []
+
+    # --- pipeline ----------------------------------------------------------
+
+    def discover(self) -> list[str]:
+        self.files = find_logs(self.log_path, self.log_pattern,
+                               self.log_suffixes, self.exclude_prefixes)
+        if not self.files:
+            raise FileNotFoundError(f"no log files found in: {self.log_path}")
+        return self.files
+
+    def load(self) -> list[Request]:
+        """Read log entries, resuming from the last processed offset.
+
+        Non-gzip logs are read incrementally (only new bytes since the last
+        scan) unless ``full`` is set; gzip archives are always fully read
+        because they cannot be seeked.
+        """
+        state: dict[str, int] = {}
+        if not self.full:
+            state = scan_state.load_state()
+            # Prune entries whose log file no longer exists (safe across
+            # analyzers: only removes dead keys, keeps live ones).
+            state = {k: v for k, v in state.items() if os.path.exists(k)}
+        for name in self.files:
+            full_path = os.path.join(self.log_path, name)
+            count = 0
+            if self.full or full_path.endswith(".gz"):
+                for line in read_lines(full_path):
+                    req = self.parse_line(line)
+                    if req is not None:
+                        self.attr_source(req, name)
+                        self.requests.append(req)
+                        count += 1
+            else:
+                try:
+                    size = os.path.getsize(full_path)
+                except OSError:
+                    size = 0
+                recorded = state.get(full_path, 0)
+                # Reset to the start if the file shrank or was truncated.
+                start = recorded if 0 < recorded <= size else 0
+                last_resume = start
+                for line, resume in read_lines_from(full_path, start):
+                    req = self.parse_line(line)
+                    if req is not None:
+                        self.attr_source(req, name)
+                        self.requests.append(req)
+                        count += 1
+                    last_resume = resume
+                state[full_path] = last_resume
+            self.per_file[name] = count
+        if not self.full:
+            scan_state.save_state(state)
+        return self.requests
+
+    def analyze(self) -> dict[str, IPStats]:
+        self.stats = aggregate(self.requests)
+        self.flagged = self.flag(self.stats)
+        return self.stats
+
+    # --- derived data ------------------------------------------------------
+
+    def error_paths(self) -> Counter:
+        return Counter(r.path for r in self.requests if r.is_error and r.path)
+
+    def flagged_split(self) -> tuple[list[IPStats], list[IPStats]]:
+        v4 = [s for s in self.flagged.values() if s.version == 4]
+        v6 = [s for s in self.flagged.values() if s.version == 6]
+        def key(s): return (-s.errors, s.ip)
+        return sorted(v4, key=key), sorted(v6, key=key)
+
+    def summary_rows(self) -> list[tuple[str, Any]]:
+        v4, v6 = split_versions(self.stats)
+        rows = [
+            ("log path", self.log_path),
+            ("log files", len(self.files)),
+            ("total requests", len(self.requests)),
+            ("unique IPs",
+             f"{len(self.stats)} (IPv4 {len(v4)} / IPv6 {len(v6)})"),
+            ("error requests", sum(1 for r in self.requests if r.is_error)),
+            (f"unique {self.paths_label}", len(self.error_paths())),
+        ]
+        rows += self.extra_summary_rows()
+        rows += [
+            (self.flag_label.lower(), len(self.flagged)),
+            ("whitelisted IPs seen",
+             sum(1 for ip in self.stats if is_whitelisted(ip))),
+            ("block candidates", len(self.block_candidates())),
+        ]
+        return rows
+
+    # --- reporting ---------------------------------------------------------
+
+    def text_report(self) -> str:
+        out = [self.title,
+               f"generated: {datetime.now().isoformat(timespec='seconds')}",
+               "",
+               render_kv(self.summary_rows()), ""]
+
+        out.append(f"== top {self.top} IPs ==")
+        rows = [columns_row(self.ip_columns, s)
+                for s in by_total(self.stats)[:self.top]]
+        out.append(render_table(columns_header(self.ip_columns), rows))
+        if len(self.stats) > self.top:
+            out.append(f"... {len(self.stats) - self.top} more IPs omitted")
+        out.append("")
+
+        out.append(f"== top {DEFAULT_TOP_PATHS} {self.paths_label} ==")
+        for path, n in self.error_paths().most_common(DEFAULT_TOP_PATHS):
+            out.append(f"  {n:>5}  {path}")
+        out.append("")
+
+        v4, v6 = self.flagged_split()
+        out.append(f"== {self.flag_label} ({len(self.flagged)}) ==")
+        if self.flag_note:
+            out.append(self.flag_note)
+        for label, group in (("IPv4", v4), ("IPv6", v6)):
+            if not group:
+                continue
+            out.append(f"-- {label} ({len(group)}) --")
+            rows = [columns_row(self.ip_columns, s) for s in group]
+            out.append(render_table(columns_header(self.ip_columns), rows))
+        out.append("")
+        return "\n".join(out)
+
+    # --- writing -----------------------------------------------------------
+
+    def write_data(self) -> list[str]:
+        ensure_dirs(self.report_dir, self.data_dir)
+        written: list[str] = []
+        d = self.data_dir
+        header = columns_header(self.ip_columns)
+        ordered = by_total(self.stats)
+
+        for version, fname in ((4, "ips_ipv4.csv"), (6, "ips_ipv6.csv")):
+            rows = [columns_row(self.ip_columns, s)
+                    for s in ordered if s.version == version]
+            written.append(write_csv(os.path.join(d, fname), header, rows))
+
+        written.append(write_json(
+            os.path.join(d, "ips.json"),
+            {s.ip: s.to_dict() for s in ordered}))
+
+        written.append(write_csv(
+            os.path.join(d, self.paths_file), ["path", "count"],
+            self.error_paths().most_common()))
+
+        v4, v6 = self.flagged_split()
+        for group, fname in zip((v4, v6), self.flag_files):
+            written.append(write_lines(
+                os.path.join(d, fname),
+                [f"{s.ip}  # {self.flag_comment(s)}" for s in group]))
+
+        blocked_v4, blocked_v6 = split_versions(
+            ip for ip in self.block_candidates() if not is_whitelisted(ip))
+        written.append(write_lines(
+            os.path.join(d, "blocked_ipv4.txt"), blocked_v4))
+        written.append(write_lines(
+            os.path.join(d, "blocked_ipv6.txt"), blocked_v6))
+
+        written.append(write_csv(
+            os.path.join(d, "requests.csv"),
+            columns_header(self.request_columns),
+            (columns_row(self.request_columns, r) for r in self.requests)))
+
+        written += self.write_extra(d)
+        return written
+
+    def write_report(self) -> str:
+        ensure_dirs(self.report_dir)
+        path = os.path.join(self.report_dir, f"{timestamp_name()}.txt")
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(self.text_report())
+        return path
+
+    def write_rsc_file(self) -> tuple[str, int, int]:
+        ensure_dirs(self.data_dir)
+        return write_mikrotik(
+            os.path.join(self.data_dir, "mikrotik-block.rsc"),
+            f"{self.name}-ids", self.block_candidates())
+
+    # --- entrypoint --------------------------------------------------------
+
+    def run(self, db_scan_id: int | None = None) -> dict[str, Any]:
+        validate_dir(self.log_path)
+        ensure_dirs(self.report_dir, self.data_dir)
+        refresh_whitelists()
+        self.discover()
+        self.load()
+        self.analyze()
+
+        if not self.quiet:
+            print(self.text_report())
+
+        # Ingest into the SQLite store (creates its own scan record when no
+        # consolidated ``db_scan_id`` is supplied).
+        self.persist_to_db(scan_id=db_scan_id)
+
+        self.write_data()
+        report_path = self.write_report()
+        self.persist_report_artifact(report_path, db_scan_id)
+        print(f"data files:     {self.data_dir}")
+        print(f"summary report: {report_path}")
+
+        result: dict[str, Any] = {"report": report_path,
+                                  "data_dir": self.data_dir}
+        if self.write_rsc:
+            rsc, n_blocked, n_whitelisted = self.write_rsc_file()
+            print(f"mikrotik script: {rsc}")
+            print(f"  {n_blocked} IPs blocked, "
+                  f"{n_whitelisted} CIDR ranges whitelisted")
+            result.update(rsc=rsc, blocked=n_blocked,
+                          whitelisted=n_whitelisted)
+        return result
+
+    def persist_report_artifact(self, path: str,
+                                scan_id: int | None = None) -> None:
+        """Store the rendered summary in SQLite while keeping its file export."""
+        try:
+            from .db import get_connection
+            from .db import repositories as repo
+            with open(path, "rb") as file:
+                content = file.read()
+            stat = os.stat(path)
+            relative = os.path.relpath(path, self.report_dir).replace(os.sep, "/")
+            repo.insert_report_artifact(
+                get_connection(), source=self.name,
+                report_type="analyzer_summary",
+                generated_at=datetime.fromtimestamp(
+                    stat.st_mtime).astimezone().isoformat(timespec="seconds"),
+                payload={"format": "text",
+                         "content": content.decode("utf-8", errors="replace")},
+                original_path=f"{self.name}/{relative}",
+                content_sha256=hashlib.sha256(content).hexdigest(),
+                scan_id=scan_id,
+            )
+        except Exception as exc:
+            if not self.quiet:
+                print(f"WARN: report artifact DB write skipped ({exc})")

app/bad_patterns.py

@@ -1,190 +1,190 @@
-#!/usr/bin/env python3
-"""Malicious-pattern detection rules for the intrusion-detection system.
-
-Defines compiled regex patterns and classifiers that identify
-probing/scanning URLs (e.g. env files, wp-admin, webshells) and
-suspicious file names.  Imported by analyzers to flag IPs hitting
-these patterns as blocklist candidates.
-"""
-
-from __future__ import annotations
-
-import re
-from dataclasses import dataclass, field
-from typing import ClassVar, Pattern
-
-# ---------------------------------------------------------------------------
-# Pattern categories
-# ---------------------------------------------------------------------------
-
-
-@dataclass(frozen=True)
-class BadPattern:
-    """A single detection rule.
-
-    Attributes:
-        name: Short identifier (e.g. "env_file", "wp_admin").
-        label: Human-readable description for reports.
-        regex: Compiled regex matched against the request path.
-        severity: 1 = moderate, 2 = high, 3 = critical (auto-block).
-    """
-
-    name: str
-    label: str
-    regex: Pattern[str]
-    severity: int = 2
-
-    def matches(self, path: str) -> bool:
-        return bool(self.regex.search(path))
-
-
-# ---------------------------------------------------------------------------
-# Compiled pattern bank
-# ---------------------------------------------------------------------------
-
-# -- env / config file probes -------------------------------------------------
-
-ENV_FILE = BadPattern(
-    "env_file",
-    "env/config file probe (.env, .aws/credentials, config.json)",
-    re.compile(
-        r"/\.(?:env|config)(?:\.\w+)*|"
-        r"/\.aws/credentials|"
-        r"/config\.json",
-        re.IGNORECASE,
-    ),
-    severity=3,
-)
-
-# -- version control probes ---------------------------------------------------
-
-VCS_PROBE = BadPattern(
-    "vcs_probe",
-    "version-control probe (.git, .svn)",
-    re.compile(
-        r"/\.(?:git|svn)(?:/|$)",
-        re.IGNORECASE,
-    ),
-    severity=3,
-)
-
-# -- WordPress / CMS probes ---------------------------------------------------
-
-WP_PROBE = BadPattern(
-    "wp_probe",
-    "WordPress/CMS probe (wp-admin, wp-login, wp-content, wp-includes)",
-    re.compile(
-        r"/wp-(?:admin|login|content|includes|load|cron|xmlrpc)(?:/|\.php)",
-        re.IGNORECASE,
-    ),
-    severity=2,
-)
-
-# -- well-known / common probe paths ------------------------------------------
-
-WELL_KNOWN_PROBE = BadPattern(
-    "well_known_probe",
-    ".well-known probe",
-    re.compile(
-        r"/\.well-known/",
-        re.IGNORECASE,
-    ),
-    severity=1,
-)
-
-# -- webshell / backdoor probes -----------------------------------------------
-
-WEBSHELL_PROBE = BadPattern(
-    "webshell_probe",
-    "webshell/backdoor probe (known shell filenames, phpinfo, file managers)",
-    re.compile(
-        r"/(?:"
-        r"phpinfo|info|shell|cmd|upload|"
-        r"wp_filemanager|filemanager|"
-        r"xstelth|adminfuns|hellopress|"
-        r"fpwch|crgio|coffexium|"
-        r"ccou|biufile|"
-        r"ms-edit|classwithtostring|"
-        r"this_is_a_new_hello_world|"
-        r"chosen|ops|simple|inputs"
-        r")\.php",
-        re.IGNORECASE,
-    ),
-    severity=3,
-)
-
-# -- suspicious generic PHP file requests -------------------------------------
-
-GENERIC_PHP_PROBE = BadPattern(
-    "generic_php_probe",
-    "generic/random PHP file probe (single-char, numeric names)",
-    re.compile(
-        r"/(?:"
-        r"[a-z]\.php|"  # single letter: a.php, k.php, x.php
-        r"\d{1,3}\.php|"  # numeric: 1.php, 222.php, 155.php
-        r"[a-z]{2}\.php|"  # two-letter: aa.php, dx.php
-        r"a\d\.php|"  # a1.php
-        r"\w+\.php"  # generic: anything.php
-        r")",
-        re.IGNORECASE,
-    ),
-    severity=1,
-)
-
-# -- sensitive file probes ----------------------------------------------------
-
-SENSITIVE_FILE = BadPattern(
-    "sensitive_file",
-    "sensitive file probe (id_rsa, id_dsa, .pem, .key)",
-    re.compile(
-        r"/(?:id_rsa|id_dsa|id_ecdsa|id_ed25519|.*\.(?:pem|key|ppk|pfx|p12))",
-        re.IGNORECASE,
-    ),
-    severity=3,
-)
-
-# -- suspicious user-agent (optional, path-based only for now) -----------------
-
-
-# ---------------------------------------------------------------------------
-# All active patterns (ordered by severity, highest first)
-# ---------------------------------------------------------------------------
-
-ALL_PATTERNS: list[BadPattern] = [
-    ENV_FILE,
-    VCS_PROBE,
-    SENSITIVE_FILE,
-    WEBSHELL_PROBE,
-    WP_PROBE,
-    GENERIC_PHP_PROBE,
-    WELL_KNOWN_PROBE,
-]
-
-# Quick lookup by name
-PATTERNS_BY_NAME: dict[str, BadPattern] = {p.name: p for p in ALL_PATTERNS}
-
-
-# ---------------------------------------------------------------------------
-# Classification API
-# ---------------------------------------------------------------------------
-
-
-def classify_path(path: str) -> list[BadPattern]:
-    """Return all patterns that match a given request path."""
-    return [p for p in ALL_PATTERNS if p.matches(path)]
-
-
-def is_malicious(path: str) -> bool:
-    """Return True if any pattern matches the path."""
-    return any(p.matches(path) for p in ALL_PATTERNS)
-
-
-def malicious_severity(path: str) -> int:
-    """Return the highest severity among matching patterns, or 0 if none."""
-    matched = classify_path(path)
-    return max((p.severity for p in matched), default=0)
-
-
-def pattern_names(path: str) -> list[str]:
-    """Return names of all patterns matching the path."""
-    return [p.name for p in classify_path(path)]
+#!/usr/bin/env python3
+"""Malicious-pattern detection rules for the intrusion-detection system.
+
+Defines compiled regex patterns and classifiers that identify
+probing/scanning URLs (e.g. env files, wp-admin, webshells) and
+suspicious file names.  Imported by analyzers to flag IPs hitting
+these patterns as blocklist candidates.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import ClassVar, Pattern
+
+# ---------------------------------------------------------------------------
+# Pattern categories
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class BadPattern:
+    """A single detection rule.
+
+    Attributes:
+        name: Short identifier (e.g. "env_file", "wp_admin").
+        label: Human-readable description for reports.
+        regex: Compiled regex matched against the request path.
+        severity: 1 = moderate, 2 = high, 3 = critical (auto-block).
+    """
+
+    name: str
+    label: str
+    regex: Pattern[str]
+    severity: int = 2
+
+    def matches(self, path: str) -> bool:
+        return bool(self.regex.search(path))
+
+
+# ---------------------------------------------------------------------------
+# Compiled pattern bank
+# ---------------------------------------------------------------------------
+
+# -- env / config file probes -------------------------------------------------
+
+ENV_FILE = BadPattern(
+    "env_file",
+    "env/config file probe (.env, .aws/credentials, config.json)",
+    re.compile(
+        r"/\.(?:env|config)(?:\.\w+)*|"
+        r"/\.aws/credentials|"
+        r"/config\.json",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- version control probes ---------------------------------------------------
+
+VCS_PROBE = BadPattern(
+    "vcs_probe",
+    "version-control probe (.git, .svn)",
+    re.compile(
+        r"/\.(?:git|svn)(?:/|$)",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- WordPress / CMS probes ---------------------------------------------------
+
+WP_PROBE = BadPattern(
+    "wp_probe",
+    "WordPress/CMS probe (wp-admin, wp-login, wp-content, wp-includes)",
+    re.compile(
+        r"/wp-(?:admin|login|content|includes|load|cron|xmlrpc)(?:/|\.php)",
+        re.IGNORECASE,
+    ),
+    severity=2,
+)
+
+# -- well-known / common probe paths ------------------------------------------
+
+WELL_KNOWN_PROBE = BadPattern(
+    "well_known_probe",
+    ".well-known probe",
+    re.compile(
+        r"/\.well-known/",
+        re.IGNORECASE,
+    ),
+    severity=1,
+)
+
+# -- webshell / backdoor probes -----------------------------------------------
+
+WEBSHELL_PROBE = BadPattern(
+    "webshell_probe",
+    "webshell/backdoor probe (known shell filenames, phpinfo, file managers)",
+    re.compile(
+        r"/(?:"
+        r"phpinfo|info|shell|cmd|upload|"
+        r"wp_filemanager|filemanager|"
+        r"xstelth|adminfuns|hellopress|"
+        r"fpwch|crgio|coffexium|"
+        r"ccou|biufile|"
+        r"ms-edit|classwithtostring|"
+        r"this_is_a_new_hello_world|"
+        r"chosen|ops|simple|inputs"
+        r")\.php",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- suspicious generic PHP file requests -------------------------------------
+
+GENERIC_PHP_PROBE = BadPattern(
+    "generic_php_probe",
+    "generic/random PHP file probe (single-char, numeric names)",
+    re.compile(
+        r"/(?:"
+        r"[a-z]\.php|"  # single letter: a.php, k.php, x.php
+        r"\d{1,3}\.php|"  # numeric: 1.php, 222.php, 155.php
+        r"[a-z]{2}\.php|"  # two-letter: aa.php, dx.php
+        r"a\d\.php|"  # a1.php
+        r"\w+\.php"  # generic: anything.php
+        r")",
+        re.IGNORECASE,
+    ),
+    severity=1,
+)
+
+# -- sensitive file probes ----------------------------------------------------
+
+SENSITIVE_FILE = BadPattern(
+    "sensitive_file",
+    "sensitive file probe (id_rsa, id_dsa, .pem, .key)",
+    re.compile(
+        r"/(?:id_rsa|id_dsa|id_ecdsa|id_ed25519|.*\.(?:pem|key|ppk|pfx|p12))",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- suspicious user-agent (optional, path-based only for now) -----------------
+
+
+# ---------------------------------------------------------------------------
+# All active patterns (ordered by severity, highest first)
+# ---------------------------------------------------------------------------
+
+ALL_PATTERNS: list[BadPattern] = [
+    ENV_FILE,
+    VCS_PROBE,
+    SENSITIVE_FILE,
+    WEBSHELL_PROBE,
+    WP_PROBE,
+    GENERIC_PHP_PROBE,
+    WELL_KNOWN_PROBE,
+]
+
+# Quick lookup by name
+PATTERNS_BY_NAME: dict[str, BadPattern] = {p.name: p for p in ALL_PATTERNS}
+
+
+# ---------------------------------------------------------------------------
+# Classification API
+# ---------------------------------------------------------------------------
+
+
+def classify_path(path: str) -> list[BadPattern]:
+    """Return all patterns that match a given request path."""
+    return [p for p in ALL_PATTERNS if p.matches(path)]
+
+
+def is_malicious(path: str) -> bool:
+    """Return True if any pattern matches the path."""
+    return any(p.matches(path) for p in ALL_PATTERNS)
+
+
+def malicious_severity(path: str) -> int:
+    """Return the highest severity among matching patterns, or 0 if none."""
+    matched = classify_path(path)
+    return max((p.severity for p in matched), default=0)
+
+
+def pattern_names(path: str) -> list[str]:
+    """Return names of all patterns matching the path."""
+    return [p.name for p in classify_path(path)]

app/copyparty.py

@@ -1,182 +1,182 @@
-#!/usr/bin/env python3
-"""copyparty access-log analyzer.
-
-Parses copyparty server logs, aggregates per-IP behaviour and flags
-"denied-only" clients: IPs that never authenticated and only ever hit
-`@* has no access to '...'`. Those plus copyparty's own auto-bans become
-the MikroTik block list.
-"""
-
-from __future__ import annotations
-
-import os
-import re
-
-from .analyzer import Analyzer
-from .constants import (
-    COPYPARTY_LOG_PATH,
-    COPYPARTY_REPORT_DIR,
-    DEFAULT_MIN_DENIED,
-)
-from .ip_utils import split_versions
-from .files import write_json, write_lines
-from .flags import flag_denied_only
-from .models import IPStats, Request
-from .utils import arg_parser
-
-# "HH:MM:SS.mmm  <ip>  <port>  LEVEL: message"
-# System lines ("hsrv", "root", ...) have no port and are skipped.
-LINE_RE = re.compile(
-    r"^(\d{2}:\d{2}:\d{2}\.\d{3})\s+"                 # timestamp
-    r"(\S+)\s+"                                        # ip
-    r"(\d+)\s+"                                        # port
-    r"(LOG|WARN|CRIT|BTW|INFO|ERR|DEBUG):\s"           # level
-    r"(.*)$"                                           # message
-)
-
-METHOD_RE = re.compile(
-    r"^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS|PROPFIND|PROPPATCH|MKCOL|"
-    r"MOVE|COPY|LOCK|UNLOCK)\s+(\S+)(?:\s+@(\S+))?")
-DENY_RE = re.compile(r"^@(\S+) has no access to '([^']+)'")
-CLIENT_BAN_RE = re.compile(r"client banned")
-BAN_SECS_RE = re.compile(r"banned for (\d+) sec")
-
-
-class CopypartyAnalyzer(Analyzer):
-    name = "copyparty"
-    title = "copyparty log analysis report"
-    log_pattern = re.compile(r"\d{4}-\d{2}-\d{2}\.txt(\.\d+)?$")
-    paths_file = "denied_paths.csv"
-    paths_label = "denied paths"
-    flag_files = ("denied_ips.txt", "denied_ipv6.txt")
-    flag_label = "Denied-only IPs"
-    flag_note = "never authenticated -- ban candidates"
-
-    ip_columns = (
-        ("ip", lambda s: s.ip),
-        ("total", lambda s: s.total),
-        ("denied", lambda s: s.denied),
-        ("banned", lambda s: s.banned),
-        ("first", lambda s: s.first),
-        ("last", lambda s: s.last),
-        ("users", lambda s: s.users_str),
-        ("whitelist", lambda s: s.whitelist),
-    )
-
-    request_columns = (
-        ("time", lambda r: r.ts),
-        ("ip", lambda r: r.ip),
-        ("port", lambda r: r.extra.get("port")),
-        ("level", lambda r: r.extra.get("level")),
-        ("method", lambda r: r.method),
-        ("path", lambda r: r.path),
-        ("user", lambda r: r.user),
-        ("denied", lambda r: r.denied),
-        ("banned", lambda r: r.banned),
-        ("ban_secs", lambda r: r.extra.get("ban_secs")),
-    )
-
-    def __init__(self, *args, min_denied: int = DEFAULT_MIN_DENIED, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.min_denied = min_denied
-
-    # --- parsing -----------------------------------------------------------
-
-    def parse_line(self, line: str) -> Request | None:
-        if not line:
-            return None
-        m = LINE_RE.match(line)
-        if not m:
-            return None
-        ts, ip, port, level, msg = m.groups()
-        req = Request(ts=ts, ip=ip, user=None,
-                      extra={"port": int(port), "level": level})
-
-        mm = METHOD_RE.match(msg)
-        if mm:
-            req.method, req.path, user = mm.groups()
-            req.user = user or "*"
-            return req
-
-        dm = DENY_RE.match(msg)
-        if dm:
-            req.user, req.path = dm.groups()
-            req.denied = True
-            return req
-
-        if CLIENT_BAN_RE.search(msg):
-            req.banned = True
-            bs = BAN_SECS_RE.search(msg)
-            if bs:
-                req.extra["ban_secs"] = int(bs.group(1))
-            return req
-
-        bs = BAN_SECS_RE.search(msg)
-        if bs:
-            req.banned = True
-            req.extra["ban_secs"] = int(bs.group(1))
-        return req
-
-    # --- flagging ----------------------------------------------------------
-
-    def flag(self, stats):
-        return flag_denied_only(stats, min_denied=self.min_denied)
-
-    def flag_comment(self, s):
-        return (f"denied={s.denied} total={s.total} "
-                f"first={s.first} last={s.last}")
-
-    def block_candidates(self) -> set[str]:
-        banned = {ip for ip, s in self.stats.items() if s.banned}
-        return set(self.flagged) | banned
-
-    # --- extras ------------------------------------------------------------
-
-    def banned_ips(self) -> dict[str, IPStats]:
-        return {ip: s for ip, s in self.stats.items() if s.banned}
-
-    def authed_users(self) -> dict[str, int]:
-        counts: dict[str, int] = {}
-        for r in self.requests:
-            if r.user and r.user != "*":
-                counts[r.user] = counts.get(r.user, 0) + 1
-        return dict(sorted(counts.items(), key=lambda kv: -kv[1]))
-
-    def extra_summary_rows(self):
-        return [("denied requests",
-                 sum(1 for r in self.requests if r.denied)),
-                ("banned IPs", len(self.banned_ips())),
-                ("authenticated users", self.authed_users() or "{}")]
-
-    def write_extra(self, data_dir: str) -> list[str]:
-        v4, v6 = split_versions(self.banned_ips())
-        return [
-            write_lines(os.path.join(data_dir, "banned_ips.txt"), v4),
-            write_lines(os.path.join(data_dir, "banned_ipv6.txt"), v6),
-            write_json(os.path.join(data_dir, "authed_users.json"),
-                       self.authed_users()),
-        ]
-
-
-def main() -> None:
-    parser = arg_parser(
-        "analyze copyparty logs, flag denied-only IPs, export MikroTik rules",
-        COPYPARTY_LOG_PATH, COPYPARTY_REPORT_DIR)
-    parser.add_argument("--min-denied", type=int, default=DEFAULT_MIN_DENIED,
-                        help=f"minimum denied hits before flagging (default: {DEFAULT_MIN_DENIED})")
-    parser.add_argument("--full", action="store_true",
-                        help="rescan all logs from the start (ignore incremental state)")
-    args = parser.parse_args()
-
-    CopypartyAnalyzer(
-        log_path=args.log_path,
-        report_dir=args.report_dir,
-        top=args.top,
-        quiet=args.quiet,
-        full=args.full,
-        min_denied=args.min_denied,
-    ).run()
-
-
-if __name__ == "__main__":
-    main()
+#!/usr/bin/env python3
+"""copyparty access-log analyzer.
+
+Parses copyparty server logs, aggregates per-IP behaviour and flags
+"denied-only" clients: IPs that never authenticated and only ever hit
+`@* has no access to '...'`. Those plus copyparty's own auto-bans become
+the MikroTik block list.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+from .analyzer import Analyzer
+from .utils.constants import (
+    COPYPARTY_LOG_PATH,
+    COPYPARTY_REPORT_DIR,
+    DEFAULT_MIN_DENIED,
+)
+from .utils.ip_utils import split_versions
+from .utils.files import write_json, write_lines
+from .flags import flag_denied_only
+from .utils.models import IPStats, Request
+from .utils.utils import arg_parser
+
+# "HH:MM:SS.mmm  <ip>  <port>  LEVEL: message"
+# System lines ("hsrv", "root", ...) have no port and are skipped.
+LINE_RE = re.compile(
+    r"^(\d{2}:\d{2}:\d{2}\.\d{3})\s+"                 # timestamp
+    r"(\S+)\s+"                                        # ip
+    r"(\d+)\s+"                                        # port
+    r"(LOG|WARN|CRIT|BTW|INFO|ERR|DEBUG):\s"           # level
+    r"(.*)$"                                           # message
+)
+
+METHOD_RE = re.compile(
+    r"^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS|PROPFIND|PROPPATCH|MKCOL|"
+    r"MOVE|COPY|LOCK|UNLOCK)\s+(\S+)(?:\s+@(\S+))?")
+DENY_RE = re.compile(r"^@(\S+) has no access to '([^']+)'")
+CLIENT_BAN_RE = re.compile(r"client banned")
+BAN_SECS_RE = re.compile(r"banned for (\d+) sec")
+
+
+class CopypartyAnalyzer(Analyzer):
+    name = "copyparty"
+    title = "copyparty log analysis report"
+    log_pattern = re.compile(r"\d{4}-\d{2}-\d{2}\.txt(\.\d+)?$")
+    paths_file = "denied_paths.csv"
+    paths_label = "denied paths"
+    flag_files = ("denied_ips.txt", "denied_ipv6.txt")
+    flag_label = "Denied-only IPs"
+    flag_note = "never authenticated -- ban candidates"
+
+    ip_columns = (
+        ("ip", lambda s: s.ip),
+        ("total", lambda s: s.total),
+        ("denied", lambda s: s.denied),
+        ("banned", lambda s: s.banned),
+        ("first", lambda s: s.first),
+        ("last", lambda s: s.last),
+        ("users", lambda s: s.users_str),
+        ("whitelist", lambda s: s.whitelist),
+    )
+
+    request_columns = (
+        ("time", lambda r: r.ts),
+        ("ip", lambda r: r.ip),
+        ("port", lambda r: r.extra.get("port")),
+        ("level", lambda r: r.extra.get("level")),
+        ("method", lambda r: r.method),
+        ("path", lambda r: r.path),
+        ("user", lambda r: r.user),
+        ("denied", lambda r: r.denied),
+        ("banned", lambda r: r.banned),
+        ("ban_secs", lambda r: r.extra.get("ban_secs")),
+    )
+
+    def __init__(self, *args, min_denied: int = DEFAULT_MIN_DENIED, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.min_denied = min_denied
+
+    # --- parsing -----------------------------------------------------------
+
+    def parse_line(self, line: str) -> Request | None:
+        if not line:
+            return None
+        m = LINE_RE.match(line)
+        if not m:
+            return None
+        ts, ip, port, level, msg = m.groups()
+        req = Request(ts=ts, ip=ip, user=None,
+                      extra={"port": int(port), "level": level})
+
+        mm = METHOD_RE.match(msg)
+        if mm:
+            req.method, req.path, user = mm.groups()
+            req.user = user or "*"
+            return req
+
+        dm = DENY_RE.match(msg)
+        if dm:
+            req.user, req.path = dm.groups()
+            req.denied = True
+            return req
+
+        if CLIENT_BAN_RE.search(msg):
+            req.banned = True
+            bs = BAN_SECS_RE.search(msg)
+            if bs:
+                req.extra["ban_secs"] = int(bs.group(1))
+            return req
+
+        bs = BAN_SECS_RE.search(msg)
+        if bs:
+            req.banned = True
+            req.extra["ban_secs"] = int(bs.group(1))
+        return req
+
+    # --- flagging ----------------------------------------------------------
+
+    def flag(self, stats):
+        return flag_denied_only(stats, min_denied=self.min_denied)
+
+    def flag_comment(self, s):
+        return (f"denied={s.denied} total={s.total} "
+                f"first={s.first} last={s.last}")
+
+    def block_candidates(self) -> set[str]:
+        banned = {ip for ip, s in self.stats.items() if s.banned}
+        return set(self.flagged) | banned
+
+    # --- extras ------------------------------------------------------------
+
+    def banned_ips(self) -> dict[str, IPStats]:
+        return {ip: s for ip, s in self.stats.items() if s.banned}
+
+    def authed_users(self) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for r in self.requests:
+            if r.user and r.user != "*":
+                counts[r.user] = counts.get(r.user, 0) + 1
+        return dict(sorted(counts.items(), key=lambda kv: -kv[1]))
+
+    def extra_summary_rows(self):
+        return [("denied requests",
+                 sum(1 for r in self.requests if r.denied)),
+                ("banned IPs", len(self.banned_ips())),
+                ("authenticated users", self.authed_users() or "{}")]
+
+    def write_extra(self, data_dir: str) -> list[str]:
+        v4, v6 = split_versions(self.banned_ips())
+        return [
+            write_lines(os.path.join(data_dir, "banned_ips.txt"), v4),
+            write_lines(os.path.join(data_dir, "banned_ipv6.txt"), v6),
+            write_json(os.path.join(data_dir, "authed_users.json"),
+                       self.authed_users()),
+        ]
+
+
+def main() -> None:
+    parser = arg_parser(
+        "analyze copyparty logs, flag denied-only IPs, export MikroTik rules",
+        COPYPARTY_LOG_PATH, COPYPARTY_REPORT_DIR)
+    parser.add_argument("--min-denied", type=int, default=DEFAULT_MIN_DENIED,
+                        help=f"minimum denied hits before flagging (default: {DEFAULT_MIN_DENIED})")
+    parser.add_argument("--full", action="store_true",
+                        help="rescan all logs from the start (ignore incremental state)")
+    args = parser.parse_args()
+
+    CopypartyAnalyzer(
+        log_path=args.log_path,
+        report_dir=args.report_dir,
+        top=args.top,
+        quiet=args.quiet,
+        full=args.full,
+        min_denied=args.min_denied,
+    ).run()
+
+
+if __name__ == "__main__":
+    main()

app/data/paloalto/scanner-ips.txt

@@ -1,12 +1,12 @@
-35.203.210.0/23
-144.86.173.0/24
-147.185.132.0/23
-162.216.149.0/24
-162.216.150.0/24
-172.105.147.0/24
-198.235.24.0/24
-205.210.31.0/24
-216.25.88.0/21
-2604:a940:300:5b6:0:0:0:0/64 
-2604:a940:301:225:0:0:0:0/64 
+35.203.210.0/23
+144.86.173.0/24
+147.185.132.0/23
+162.216.149.0/24
+162.216.150.0/24
+172.105.147.0/24
+198.235.24.0/24
+205.210.31.0/24
+216.25.88.0/21
+2604:a940:300:5b6:0:0:0:0/64 
+2604:a940:301:225:0:0:0:0/64 
 2604:a940:302:118:0:0:0:0/64

app/db/__init__.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+"""Local SQLite database layer for the intrusion-detection system.
+
+Owns the shared connection factory (WAL mode, busy timeout, single mutable
+connection reused across the background-scan thread and web readers) and
+applies schema migrations on open. Query helpers live in :mod:`repositories`.
+"""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+import threading
+
+from ..utils.constants import DATA_DB_PATH
+from .schema import apply_migrations
+
+_connection: sqlite3.Connection | None = None
+_lock = threading.RLock()
+
+
+def connect(path: str | None = None) -> sqlite3.Connection:
+    """Open an initialized SQLite connection at ``path``.
+
+    Enables WAL for concurrent readers during the single-writer background
+    scan and sets a busy timeout so short write contention does not raise.
+    Foreign keys are on and migrations are applied.
+    """
+    db_path = path or DATA_DB_PATH
+    os.makedirs(os.path.dirname(os.path.abspath(db_path)) or ".", exist_ok=True)
+    conn = sqlite3.connect(
+        db_path,
+        check_same_thread=False,  # background scan thread + web handlers
+        isolation_level=None,     # autocommit; use explicit BEGIN for batches
+    )
+    conn.execute("PRAGMA journal_mode=WAL")
+    conn.execute("PRAGMA busy_timeout=5000")
+    conn.execute("PRAGMA synchronous=NORMAL")
+    conn.row_factory = sqlite3.Row
+    apply_migrations(conn)
+    return conn
+
+
+def get_connection() -> sqlite3.Connection:
+    """Return the shared, lazily-created connection (thread-safe)."""
+    global _connection
+    with _lock:
+        if _connection is None:
+            _connection = connect()
+        return _connection
+
+
+def close() -> None:
+    """Close the shared connection, if any."""
+    global _connection
+    with _lock:
+        if _connection is not None:
+            try:
+                _connection.close()
+            except sqlite3.Error:
+                pass
+            _connection = None

app/db/__main__.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""CLI entry point for the SQLite store.
+
+Usage:
+    python -m app.db --migrate        applies pending migrations (idempotent)
+    python -m app.db --info           prints schema version + table counts, exits
+    python -m app.db --path           prints the resolved DB path, exits
+
+Used by ``deploy.sh`` (D004) to ensure the DB is up to date after a ``git pull``
+without needing the web server up.
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+
+from ..utils.constants import DATA_DB_PATH
+from .schema import SCHEMA_VERSION, apply_migrations
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(
+        prog="python -m app.db",
+        description="Intrusion-detection SQLite store utilities.")
+    parser.add_argument("--migrate", action="store_true",
+                        help="apply pending schema migrations (idempotent)")
+    parser.add_argument("--info", action="store_true",
+                        help="print schema version + table/row counts")
+    parser.add_argument("--path", action="store_true",
+                        help="print the resolved DB path and exit")
+    args = parser.parse_args(argv)
+
+    if args.path:
+        print(DATA_DB_PATH)
+        return 0
+
+    from . import get_connection
+
+    conn = get_connection()
+    current = conn.execute("PRAGMA user_version").fetchone()[0]
+
+    if args.info:
+        counts = {}
+        tables = conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table' "
+            "AND name NOT LIKE 'sqlite_%' ORDER BY name").fetchall()
+        for t in tables:
+            name = t["name"]
+            try:
+                counts[name] = conn.execute(
+                    f"SELECT COUNT(*) FROM \"{name}\"").fetchone()[0]
+            except Exception:
+                counts[name] = None
+        print(f"db_path:  {DATA_DB_PATH}")
+        print(f"version:  {current} (latest {SCHEMA_VERSION})")
+        print("tables:")
+        for name, n in counts.items():
+            print(f"  {name:<32} {n} rows")
+        return 0
+
+    if args.migrate or current < SCHEMA_VERSION:
+        before = current
+        apply_migrations(conn)
+        print(f"schema version {before} -> "
+              f"{conn.execute('PRAGMA user_version').fetchone()[0]}")
+        return 0
+
+    print(f"schema is already up to date (version {current})")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

app/db/backfill.py

@@ -0,0 +1,361 @@
+#!/usr/bin/env python3
+"""R001 — one-time backfill of legacy runtime data into the SQLite store.
+
+Imports existing JSON/CSV/git-history data so the DB becomes the authoritative
+source without losing anything already accumulated on a deployed instance:
+
+- cumulative blocklist from the git-history reconstruction (old source of
+  truth) -> ``blocklist_entries`` (grow-only ledger)
+- firewall snapshots from ``data/mikrotik/*.json`` -> ``firewall_snapshots``
+- per-source requests from ``reports/<source>/data/requests.csv`` (best effort)
+
+Idempotent: skips rows already present. Safe to run repeatedly.
+
+Usage:
+    python -m app.db.backfill [--dry-run]
+"""
+
+from __future__ import annotations
+
+import argparse
+import csv
+import hashlib
+import json
+import os
+import re
+import sys
+from datetime import datetime, timezone
+
+from ..utils.constants import (
+    APP_REPORT_DIR,
+    APP_REPORT_DATA_DIR,
+    DATA_DIR,
+    MIKROTIK_DATA_DIR,
+    MIKROTIK_REPORT_DIR,
+    NPM_REPORT_DIR,
+    COPYPARTY_REPORT_DIR,
+)
+
+MAX_REPORT_ARTIFACT_BYTES = 10 * 1024 * 1024
+CANONICAL_REPORT_EXPORTS = {
+    "requests.csv", "ips.json", "ips_ipv4.csv", "ips_ipv6.csv",
+    "blocked_ipv4.txt", "blocked_ipv6.txt", "address_list_ipv4.json",
+    "address_list_ipv6.json", "firewall_ipv4.json", "firewall_ipv6.json",
+}
+
+
+def utcnow() -> str:
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+def _read_cidr_lines(path: str) -> set[str]:
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return {line.strip() for line in f if line.strip()}
+    except OSError:
+        return set()
+
+
+def backfill_blocklist(conn, dry_run: bool = False) -> dict:
+    """Import the legacy cumulative blocklist into the ledger."""
+    from . import repositories as repo
+    from ..utils.blocklist_history import (
+        committed_lines, current_file_lines,
+    )
+
+    v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
+    v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
+    # Legacy source of truth: union of all git revisions + working file +
+    # (there is no earlier scan target to add here).
+    complete_v4 = committed_lines(v4_path) | current_file_lines(v4_path)
+    complete_v6 = committed_lines(v6_path) | current_file_lines(v6_path)
+    cidrs = sorted(complete_v4 | complete_v6)
+    if not cidrs:
+        return {"blocklist": {"cidrs": 0, "added": 0}}
+    if dry_run:
+        return {"blocklist": {"cidrs": len(cidrs), "added": None, "dry_run": True}}
+    result = repo.record_blocklist(conn, cidrs, added_by="backfill")
+    return {"blocklist": result}
+
+
+def backfill_firewall(conn, dry_run: bool = False) -> dict:
+    """Import the latest firewall snapshot from ``data/mikrotik/*.json``."""
+    from . import repositories as repo
+
+    def _load(name: str) -> list:
+        try:
+            with open(os.path.join(MIKROTIK_DATA_DIR, name), "r",
+                      encoding="utf-8") as f:
+                data = json.load(f)
+            return data if isinstance(data, list) else []
+        except (OSError, ValueError):
+            return []
+
+    addr_v4 = _load("address_list_ipv4.json")
+    addr_v6 = _load("address_list_ipv6.json")
+    fw_v4 = _load("firewall_ipv4.json")
+    fw_v6 = _load("firewall_ipv6.json")
+    if not (addr_v4 or addr_v6 or fw_v4 or fw_v6):
+        return {"firewall": {"dry_run": dry_run, "snapshot": None}}
+    # Idempotent: if a firewall snapshot already exists (previous backfill),
+    # don't create another copy of the same latest state.
+    n = conn.execute(
+        "SELECT COUNT(*) FROM firewall_snapshots").fetchone()[0]
+    if n:
+        return {"firewall": {"dry_run": dry_run, "snapshot": "exists",
+                             "skipped": True}}
+    if dry_run:
+        return {"firewall": {"dry_run": True, "snapshot": None}}
+    sid = repo.save_firewall_snapshot(
+        conn, address_lists_v4=addr_v4, address_lists_v6=addr_v6,
+        rules_v4=fw_v4, rules_v6=fw_v6)
+    return {"firewall": {"snapshot": sid}}
+
+
+def backfill_requests(conn, dry_run: bool = False) -> dict:
+    """Best-effort import of per-source ``requests.csv`` files.
+
+    Only ingests rows that don't already exist (keyed on source+log_file+ts+ip).
+    Legacy CSV columns vary by source; we keep a conservative mapping and never
+    fail the whole backfill over a single bad file.
+    """
+    from . import repositories as repo
+
+    seen = set()
+    for r in conn.execute(
+            "SELECT source, log_file, ts, ip FROM requests"):
+        seen.add((r["source"], r["log_file"], r["ts"], r["ip"]))
+
+    candidates = [
+        (os.path.join(COPYPARTY_REPORT_DIR, "data", "requests.csv"), "copyparty"),
+        (os.path.join(NPM_REPORT_DIR, "data", "requests.csv"), "npm"),
+    ]
+    total = 0
+    for path, source in candidates:
+        if not os.path.exists(path):
+            continue
+        try:
+            with open(path, "r", encoding="utf-8", errors="replace") as f:
+                reader = csv.DictReader(f)
+                rows = []
+                for row in reader:
+                    ip = (row.get("ip") or "").strip()
+                    ts = row.get("ts") or row.get("timestamp") or ""
+                    log_file = row.get("log_file") or row.get("file") or ""
+                    if not ip:
+                        continue
+                    key = (source, log_file, ts, ip)
+                    if key in seen:
+                        continue
+                    seen.add(key)
+                    rows.append({
+                        "source": source, "scan_id": None, "log_file": log_file,
+                        "ts": ts, "ip": ip,
+                        "version": 4 if ":" not in ip else 6,
+                        "method": row.get("method"), "path": row.get("path"),
+                        "status": _int_or_none(row.get("status")),
+                        "upstream_status": _int_or_none(
+                            row.get("upstream_status")),
+                        "host": row.get("host"), "user": row.get("user"),
+                        "ua": row.get("ua") or row.get("user_agent"),
+                        "referer": row.get("referer"),
+                        "denied": _bool_str(row.get("denied")),
+                        "banned": _bool_str(row.get("banned")),
+                        "is_error": _bool_str(row.get("is_error")),
+                        "extra": None,
+                    })
+                    total += 1
+            if not dry_run and total:
+                repo.insert_requests(conn, rows)
+        except Exception as exc:
+            print(f"  WARN: requests.csv backfill {source}: {exc}")
+    return {"requests": {"imported": total, "dry_run": dry_run}}
+
+
+def backfill_report_artifacts(conn, dry_run: bool = False,
+                              roots: dict[str, str] | None = None) -> dict:
+    """Import supported report files as versioned, hash-addressed artifacts.
+
+    Canonical event exports and current firewall/blocklist data are skipped;
+    those belong in their normalized tables. Original files are never changed
+    or removed. Oversized or unreadable files are reported for manual review.
+    """
+    from . import repositories as repo
+
+    roots = roots or {
+        "copyparty": COPYPARTY_REPORT_DIR,
+        "npm": NPM_REPORT_DIR,
+        "app": APP_REPORT_DIR,
+        "mikrotik": MIKROTIK_REPORT_DIR,
+    }
+    result = {
+        "discovered": 0, "imported": 0, "would_import": 0,
+        "already_imported": 0, "skipped_canonical": [], "unmapped": [],
+        "by_source": {}, "by_format": {}, "dry_run": dry_run,
+    }
+
+    def count(bucket: str, key: str, field: str) -> None:
+        stats = result[bucket].setdefault(key, {})
+        stats[field] = stats.get(field, 0) + 1
+
+    for source, root in roots.items():
+        if not os.path.isdir(root):
+            continue
+        for directory, _, filenames in os.walk(root):
+            for filename in sorted(filenames):
+                path = os.path.join(directory, filename)
+                suffix = os.path.splitext(filename)[1].lower()
+                if suffix not in (".txt", ".json", ".csv", ".rsc"):
+                    continue
+                relative = os.path.relpath(path, root).replace(os.sep, "/")
+                result["discovered"] += 1
+                report_format = suffix.lstrip(".")
+                count("by_source", source, "discovered")
+                count("by_format", report_format, "discovered")
+                if filename.lower() in CANONICAL_REPORT_EXPORTS:
+                    result["skipped_canonical"].append(
+                        {"source": source, "path": relative})
+                    count("by_source", source, "skipped_canonical")
+                    count("by_format", report_format, "skipped_canonical")
+                    continue
+
+                try:
+                    size = os.path.getsize(path)
+                    if size > MAX_REPORT_ARTIFACT_BYTES:
+                        raise ValueError(
+                            f"file exceeds {MAX_REPORT_ARTIFACT_BYTES} bytes")
+                    with open(path, "rb") as file:
+                        raw = file.read()
+                    text = raw.decode("utf-8", errors="replace")
+                    payload = _report_payload(suffix, text)
+                    digest = hashlib.sha256(raw).hexdigest()
+                    modified = datetime.fromtimestamp(
+                        os.path.getmtime(path), timezone.utc)
+                    generated_at = _report_generated_at(filename, modified)
+                except (OSError, ValueError) as exc:
+                    result["unmapped"].append(
+                        {"source": source, "path": relative,
+                         "reason": str(exc)})
+                    count("by_source", source, "unmapped")
+                    count("by_format", report_format, "unmapped")
+                    continue
+
+                original_path = f"{source}/{relative}"
+                exists = conn.execute(
+                    "SELECT 1 FROM report_artifacts "
+                    "WHERE original_path=? AND content_sha256=?",
+                    (original_path, digest),
+                ).fetchone()
+                if exists:
+                    result["already_imported"] += 1
+                    count("by_source", source, "already_imported")
+                    count("by_format", report_format, "already_imported")
+                    continue
+                if dry_run:
+                    result["would_import"] += 1
+                    count("by_source", source, "would_import")
+                    count("by_format", report_format, "would_import")
+                    continue
+
+                try:
+                    repo.insert_report_artifact(
+                        conn, source=source,
+                        report_type=_report_type(suffix),
+                        generated_at=generated_at, payload=payload,
+                        original_path=original_path,
+                        content_sha256=digest,
+                    )
+                    result["imported"] += 1
+                    count("by_source", source, "imported")
+                    count("by_format", report_format, "imported")
+                except (OSError, ValueError, TypeError) as exc:
+                    result["unmapped"].append(
+                        {"source": source, "path": relative,
+                         "reason": str(exc)})
+                    count("by_source", source, "unmapped")
+                    count("by_format", report_format, "unmapped")
+    return {"reports": result}
+
+
+def _report_payload(suffix: str, text: str) -> dict:
+    if suffix == ".json":
+        try:
+            return {"format": "json", "data": json.loads(text)}
+        except ValueError as exc:
+            return {"format": "json", "content": text,
+                    "parse_error": str(exc)}
+    if suffix == ".csv":
+        try:
+            reader = csv.DictReader(text.splitlines())
+            return {"format": "csv", "columns": reader.fieldnames or [],
+                    "rows": list(reader)}
+        except csv.Error as exc:
+            return {"format": "csv", "content": text,
+                    "parse_error": str(exc)}
+    return {"format": "text", "content": text}
+
+
+def _report_type(suffix: str) -> str:
+    return {".json": "json_report", ".csv": "csv_report",
+            ".rsc": "routeros_script"}.get(suffix, "text_report")
+
+
+def _report_generated_at(filename: str, modified: datetime) -> str:
+    match = re.search(
+        r"(?P<date>\d{4}-\d{2}-\d{2})(?:[-_T](?P<hour>\d{2})[-_:]"
+        r"(?P<minute>\d{2})[-_:](?P<second>\d{2}))?", filename)
+    if not match:
+        return modified.isoformat(timespec="seconds")
+    values = match.groupdict()
+    hour = int(values["hour"] or 0)
+    minute = int(values["minute"] or 0)
+    second = int(values["second"] or 0)
+    try:
+        parsed = datetime.strptime(values["date"], "%Y-%m-%d").replace(
+            hour=hour, minute=minute, second=second, tzinfo=timezone.utc)
+    except ValueError:
+        return modified.isoformat(timespec="seconds")
+    return parsed.isoformat(timespec="seconds")
+
+
+def _int_or_none(v):
+    try:
+        if v in (None, "", "-"):
+            return None
+        return int(v)
+    except (TypeError, ValueError):
+        return None
+
+
+def _bool_str(v):
+    if isinstance(v, str):
+        return int(v.strip().lower() in ("1", "true", "yes", "t"))
+    return int(bool(v))
+
+
+def main(argv: list[str] | None = None) -> None:
+    p = argparse.ArgumentParser(prog="python -m app.db.backfill",
+                                description="Backfill legacy runtime data into SQLite.")
+    p.add_argument("--dry-run", action="store_true",
+                   help="report what would be imported without writing")
+    p.add_argument("--reports-only", action="store_true",
+                   help="import legacy report artifacts only")
+    opts = p.parse_args(argv)
+
+    from . import get_connection
+    conn = get_connection()
+    print(f"== backfill (dry_run={opts.dry_run}) ==")
+    if opts.reports_only:
+        print("reports :", backfill_report_artifacts(
+            conn, dry_run=opts.dry_run))
+        print("== done ==")
+        return
+    print("blocklist:", backfill_blocklist(conn, dry_run=opts.dry_run))
+    print("firewall :", backfill_firewall(conn, dry_run=opts.dry_run))
+    print("requests :", backfill_requests(conn, dry_run=opts.dry_run))
+    print("reports  :", backfill_report_artifacts(conn, dry_run=opts.dry_run))
+    print("== done ==")
+
+
+if __name__ == "__main__":
+    sys.exit(main())

app/db/repositories.py

@@ -0,0 +1,690 @@
+#!/usr/bin/env python3
+"""Query helpers for the intrusion-detection SQLite store.
+
+Functions are grouped by domain and take an explicit ``sqlite3.Connection`` so
+callers control transaction boundaries. Bulk inserts are batched with
+``executemany`` inside a single transaction for speed.
+"""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+import base64
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+import ipaddress
+from typing import Any, Iterator, Sequence
+
+from ..utils.models import IPStats
+
+
+def utcnow() -> str:
+    """ISO-8601 timestamp with seconds precision (UTC)."""
+    return datetime.now(timezone.utc).isoformat(timespec="seconds")
+
+
+@contextmanager
+def transaction(conn: sqlite3.Connection) -> Iterator[sqlite3.Connection]:
+    """Context manager for an explicit transaction (commit or rollback)."""
+    conn.execute("BEGIN")
+    try:
+        yield conn
+    except Exception:
+        conn.execute("ROLLBACK")
+        raise
+    else:
+        conn.execute("COMMIT")
+
+
+# ---------------------------------------------------------------------------
+# Scans
+# ---------------------------------------------------------------------------
+
+def start_scan(conn: sqlite3.Connection, kind: str, *,
+               params: dict[str, Any] | None = None,
+               full: bool = False, dry_run: bool = False) -> int:
+    """Record a scan start and return its id."""
+    cur = conn.execute(
+        "INSERT INTO scans (kind, started_at, status, full, dry_run, params_json) "
+        "VALUES (?, ?, 'running', ?, ?, ?)",
+        (kind, utcnow(), int(full), int(dry_run),
+         json.dumps(params) if params is not None else None))
+    return int(cur.lastrowid)
+
+
+def finish_scan(conn: sqlite3.Connection, scan_id: int, *,
+                status: str, result: dict[str, Any] | None = None,
+                log_files: int = 0, request_count: int = 0,
+                error: str | None = None) -> None:
+    """Mark a scan complete with outcome counters."""
+    conn.execute(
+        "UPDATE scans SET finished_at=?, status=?, result_json=?, "
+        "log_files=?, request_count=?, error=? WHERE id=?",
+        (utcnow(), status,
+         json.dumps(result) if result is not None else None,
+         log_files, request_count, error, scan_id))
+
+
+def set_scan_source_counts(conn: sqlite3.Connection, scan_id: int,
+                           source: str, requests: int, files: int) -> None:
+    """Record per-source request/file counts for a scan."""
+    conn.execute(
+        "INSERT INTO scans_sources (scan_id, source, requests, files) "
+        "VALUES (?, ?, ?, ?) "
+        "ON CONFLICT(scan_id, source) DO UPDATE SET "
+        "requests=excluded.requests, files=excluded.files",
+        (scan_id, source, requests, files))
+
+
+def latest_scans(conn: sqlite3.Connection, limit: int = 10) -> list[dict[str, Any]]:
+    """Most recent scan metadata rows."""
+    rows = conn.execute(
+        "SELECT id, kind, started_at, finished_at, status, full, dry_run, "
+        "log_files, request_count, error FROM scans "
+        "ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def get_scan(conn: sqlite3.Connection, scan_id: int) -> dict[str, Any] | None:
+    row = conn.execute(
+        "SELECT id, kind, started_at, finished_at, status, full, dry_run, "
+        "params_json, result_json, log_files, request_count, error "
+        "FROM scans WHERE id=?", (scan_id,)).fetchone()
+    return dict(row) if row is not None else None
+
+
+# ---------------------------------------------------------------------------
+# Logs + raw requests
+# ---------------------------------------------------------------------------
+
+def _parse_ts(ts: str) -> tuple[str, float | None]:
+    """Best-effort ISO timestamp extraction from a log line timestamp string."""
+    if not ts:
+        return (ts, None)
+    epoch = None
+    cleaned = ts.strip()
+    # Accept a trailing timezone like +HH:MM or Z; fall back to naive parse.
+    try:
+        epoch = datetime.fromisoformat(
+            cleaned.replace("Z", "+00:00")).timestamp()
+    except ValueError:
+        try:
+            epoch = datetime.fromisoformat(cleaned).timestamp()
+        except ValueError:
+            epoch = None
+    return (ts, epoch)
+
+
+def insert_requests(conn: sqlite3.Connection, rows: Sequence[dict[str, Any]]) -> int:
+    """Bulk-insert ``requests`` rows. Returns inserted count.
+
+    Each ``row`` maps keys: source, scan_id, log_file, ts, ip, version,
+    method, path, status, upstream_status, host, user, ua, referer, denied,
+    banned, is_error, malicious_hits, extra(dict|None) — matching
+    :mod:`app.utils.models.Request` plus provenance fields.
+    """
+    if not rows:
+        return 0
+    created = utcnow()
+    tuples: list[tuple] = []
+    for r in rows:
+        ts, epoch = _parse_ts(r.get("ts") or "")
+        extra = r.get("extra")
+        tuples.append((
+            r.get("source"), r.get("scan_id"), r.get("log_file"), ts, epoch,
+            r.get("ip"), r.get("version"),
+            r.get("method"), r.get("path"),
+            r.get("status"), r.get("upstream_status"),
+            r.get("host"), r.get("user"), r.get("ua"), r.get("referer"),
+            int(bool(r.get("denied"))), int(bool(r.get("banned"))),
+            int(bool(r.get("is_error"))),
+            int(r.get("malicious_hits") or 0),
+            json.dumps(extra) if extra is not None else None,
+            created,
+        ))
+    with transaction(conn):
+        conn.executemany(
+            "INSERT INTO requests (source, scan_id, log_file, ts, ts_epoch, ip, "
+            "version, method, path, status, upstream_status, host, user, ua, "
+            "referer, denied, banned, is_error, malicious_hits, extra, created_at) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            tuples)
+    return len(tuples)
+
+
+def query_requests(conn: sqlite3.Connection, *,
+                   ip: str | None = None, source: str | None = None,
+                   since: str | None = None, until: str | None = None,
+                   path: str | None = None, limit: int = 500) -> list[dict[str, Any]]:
+    """Return matching request rows (optionally filtered), newest last."""
+    clauses: list[str] = []
+    args: list[Any] = []
+    if ip:
+        clauses.append("ip = ?")
+        args.append(ip)
+    if source:
+        clauses.append("source = ?")
+        args.append(source)
+    if since:
+        clauses.append("COALESCE(ts, '') >= ?")
+        args.append(since)
+    if until:
+        clauses.append("COALESCE(ts, '') <= ?")
+        args.append(until)
+    if path:
+        clauses.append("path LIKE ?")
+        args.append(path)
+    sql = "SELECT * FROM requests"
+    if clauses:
+        sql += " WHERE " + " AND ".join(clauses)
+    sql += " ORDER BY id LIMIT ?"
+    args.append(int(limit))
+    return [dict(r) for r in conn.execute(sql, args).fetchall()]
+
+
+# ---------------------------------------------------------------------------
+# IP aggregates
+# ---------------------------------------------------------------------------
+
+def insert_ip_stats(conn: sqlite3.Connection, scan_id: int,
+                    stats: Sequence[IPStats],
+                    blocked: set[str]) -> int:
+    """Persist per-IP aggregates for a scan. ``blocked`` are direct blocked IPs."""
+    if not stats:
+        return 0
+    tuples: list[tuple] = []
+    for s in stats:
+        tuples.append((
+            scan_id, s.ip, s.version, s.total, s.errors, s.c4xx, s.c5xx,
+            s.denied, s.banned, s.first, s.last, s.malicious_hits,
+            json.dumps(dict(s.paths)), json.dumps(dict(s.methods)),
+            json.dumps(dict(s.hosts)), json.dumps(dict(s.uas)),
+            json.dumps(sorted(s.users)), s.whitelist or None,
+            int(s.ip in blocked),
+        ))
+    with transaction(conn):
+        conn.executemany(
+            "INSERT INTO ip_stats (scan_id, ip, version, total, errors, c4xx, "
+            "c5xx, denied, banned, first_ts, last_ts, malicious_hits, paths_json, "
+            "methods_json, hosts_json, uas_json, users_json, whitelist, blocked) "
+            "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)",
+            tuples)
+    return len(tuples)
+
+
+def ip_timeline(conn: sqlite3.Connection, ip: str,
+                limit: int = 2000) -> list[dict[str, Any]]:
+    """Chronological request history for one IP across all sources."""
+    rows = conn.execute(
+        "SELECT source, log_file, ts, ts_epoch, method, path, status, host, "
+        "ua, denied, banned, is_error, malicious_hits FROM requests "
+        "WHERE ip = ? ORDER BY COALESCE(ts_epoch, 0), id LIMIT ?",
+        (ip, limit)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def count_requests(conn: sqlite3.Connection) -> int:
+    """Total persisted request records."""
+    return int(conn.execute("SELECT COUNT(*) FROM requests").fetchone()[0])
+
+
+def count_scans(conn: sqlite3.Connection) -> int:
+    """Total persisted scan records."""
+    return int(conn.execute("SELECT COUNT(*) FROM scans").fetchone()[0])
+
+
+# ---------------------------------------------------------------------------
+# Report artifacts
+# ---------------------------------------------------------------------------
+
+def insert_report_artifact(conn: sqlite3.Connection, *, source: str,
+                           report_type: str, generated_at: str,
+                           payload: Any, original_path: str,
+                           content_sha256: str, scan_id: int | None = None,
+                           schema_version: int = 1) -> int:
+    """Insert a structured report artifact, idempotent by path and hash."""
+    with transaction(conn):
+        conn.execute(
+            "INSERT OR IGNORE INTO report_artifacts "
+            "(source, scan_id, report_type, generated_at, schema_version, "
+            "payload_json, original_path, content_sha256, created_at) "
+            "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
+            (source, scan_id, report_type, generated_at, schema_version,
+             json.dumps(payload, ensure_ascii=False), original_path,
+             content_sha256, utcnow()),
+        )
+        row = conn.execute(
+            "SELECT id FROM report_artifacts "
+            "WHERE original_path=? AND content_sha256=?",
+            (original_path, content_sha256),
+        ).fetchone()
+    return int(row["id"])
+
+
+def list_report_artifacts(conn: sqlite3.Connection, *,
+                          source: str | None = None,
+                          report_type: str | None = None,
+                          since: str | None = None,
+                          until: str | None = None,
+                          limit: int = 50,
+                          cursor: str | None = None) -> dict[str, Any]:
+    """List report metadata using bounded keyset pagination."""
+    limit = min(max(int(limit), 1), 200)
+    clauses: list[str] = []
+    args: list[Any] = []
+    if source:
+        clauses.append("source=?")
+        args.append(source)
+    if report_type:
+        clauses.append("report_type=?")
+        args.append(report_type)
+    if since:
+        clauses.append("generated_at>=?")
+        args.append(since)
+    if until:
+        clauses.append("generated_at<=?")
+        args.append(until)
+    if cursor:
+        generated_at, artifact_id = _decode_report_cursor(cursor)
+        clauses.append("(generated_at<? OR (generated_at=? AND id<?))")
+        args.extend((generated_at, generated_at, artifact_id))
+
+    sql = (
+        "SELECT id, source, scan_id, report_type, generated_at, "
+        "schema_version, original_path, content_sha256 "
+        "FROM report_artifacts"
+    )
+    if clauses:
+        sql += " WHERE " + " AND ".join(clauses)
+    sql += " ORDER BY generated_at DESC, id DESC LIMIT ?"
+    rows = [dict(row) for row in conn.execute(sql, (*args, limit + 1))]
+    has_more = len(rows) > limit
+    rows = rows[:limit]
+    next_cursor = (
+        _encode_report_cursor(rows[-1]["generated_at"], rows[-1]["id"])
+        if has_more and rows else None
+    )
+    return {"items": rows, "next_cursor": next_cursor}
+
+
+def get_report_artifact(conn: sqlite3.Connection,
+                        artifact_id: int) -> dict[str, Any] | None:
+    """Fetch one artifact and decode its structured payload."""
+    row = conn.execute(
+        "SELECT id, source, scan_id, report_type, generated_at, "
+        "schema_version, payload_json, original_path, content_sha256, created_at "
+        "FROM report_artifacts WHERE id=?", (artifact_id,)).fetchone()
+    if row is None:
+        return None
+    artifact = dict(row)
+    artifact["payload"] = json.loads(artifact.pop("payload_json"))
+    return artifact
+
+
+def _encode_report_cursor(generated_at: str, artifact_id: int) -> str:
+    raw = f"{generated_at}\n{artifact_id}".encode("utf-8")
+    return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
+
+
+def _decode_report_cursor(cursor: str) -> tuple[str, int]:
+    try:
+        padded = cursor + "=" * (-len(cursor) % 4)
+        generated_at, artifact_id = base64.urlsafe_b64decode(
+            padded.encode("ascii")).decode("utf-8").split("\n", 1)
+        return generated_at, int(artifact_id)
+    except (ValueError, UnicodeError) as exc:
+        raise ValueError("invalid report cursor") from exc
+
+
+def correlate_across_sources(conn: sqlite3.Connection,
+                             limit: int = 100) -> list[dict[str, Any]]:
+    """IPs seen in more than one log source, with per-source counts."""
+    rows = conn.execute(
+        "SELECT ip, COUNT(DISTINCT source) AS sources, "
+        "GROUP_CONCAT(DISTINCT source) AS source_list, COUNT(*) AS total "
+        "FROM requests WHERE ip IS NOT NULL AND ip != '' "
+        "GROUP BY ip HAVING sources > 1 "
+        "ORDER BY total DESC LIMIT ?", (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def top_ips(conn: sqlite3.Connection, since: str | None = None,
+            limit: int = 50) -> list[dict[str, Any]]:
+    """Top IPs by request count within an optional time window."""
+    rows = conn.execute(
+        "SELECT ip, version, COUNT(*) AS total, "
+        "SUM(is_error) AS errors, SUM(malicious_hits) AS malicious_hits, "
+        "MIN(COALESCE(ts_epoch, 0)) AS first_epoch, "
+        "MAX(COALESCE(ts_epoch, 0)) AS last_epoch "
+        "FROM requests WHERE ip IS NOT NULL AND ip != '' "
+        + ("AND COALESCE(ts, '') >= ? " if since else "")
+        + "GROUP BY ip ORDER BY total DESC LIMIT ?",
+        ((since,) if since else ()) + (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+# ---------------------------------------------------------------------------
+# Blocklist ledger
+# ---------------------------------------------------------------------------
+
+def active_blocklist(conn: sqlite3.Connection) -> list[dict[str, Any]]:
+    rows = conn.execute(
+        "SELECT id, cidr, version, added_at, added_by, last_seen "
+        "FROM blocklist_entries WHERE status='active' ORDER BY cidr").fetchall()
+    return [dict(r) for r in rows]
+
+
+def block_entry(conn: sqlite3.Connection, cidr: str, *,
+                added_by: str | None = None, last_seen: str | None = None,
+                reasons: list[dict[str, Any]] | None = None) -> None:
+    """Add a CIDR to the grow-only ledger (idempotent when already active).
+
+    Manages its own transaction for standalone use."""
+    with transaction(conn):
+        _block_entry_inner(conn, cidr, added_by=added_by,
+                           last_seen=last_seen, reasons=reasons)
+
+
+def _block_entry_inner(conn: sqlite3.Connection, cidr: str, *,
+                       added_by: str | None = None,
+                       last_seen: str | None = None,
+                       reasons: list[dict[str, Any]] | None = None) -> None:
+    """Low-level block insert; caller owns the transaction."""
+    now = utcnow()
+    version = _cidr_version(cidr)
+    existing = conn.execute(
+        "SELECT id FROM blocklist_entries WHERE cidr = ? AND status='active'",
+        (cidr,)).fetchone()
+    if existing is not None:
+        entry_id = int(existing["id"])
+        conn.execute(
+            "UPDATE blocklist_entries SET last_seen = COALESCE(?, last_seen) "
+            "WHERE id = ?", (last_seen, entry_id))
+        if reasons:
+            insert_reasons(conn, entry_id, last_seen or "", reasons)
+        return
+    cur = conn.execute(
+        "INSERT INTO blocklist_entries (cidr, version, status, added_at, "
+        "added_by, last_seen) VALUES (?, ?, 'active', ?, ?, ?)",
+        (cidr, version, now, added_by, last_seen))
+    entry_id = int(cur.lastrowid)
+    if reasons:
+        insert_reasons(conn, entry_id, last_seen or "", reasons)
+
+
+def unblock_entry(conn: sqlite3.Connection, cidr: str, *,
+                  actor: str | None = None) -> bool:
+    """Remove a CIDR from active blocklist (idempotent). Returns True if removed."""
+    now = utcnow()
+    cur = conn.execute(
+        "UPDATE blocklist_entries SET status='removed', removed_at=? "
+        "WHERE cidr=? AND status='active'", (now, cidr))
+    removed = cur.rowcount > 0
+    conn.execute(
+        "INSERT INTO blocklist_events (action, cidr, actor, fired_at, detail) "
+        "VALUES ('remove', ?, ?, ?, ?)",
+        (cidr, actor, now, None))
+    return removed
+
+
+def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *,
+                     added_by: str | None = None, reasons=None) -> dict[str, Any]:
+    """Grow-only record: add any ``cidrs`` not already active to the ledger.
+
+    Unlike :func:`sync_blocklist`, this **never removes** entries. It mirrors
+    the router's cumulative model: once an IP/subnet has ever been blocked it
+    stays active even after it leaves the scan window (e.g. NPM ~30-day log
+    retention), exactly like the old git-history backfill. Returns the added
+    delta and the resulting active count.
+    """
+    cidr_set = set(cidrs)
+    before = {r["cidr"] for r in active_blocklist(conn)}
+    to_add = sorted(cidr_set - before)
+    now = utcnow()
+    with transaction(conn):
+        for cidr in to_add:
+            _block_entry_inner(conn, cidr, added_by=added_by, last_seen=now,
+                               reasons=reasons)
+        # Refresh last_seen on entries that are still desired so the ledger
+        # reflects recency for existing rows too.
+        conn.executemany(
+            "UPDATE blocklist_entries SET last_seen=? WHERE cidr=? AND status='active'",
+            [(now, cidr) for cidr in cidr_set.intersection(before)])
+    return {"added": to_add, "active": len(before) + len(to_add)}
+
+
+def prune_old_single_ip_blocks(conn: sqlite3.Connection, *,
+                               older_than_days: int = 30,
+                               actor: str | None = "retention") -> list[str]:
+    """Remove active single-host entries older than ``older_than_days``.
+
+    Keeps CIDR ranges/subnets intact. Host entries are detected via
+    ``ipaddress.ip_network(..., strict=False)`` and only removed when the
+    network is exactly one address (/32 for IPv4, /128 for IPv6).
+    """
+    cutoff = datetime.now(timezone.utc) - \
+        timedelta(days=max(0, older_than_days))
+    rows = conn.execute(
+        "SELECT cidr, added_at, last_seen FROM blocklist_entries "
+        "WHERE status='active'").fetchall()
+
+    to_remove: list[str] = []
+    for row in rows:
+        cidr = str(row["cidr"])
+        if not _is_single_host_cidr(cidr):
+            continue
+        seen = _parse_iso_ts(
+            row["last_seen"]) or _parse_iso_ts(row["added_at"])
+        if seen is None:
+            continue
+        if seen < cutoff:
+            to_remove.append(cidr)
+
+    if not to_remove:
+        return []
+
+    now = utcnow()
+    with transaction(conn):
+        conn.executemany(
+            "UPDATE blocklist_entries SET status='removed', removed_at=? "
+            "WHERE cidr=? AND status='active'",
+            [(now, cidr) for cidr in to_remove],
+        )
+        conn.executemany(
+            "INSERT INTO blocklist_events (action, cidr, actor, fired_at, detail) "
+            "VALUES ('remove', ?, ?, ?, ?)",
+            [(cidr, actor, now, f"single-ip-retention>{older_than_days}d")
+             for cidr in to_remove],
+        )
+    return sorted(to_remove)
+
+
+def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *,
+                   added_by: str | None = None) -> dict[str, Any]:
+    """Reconcile active blocklist to ``desired_cidrs``.
+
+    Adds missing entries; marks active entries absent from ``desired_cidrs``
+    as removed (only ones the system added) so the ledger stays grow-only but
+    reflects the current policy. Returns the add/remove delta.
+    """
+    desired = set(desired_cidrs)
+    current = {r["cidr"] for r in active_blocklist(conn)}
+    added = sorted(desired - current)
+    removed = sorted(current - desired)
+    now = utcnow()
+    with transaction(conn):
+        for cidr in added:
+            _block_entry_inner(conn, cidr, added_by=added_by, last_seen=now)
+        conn.executemany(
+            "UPDATE blocklist_entries SET status='removed', removed_at=? "
+            "WHERE cidr=? AND status='active'",
+            [(now, cidr) for cidr in removed])
+    return {"added": added, "removed": removed}
+
+
+def insert_reasons(conn: sqlite3.Connection, entry_id: int, scan_session: str,
+                   reasons: list[dict[str, Any]]) -> None:
+    tuples = [
+        (entry_id, r.get("scan_id"), r.get("ip") or "", r.get("reason"),
+         r.get("pattern"), r.get("severity"), r.get("sample_path"),
+         int(r.get("hits") or 0))
+        for r in reasons
+    ]
+    conn.executemany(
+        "INSERT INTO blocklist_reasons (entry_id, scan_id, ip, reason, pattern, "
+        "severity, sample_path, hits) VALUES (?,?,?,?,?,?,?,?)",
+        tuples)
+
+
+def blocklist_history(conn: sqlite3.Connection, limit: int = 500) -> list[dict[str, Any]]:
+    rows = conn.execute(
+        "SELECT id, cidr, version, status, added_at, removed_at, added_by, "
+        "last_seen FROM blocklist_entries ORDER BY id DESC LIMIT ?",
+        (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+def _cidr_version(cidr: str) -> int:
+    try:
+        import ipaddress
+        net = ipaddress.ip_network(cidr, strict=False)
+        return net.version
+    except ValueError:
+        return 4 if ":" not in str(cidr) else 6
+
+
+def _is_single_host_cidr(cidr: str) -> bool:
+    try:
+        net = ipaddress.ip_network(cidr, strict=False)
+        return net.num_addresses == 1
+    except ValueError:
+        return False
+
+
+def _parse_iso_ts(value: str | None) -> datetime | None:
+    if not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        return parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+# ---------------------------------------------------------------------------
+# Whitelists
+# ---------------------------------------------------------------------------
+
+def set_whitelist(conn: sqlite3.Connection, provider: str,
+                  cidrs: Sequence[str], source: str | None = None) -> None:
+    """Replace the CIDR set for a provider (mark old inactive, upsert new)."""
+    now = utcnow()
+    with transaction(conn):
+        conn.execute(
+            "UPDATE whitelist_entries SET active=0 WHERE provider=?", (provider,))
+        for cidr in cidrs:
+            version = _cidr_version(cidr)
+            conn.execute(
+                "INSERT INTO whitelist_entries (provider, cidr, version, source, "
+                "active, updated_at) VALUES (?,?,?,?,1,?) "
+                "ON CONFLICT(provider, cidr) DO UPDATE SET active=1, "
+                "source=excluded.source, updated_at=excluded.updated_at",
+                (provider, cidr, version, source, now))
+        conn.execute(
+            "INSERT INTO whitelist_sources (provider, source_type, last_refresh_at, "
+            "entry_count) VALUES (?, ?, ?, ?) ON CONFLICT(provider) DO UPDATE SET "
+            "source_type=excluded.source_type, last_refresh_at=excluded.last_refresh_at, "
+            "entry_count=excluded.entry_count",
+            (provider, source, now, len(cidrs)))
+
+
+def is_whitelisted_db(conn: sqlite3.Connection, ip: str) -> bool:
+    """Check whether an IP falls in any active whitelist CIDR."""
+    import ipaddress
+    try:
+        addr = ipaddress.ip_address(ip)
+    except ValueError:
+        return False
+    rows = conn.execute(
+        "SELECT cidr FROM whitelist_entries WHERE active=1").fetchall()
+    for r in rows:
+        try:
+            net = ipaddress.ip_network(r["cidr"], strict=False)
+        except ValueError:
+            continue
+        if addr.version == net.version and addr in net:
+            return True
+    return False
+
+
+# ---------------------------------------------------------------------------
+# Firewall snapshots
+# ---------------------------------------------------------------------------
+
+def save_firewall_snapshot(conn: sqlite3.Connection, *,
+                           address_lists_v4: list[dict],
+                           address_lists_v6: list[dict],
+                           rules_v4: list[dict], rules_v6: list[dict]) -> int:
+    """Persist a full firewall snapshot; returns the snapshot id."""
+    now = utcnow()
+    cur = conn.execute(
+        "INSERT INTO firewall_snapshots (captured_at, kind, trigger) "
+        "VALUES (?, 'snapshot', 'scan')", (now,))
+    snapshot_id = int(cur.lastrowid)
+    with transaction(conn):
+        tuples = [
+            (snapshot_id, "ipv4", e.get("list"), e.get("address"),
+             int(bool(e.get("dynamic"))), e.get("timeout"))
+            for e in address_lists_v4
+        ] + [
+            (snapshot_id, "ipv6", e.get("list"), e.get("address"),
+             int(bool(e.get("dynamic"))), e.get("timeout"))
+            for e in address_lists_v6
+        ]
+        conn.executemany(
+            "INSERT INTO firewall_address_list_entries (snapshot_id, network, "
+            "list, address, dynamic, timeout) VALUES (?,?,?,?,?,?)", tuples)
+        rule_tuples = [
+            (snapshot_id, "ipv4", idx, e.get("chain"), e.get("action"),
+             e.get("dst-list"), None, e.get("comment"), json.dumps(e))
+            for idx, e in enumerate(rules_v4)
+        ] + [
+            (snapshot_id, "ipv6", idx, e.get("chain"), e.get("action"),
+             e.get("dst-list"), None, e.get("comment"), json.dumps(e))
+            for idx, e in enumerate(rules_v6)
+        ]
+        conn.executemany(
+            "INSERT INTO firewall_rule_entries (snapshot_id, network, rule_index, "
+            "chain, action, dst_list, proto_port, comment, raw_json) "
+            "VALUES (?,?,?,?,?,?,?,?,?)", rule_tuples)
+    return snapshot_id
+
+
+def recent_firewall_snapshots(conn: sqlite3.Connection,
+                              limit: int = 10) -> list[dict[str, Any]]:
+    """Most recent firewall snapshots with entry counts (F001)."""
+    rows = conn.execute(
+        "SELECT fs.id, fs.captured_at, fs.kind, fs.trigger, "
+        "(SELECT COUNT(*) FROM firewall_address_list_entries e "
+        " WHERE e.snapshot_id = fs.id) AS addr_entries, "
+        "(SELECT COUNT(*) FROM firewall_rule_entries r "
+        " WHERE r.snapshot_id = fs.id) AS rule_entries "
+        "FROM firewall_snapshots fs ORDER BY fs.id DESC LIMIT ?",
+        (limit,)).fetchall()
+    return [dict(r) for r in rows]
+
+
+# ---------------------------------------------------------------------------
+# WHOIS cache
+# ---------------------------------------------------------------------------
+
+def set_whois(conn: sqlite3.Connection, ip: str, name: str | None,
+              cidr: str | None) -> None:
+    conn.execute(
+        "INSERT INTO whois_info (ip, name, cidr, fetched_at) VALUES (?,?,?,?) "
+        "ON CONFLICT(ip) DO UPDATE SET name=excluded.name, cidr=excluded.cidr, "
+        "fetched_at=excluded.fetched_at",
+        (ip, name, cidr, utcnow()))

app/db/schema.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""SQLite schema and migrations for the intrusion-detection system.
+
+Definitions live here as the single source of truth and are applied through
+``PRAGMA user_version`` migrations (see ``apply_migrations``). Each entry is an
+ordered list of DDL statements; the version is bumped in lock-step so the
+migrations are idempotent.
+"""
+
+from __future__ import annotations
+
+import sqlite3
+
+from typing import Iterable
+
+#: Latest schema version. Bump by one whenever a new migration is appended.
+SCHEMA_VERSION = 2
+
+#: Ordered migrations: index -> list of DDL statements. A migration N upgrades a
+#: database from user_version N-1 to N. Never edit an older migration in place;
+#: append a new one instead.
+MIGRATIONS: dict[int, list[str]] = {
+    1: [
+        # ------------------------------------------------------------------
+        # log_sources — canonical provenance
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS log_sources (
+            id INTEGER PRIMARY KEY,
+            name TEXT UNIQUE NOT NULL,
+            log_path TEXT,
+            enabled INTEGER NOT NULL DEFAULT 1
+        )
+        """
+    ],
+
+    2: [
+        """
+        CREATE TABLE IF NOT EXISTS report_artifacts (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source TEXT NOT NULL,
+            scan_id INTEGER,
+            report_type TEXT NOT NULL,
+            generated_at TEXT NOT NULL,
+            schema_version INTEGER NOT NULL DEFAULT 1,
+            payload_json TEXT NOT NULL,
+            original_path TEXT NOT NULL,
+            content_sha256 TEXT NOT NULL,
+            created_at TEXT NOT NULL,
+            UNIQUE (original_path, content_sha256)
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_report_artifacts_source_time "
+        "ON report_artifacts (source, generated_at DESC, id DESC)",
+        "CREATE INDEX IF NOT EXISTS idx_report_artifacts_type_time "
+        "ON report_artifacts (report_type, generated_at DESC, id DESC)",
+        """
+        CREATE TABLE IF NOT EXISTS requests (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            source TEXT NOT NULL,
+            scan_id INTEGER,
+            log_file TEXT,
+            ts TEXT,
+            ts_epoch REAL,
+            ip TEXT NOT NULL,
+            version INTEGER,
+            method TEXT,
+            path TEXT,
+            status INTEGER,
+            upstream_status INTEGER,
+            host TEXT,
+            user TEXT,
+            ua TEXT,
+            referer TEXT,
+            denied INTEGER NOT NULL DEFAULT 0,
+            banned INTEGER NOT NULL DEFAULT 0,
+            is_error INTEGER NOT NULL DEFAULT 0,
+            malicious_hits INTEGER NOT NULL DEFAULT 0,
+            extra TEXT,
+            created_at TEXT NOT NULL
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_requests_ip_ts ON requests (ip, ts)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_source_ts ON requests (source, ts)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_ts_epoch ON requests (ts_epoch)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_path ON requests (path)",
+        "CREATE INDEX IF NOT EXISTS idx_requests_scan ON requests (scan_id)",
+        # ------------------------------------------------------------------
+        # ip_stats — per-IP aggregate per scan
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS ip_stats (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            scan_id INTEGER,
+            ip TEXT NOT NULL,
+            version INTEGER,
+            total INTEGER NOT NULL DEFAULT 0,
+            errors INTEGER NOT NULL DEFAULT 0,
+            c4xx INTEGER NOT NULL DEFAULT 0,
+            c5xx INTEGER NOT NULL DEFAULT 0,
+            denied INTEGER NOT NULL DEFAULT 0,
+            banned INTEGER NOT NULL DEFAULT 0,
+            first_ts TEXT,
+            last_ts TEXT,
+            malicious_hits INTEGER NOT NULL DEFAULT 0,
+            paths_json TEXT,
+            methods_json TEXT,
+            hosts_json TEXT,
+            uas_json TEXT,
+            users_json TEXT,
+            whitelist TEXT,
+            blocked INTEGER NOT NULL DEFAULT 0
+        )
+        """,
+        "CREATE UNIQUE INDEX IF NOT EXISTS idx_ip_stats_scan_ip ON ip_stats (scan_id, ip)",
+        # ------------------------------------------------------------------
+        # scans + scans_sources — one row per analysis run
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS scans (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            kind TEXT NOT NULL,
+            started_at TEXT NOT NULL,
+            finished_at TEXT,
+            status TEXT NOT NULL DEFAULT 'running',
+            full INTEGER NOT NULL DEFAULT 0,
+            dry_run INTEGER NOT NULL DEFAULT 0,
+            params_json TEXT,
+            result_json TEXT,
+            log_files INTEGER NOT NULL DEFAULT 0,
+            request_count INTEGER NOT NULL DEFAULT 0,
+            error TEXT
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_scans_started ON scans (started_at)",
+        """
+        CREATE TABLE IF NOT EXISTS scans_sources (
+            scan_id INTEGER NOT NULL,
+            source TEXT NOT NULL,
+            requests INTEGER NOT NULL DEFAULT 0,
+            files INTEGER NOT NULL DEFAULT 0,
+            PRIMARY KEY (scan_id, source)
+        )
+        """,
+        # ------------------------------------------------------------------
+        # blocklist — grow-only ledger + reasons + audit events
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS blocklist_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            cidr TEXT NOT NULL,
+            version INTEGER,
+            status TEXT NOT NULL DEFAULT 'active',
+            added_at TEXT NOT NULL,
+            removed_at TEXT,
+            added_by TEXT,
+            last_seen TEXT
+        )
+        """,
+        """CREATE UNIQUE INDEX IF NOT EXISTS
+           idx_blocklist_active_cidr ON blocklist_entries (cidr) WHERE status='active'""",
+        """
+        CREATE TABLE IF NOT EXISTS blocklist_reasons (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            entry_id INTEGER,
+            scan_id INTEGER,
+            ip TEXT NOT NULL,
+            reason TEXT,
+            pattern TEXT,
+            severity INTEGER,
+            sample_path TEXT,
+            hits INTEGER NOT NULL DEFAULT 0
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_blereasons_ip ON blocklist_reasons (ip)",
+        """
+        CREATE TABLE IF NOT EXISTS blocklist_events (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            action TEXT NOT NULL,
+            cidr TEXT NOT NULL,
+            actor TEXT,
+            fired_at TEXT NOT NULL,
+            detail TEXT
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_blevents_cidr ON blocklist_events (cidr)",
+        # ------------------------------------------------------------------
+        # whitelist — current entries + source refresh tracking
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS whitelist_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            provider TEXT NOT NULL,
+            cidr TEXT NOT NULL,
+            version INTEGER,
+            source TEXT,
+            active INTEGER NOT NULL DEFAULT 1,
+            updated_at TEXT
+        )
+        """,
+        "CREATE UNIQUE INDEX IF NOT EXISTS idx_wl_provider_cidr ON whitelist_entries (provider, cidr)",
+        """
+        CREATE TABLE IF NOT EXISTS whitelist_sources (
+            provider TEXT PRIMARY KEY,
+            source_type TEXT,
+            url TEXT,
+            file TEXT,
+            last_refresh_at TEXT,
+            entry_count INTEGER NOT NULL DEFAULT 0,
+            refresh_error TEXT
+        )
+        """,
+        # ------------------------------------------------------------------
+        # firewall — snapshots of address lists + rules
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS firewall_snapshots (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            captured_at TEXT NOT NULL,
+            kind TEXT NOT NULL DEFAULT 'snapshot',
+            trigger TEXT NOT NULL DEFAULT 'scan'
+        )
+        """,
+        """
+        CREATE TABLE IF NOT EXISTS firewall_address_list_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            snapshot_id INTEGER,
+            network TEXT,
+            list TEXT,
+            address TEXT,
+            dynamic INTEGER NOT NULL DEFAULT 0,
+            timeout INTEGER
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_fwal_snapshot ON firewall_address_list_entries (snapshot_id)",
+        "CREATE INDEX IF NOT EXISTS idx_fwal_list_addr ON firewall_address_list_entries (list, address)",
+        """
+        CREATE TABLE IF NOT EXISTS firewall_rule_entries (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            snapshot_id INTEGER,
+            network TEXT,
+            rule_index INTEGER,
+            chain TEXT,
+            action TEXT,
+            dst_list TEXT,
+            proto_port TEXT,
+            comment TEXT,
+            raw_json TEXT
+        )
+        """,
+        "CREATE INDEX IF NOT EXISTS idx_fwr_snapshot ON firewall_rule_entries (snapshot_id)",
+        # ------------------------------------------------------------------
+        # whois — structured cache
+        # ------------------------------------------------------------------
+        """
+        CREATE TABLE IF NOT EXISTS whois_info (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            ip TEXT UNIQUE NOT NULL,
+            name TEXT,
+            cidr TEXT,
+            fetched_at TEXT NOT NULL
+        )
+        """,
+    ],
+}
+
+
+def apply_migrations(conn: sqlite3.Connection) -> None:
+    """Apply all pending migrations, bumping ``user_version`` in lock-step.
+
+    Safe to call on every process start: it only runs migrations whose version
+    is greater than the current ``PRAGMA user_version``.
+    """
+    conn.execute("PRAGMA foreign_keys=ON")
+    current = conn.execute("PRAGMA user_version").fetchone()[0]
+    for version in range(current + 1, SCHEMA_VERSION + 1):
+        statements = MIGRATIONS.get(version)
+        if statements is None:
+            raise RuntimeError(
+                f"missing migration for schema version {version}")
+        with conn:
+            for ddl in statements:
+                conn.execute(ddl)
+            conn.execute(f"PRAGMA user_version={version}")

app/firewall.py

@@ -1,349 +1,388 @@
-"""
-firewall.py -
-Module for interacting with the Mikrotik routeros shell to manage firewall rules and IP address lists.
-Provides functions to read the current firewall configuration, add and remove IPs from the blocklist.
-Reads the latest report from the IDS scan, reads the current firewall configuration, and updates the router's blocklist accordingly.
-"""
-
-from __future__ import annotations
-
-import os
-import json
-import paramiko
-from datetime import datetime
-import shlex
-from typing import Iterable
-from .constants import (
-    BLOCKLIST_NAME,
-    MIKROTIK_DATA_DIR,
-    REPORT_DIR,
-    ROUTER_IP,
-    ROUTER_PASSWORD,
-    ROUTER_USER,
-    SSH_PORT,
-    SSH_TIMEOUT,
-    WHITELIST_NAMES,
-)
-from .files import write_lines
-from .ip_utils import is_whitelisted
-
-
-class MikrotikFirewall:
-    """
-    Class to manage Mikrotik firewall rules and IP address lists.
-    Provides methods to read the current blocklist, add/remove IPs, and update the blocklist based on IDS scan reports.
-    """
-
-    def __init__(self, router_ip: str = ROUTER_IP, username: str = ROUTER_USER,
-                 password: str = ROUTER_PASSWORD):
-        self.router_ip = router_ip
-        self.username = username
-        self.password = password
-        self.client: paramiko.SSHClient | None = None
-
-    def connect(self) -> None:
-        """Establish an SSH connection to the Mikrotik router."""
-        client = paramiko.SSHClient()
-        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
-        try:
-            # RouterOS 7.20.x exposes an RSA host key and classic SSH ciphers.
-            client.connect(
-                self.router_ip,
-                port=SSH_PORT,
-                username=self.username,
-                password=self.password,
-                timeout=SSH_TIMEOUT,
-                banner_timeout=SSH_TIMEOUT,
-                auth_timeout=SSH_TIMEOUT,
-                allow_agent=False,
-                look_for_keys=False,
-                compress=False,
-                disabled_algorithms={"pubkeys": ["rsa-sha2-512"]},
-            )
-        except Exception:
-            client.close()
-            raise
-        self.client = client
-
-    def disconnect(self) -> None:
-        """Close the SSH connection."""
-        if self.client:
-            self.client.close()
-            self.client = None
-
-    def execute_command(self, command: str) -> str:
-        """Execute a command on the Mikrotik router via SSH and return the output."""
-        if not self.client:
-            raise RuntimeError("SSH client is not connected.")
-        stdin, stdout, stderr = self.client.exec_command(command)
-        output = stdout.read().decode("utf-8")
-        error = stderr.read().decode("utf-8")
-        if error:
-            raise RuntimeError(f"Error executing command '{command}': {error}")
-        return output
-
-    def get_address_list_ipv4(self, name: str) -> set[str]:
-        """Retrieve the current IPv4 address list from the Mikrotik router."""
-        output = self.execute_command("/ip firewall address-list print")
-        ips = set()
-        for line in output.splitlines():
-            parts = line.split()
-            if len(parts) >= 3 and parts[1] == "list" and parts[2] == name:
-                ip = parts[0]
-                ips.add(ip)
-        return ips
-
-    def get_address_list_ipv6(self, name: str) -> set[str]:
-        """Retrieve the current IPv6 address list from the Mikrotik router."""
-        output = self.execute_command("/ipv6 firewall address-list print")
-        ips = set()
-        for line in output.splitlines():
-            parts = line.split()
-            if len(parts) >= 3 and parts[1] == "list" and parts[2] == name:
-                ip = parts[0]
-                ips.add(ip)
-        return ips
-
-    @staticmethod
-    def _parse_terse_output(output: str) -> list[dict[str, str]]:
-        """Parse RouterOS `print terse` records into key/value dictionaries."""
-        records = []
-        for line in output.splitlines():
-            fields = {}
-            for field in shlex.split(line):
-                key, separator, value = field.partition("=")
-                if separator:
-                    fields[key] = value
-            if fields:
-                records.append(fields)
-        return records
-
-    def get_address_lists(self, ipv6: bool = False) -> list[dict[str, str]]:
-        """Retrieve all address-list entries in structured form."""
-        path = "/ipv6" if ipv6 else "/ip"
-        output = self.execute_command(
-            f"{path} firewall address-list print terse")
-        return self._parse_terse_output(output)
-
-    def get_firewall_rules(self, ipv6: bool = False) -> list[dict[str, str]]:
-        """Retrieve all firewall filter rules in structured form."""
-        path = "/ipv6" if ipv6 else "/ip"
-        output = self.execute_command(
-            f"{path} firewall filter print terse")
-        return self._parse_terse_output(output)
-
-    @staticmethod
-    def _write_json(path: str, obj) -> None:
-        with open(path, "w", encoding="utf-8") as file:
-            json.dump(obj, file, indent=2, sort_keys=True)
-
-    def save_state_report(self, report_dir: str = REPORT_DIR,
-                          changes: dict | None = None) -> str:
-        """Save current address lists and firewall rules for auditing.
-
-        Writes structured snapshots to ``<report_dir>/mikrotik/data/`` and to
-        the common ``data/mikrotik/`` directory (overwritten every run so git
-        can diff changes to the firewall state over time), plus a timestamped
-        JSON report under ``<report_dir>/mikrotik/``.
-        """
-        state = {
-            "router": self.router_ip,
-            "collected_at": datetime.now().astimezone().isoformat(),
-            "address_lists": {
-                "ipv4": self.get_address_lists(),
-                "ipv6": self.get_address_lists(ipv6=True),
-            },
-            "firewall_rules": {
-                "ipv4": self.get_firewall_rules(),
-                "ipv6": self.get_firewall_rules(ipv6=True),
-            },
-        }
-        if changes is not None:
-            state["changes"] = changes
-
-        report_data_dir = os.path.join(report_dir, "mikrotik", "data")
-        os.makedirs(report_data_dir, exist_ok=True)
-        os.makedirs(MIKROTIK_DATA_DIR, exist_ok=True)
-        for version in ("ipv4", "ipv6"):
-            for data_dir in (report_data_dir, MIKROTIK_DATA_DIR):
-                self._write_json(
-                    os.path.join(data_dir, f"address_list_{version}.json"),
-                    state["address_lists"][version])
-                self._write_json(
-                    os.path.join(data_dir, f"firewall_{version}.json"),
-                    state["firewall_rules"][version])
-
-        timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
-        report_path = os.path.join(report_dir, "mikrotik", f"{timestamp}.json")
-        self._write_json(report_path, state)
-        return report_path
-
-    def sync_blocklist(self, blocked_ipv4: Iterable[str],
-                       blocked_ipv6: Iterable[str],
-                       dry_run: bool = False,
-                       report_dir: str = REPORT_DIR) -> dict[str, list[str]]:
-        """Synchronize the Blocked list with scan targets and audit the result."""
-        desired_ipv4 = set(blocked_ipv4)
-        desired_ipv6 = set(blocked_ipv6)
-        current_ipv4 = self.get_address_list_ipv4(BLOCKLIST_NAME)
-        current_ipv6 = self.get_address_list_ipv6(BLOCKLIST_NAME)
-        whitelisted_ipv4 = set()
-        whitelisted_ipv6 = set()
-        for name in WHITELIST_NAMES:
-            whitelisted_ipv4.update(self.get_address_list_ipv4(name))
-            whitelisted_ipv6.update(self.get_address_list_ipv6(name))
-
-        changes = {
-            "added_ipv4": sorted(desired_ipv4 - current_ipv4 - whitelisted_ipv4),
-            "added_ipv6": sorted(desired_ipv6 - current_ipv6 - whitelisted_ipv6),
-            "removed_ipv4": sorted(current_ipv4 - desired_ipv4),
-            "removed_ipv6": sorted(current_ipv6 - desired_ipv6),
-            "dry_run": dry_run,
-        }
-        if not dry_run:
-            for ip in changes["removed_ipv4"]:
-                self.execute_command(
-                    f"/ip firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
-            for ip in changes["removed_ipv6"]:
-                self.execute_command(
-                    f"/ipv6 firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
-            self.write_address_list_ipv4(BLOCKLIST_NAME, changes["added_ipv4"])
-            self.write_address_list_ipv6(BLOCKLIST_NAME, changes["added_ipv6"])
-        self.save_state_report(report_dir, changes)
-        return changes
-
-    def write_address_list_ipv4(self, name: str, ips: Iterable[str]) -> None:
-        """Write a list of IPv4 addresses to the Mikrotik router's address list."""
-        for ip in ips:
-            if is_whitelisted(ip):
-                continue  # Skip whitelisted IPs
-            self.execute_command(
-                f"/ip firewall address-list add list={name} address={ip}")
-
-    def write_address_list_ipv6(self, name: str, ips: Iterable[str]) -> None:
-        """Write a list of IPv6 addresses to the Mikrotik router's address list."""
-        for ip in ips:
-            if is_whitelisted(ip):
-                continue  # Skip whitelisted IPs
-            self.execute_command(
-                f"/ipv6 firewall address-list add list={name} address={ip}")
-
-
-def write_blocklist_files(blocked_ipv4: Iterable[str],
-                          blocked_ipv6: Iterable[str],
-                          report_dir: str = REPORT_DIR) -> tuple[str, str]:
-    """Write the final consolidated blocklist to ``<report_dir>/app/data/``.
-
-    Returns the paths of the IPv4 and IPv6 blocklist files
-    (``blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``).
-    """
-    data_dir = os.path.join(report_dir, "app", "data")
-    os.makedirs(data_dir, exist_ok=True)
-    v4_path = os.path.join(data_dir, "blocklist_ipv4.txt")
-    v6_path = os.path.join(data_dir, "blocklist_ipv6.txt")
-    write_lines(v4_path, sorted(blocked_ipv4))
-    write_lines(v6_path, sorted(blocked_ipv6))
-    return v4_path, v6_path
-
-
-def get_scan_report(report_dir: str = REPORT_DIR) -> dict[str, list[str]]:
-    """Read the latest scan report `mikrotik-block.rsc` from the report directory and return the list of blocked IPs."""
-    report_path = os.path.join(report_dir, "mikrotik-block.rsc")
-    if not os.path.exists(report_path):
-        raise FileNotFoundError(f"Scan report not found: {report_path}")
-
-    # add sections, they start with `/ip firewall address-list` or `/ipv6 firewall address-list` and end with a blank line or EOF
-    sections = {
-        "/ip firewall address-list": [],
-        "/ipv6 firewall address-list": [],
-    }
-    # add list=Cloudflare or list=Openfactory to whitelist sections, list=Blocked to blocked section
-    with open(report_path, "r", encoding="utf-8") as f:
-        current_section = None
-        for line in f:
-            line = line.strip()
-            if not line:
-                current_section = None
-                continue
-            if line.startswith("/ip firewall address-list"):
-                current_section = "/ip firewall address-list"
-                continue
-            if line.startswith("/ipv6 firewall address-list"):
-                current_section = "/ipv6 firewall address-list"
-                continue
-            if current_section:
-                sections[current_section].append(line)
-
-    return sections
-
-
-def main():
-    """Legacy entry point for applying the latest generated RouterOS report."""
-    # Read the latest scan report
-    scan_report = get_scan_report(REPORT_DIR)
-
-    # Connect to the Mikrotik router
-    firewall = MikrotikFirewall()
-    firewall.connect()
-
-    # read current blocked IPs from the router
-    current_blocked_ipv4 = firewall.get_address_list_ipv4(BLOCKLIST_NAME)
-    current_blocked_ipv6 = firewall.get_address_list_ipv6(BLOCKLIST_NAME)
-
-    # read whitelisted IPs from the router
-    whitelisted_ipv4 = set()
-    whitelisted_ipv6 = set()
-    for wl_name in WHITELIST_NAMES:
-        whitelisted_ipv4.update(firewall.get_address_list_ipv4(wl_name))
-        whitelisted_ipv6.update(firewall.get_address_list_ipv6(wl_name))
-
-    # read blocked IPs from the scan report
-    blocked_ipv4 = set()
-    blocked_ipv6 = set()
-    for line in scan_report["/ip firewall address-list"]:
-        if "list=Blocked" in line:
-            parts = line.split()
-            for part in parts:
-                if part.startswith("address="):
-                    ip = part.split("=")[1]
-                    blocked_ipv4.add(ip)
-    for line in scan_report["/ipv6 firewall address-list"]:
-        if "list=Blocked" in line:
-            parts = line.split()
-            for part in parts:
-                if part.startswith("address="):
-                    ip = part.split("=")[1]
-                    blocked_ipv6.add(ip)
-
-    # Determine which IPs to add and remove
-    to_add_ipv4 = blocked_ipv4 - current_blocked_ipv4 - whitelisted_ipv4
-    to_add_ipv6 = blocked_ipv6 - current_blocked_ipv6 - whitelisted_ipv6
-    to_remove_ipv4 = current_blocked_ipv4 - blocked_ipv4
-    to_remove_ipv6 = current_blocked_ipv6 - blocked_ipv6
-
-    # Remove old blocked IPs
-    for ip in to_remove_ipv4:
-        firewall.execute_command(
-            f"/ip firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
-    for ip in to_remove_ipv6:
-        firewall.execute_command(
-            f"/ipv6 firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
-
-    # Add new blocked IPs
-    firewall.write_address_list_ipv4(BLOCKLIST_NAME, to_add_ipv4)
-    firewall.write_address_list_ipv6(BLOCKLIST_NAME, to_add_ipv6)
-
-    firewall.save_state_report(REPORT_DIR)
-
-    # Disconnect from the router
-    firewall.disconnect()
-
-    # Print summary
-    print(
-        f"Removed {len(to_remove_ipv4)} IPv4 and {len(to_remove_ipv6)} IPv6 addresses from the blocklist.")
-    print(
-        f"Added {len(to_add_ipv4)} IPv4 and {len(to_add_ipv6)} IPv6 addresses to the blocklist.")
-
-
-if __name__ == "__main__":
-    main()
+"""
+firewall.py -
+Module for interacting with the Mikrotik routeros shell to manage firewall rules and IP address lists.
+Provides functions to read the current firewall configuration, add and remove IPs from the blocklist.
+Reads the latest report from the IDS scan, reads the current firewall configuration, and updates the router's blocklist accordingly.
+"""
+
+from __future__ import annotations
+
+import os
+import json
+import paramiko
+from datetime import datetime
+import shlex
+from typing import Iterable
+from .utils.constants import (
+    BLOCKLIST_NAME,
+    MIKROTIK_DATA_DIR,
+    REPORT_DIR,
+    ROUTER_IP,
+    ROUTER_PASSWORD,
+    ROUTER_USER,
+    SSH_PORT,
+    SSH_TIMEOUT,
+    WHITELIST_NAMES,
+)
+from .utils.files import write_lines
+from .utils.ip_utils import is_whitelisted
+from .utils.blocklist_history import compute_changes, cumulative_blocklist
+
+
+class MikrotikFirewall:
+    """
+    Class to manage Mikrotik firewall rules and IP address lists.
+    Provides methods to read the current blocklist, add/remove IPs, and update the blocklist based on IDS scan reports.
+    """
+
+    def __init__(self, router_ip: str = ROUTER_IP, username: str = ROUTER_USER,
+                 password: str = ROUTER_PASSWORD):
+        self.router_ip = router_ip
+        self.username = username
+        self.password = password
+        self.client: paramiko.SSHClient | None = None
+
+    def connect(self) -> None:
+        """Establish an SSH connection to the Mikrotik router."""
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            # RouterOS 7.20.x exposes an RSA host key and classic SSH ciphers.
+            client.connect(
+                self.router_ip,
+                port=SSH_PORT,
+                username=self.username,
+                password=self.password,
+                timeout=SSH_TIMEOUT,
+                banner_timeout=SSH_TIMEOUT,
+                auth_timeout=SSH_TIMEOUT,
+                allow_agent=False,
+                look_for_keys=False,
+                compress=False,
+                disabled_algorithms={"pubkeys": ["rsa-sha2-512"]},
+            )
+        except Exception:
+            client.close()
+            raise
+        self.client = client
+
+    def disconnect(self) -> None:
+        """Close the SSH connection."""
+        if self.client:
+            self.client.close()
+            self.client = None
+
+    def execute_command(self, command: str) -> str:
+        """Execute a command on the Mikrotik router via SSH and return the output."""
+        if not self.client:
+            raise RuntimeError("SSH client is not connected.")
+        stdin, stdout, stderr = self.client.exec_command(command)
+        output = stdout.read().decode("utf-8")
+        error = stderr.read().decode("utf-8")
+        if error:
+            raise RuntimeError(f"Error executing command '{command}': {error}")
+        return output
+
+    def get_address_list_ipv4(self, name: str) -> set[str]:
+        """Retrieve the current IPv4 address list from the Mikrotik router."""
+        output = self.execute_command("/ip firewall address-list print")
+        ips = set()
+        for line in output.splitlines():
+            parts = line.split()
+            if len(parts) >= 3 and parts[1] == "list" and parts[2] == name:
+                ip = parts[0]
+                ips.add(ip)
+        return ips
+
+    def get_address_list_ipv6(self, name: str) -> set[str]:
+        """Retrieve the current IPv6 address list from the Mikrotik router."""
+        output = self.execute_command("/ipv6 firewall address-list print")
+        ips = set()
+        for line in output.splitlines():
+            parts = line.split()
+            if len(parts) >= 3 and parts[1] == "list" and parts[2] == name:
+                ip = parts[0]
+                ips.add(ip)
+        return ips
+
+    @staticmethod
+    def _parse_terse_output(output: str) -> list[dict[str, str]]:
+        """Parse RouterOS `print terse` records into key/value dictionaries."""
+        records = []
+        for line in output.splitlines():
+            fields = {}
+            for field in shlex.split(line):
+                key, separator, value = field.partition("=")
+                if separator:
+                    fields[key] = value
+            if fields:
+                records.append(fields)
+        return records
+
+    def get_address_lists(self, ipv6: bool = False) -> list[dict[str, str]]:
+        """Retrieve all address-list entries in structured form."""
+        path = "/ipv6" if ipv6 else "/ip"
+        output = self.execute_command(
+            f"{path} firewall address-list print terse")
+        return self._parse_terse_output(output)
+
+    def get_firewall_rules(self, ipv6: bool = False) -> list[dict[str, str]]:
+        """Retrieve all firewall filter rules in structured form."""
+        path = "/ipv6" if ipv6 else "/ip"
+        output = self.execute_command(
+            f"{path} firewall filter print terse")
+        return self._parse_terse_output(output)
+
+    @staticmethod
+    def _write_json(path: str, obj) -> None:
+        with open(path, "w", encoding="utf-8") as file:
+            json.dump(obj, file, indent=2, sort_keys=True)
+
+    def save_state_report(self, report_dir: str = REPORT_DIR,
+                          changes: dict | None = None) -> str:
+        """Save current address lists and firewall rules for auditing.
+
+        Writes structured snapshots to ``<report_dir>/mikrotik/data/`` and to
+        the common ``data/mikrotik/`` directory (scratch mirror for debugging;
+        **not** a git audit trail), plus a timestamped JSON report under
+        ``<report_dir>/mikrotik/``. The authoritative long-term audit trail is
+        the SQLite ``firewall_snapshots`` tables (see ``app.db``), which are
+        appended here regardless of the file writes.
+        """
+        state = {
+            "router": self.router_ip,
+            "collected_at": datetime.now().astimezone().isoformat(),
+            "address_lists": {
+                "ipv4": self.get_address_lists(),
+                "ipv6": self.get_address_lists(ipv6=True),
+            },
+            "firewall_rules": {
+                "ipv4": self.get_firewall_rules(),
+                "ipv6": self.get_firewall_rules(ipv6=True),
+            },
+        }
+        if changes is not None:
+            state["changes"] = changes
+
+        report_data_dir = os.path.join(report_dir, "mikrotik", "data")
+        os.makedirs(report_data_dir, exist_ok=True)
+        os.makedirs(MIKROTIK_DATA_DIR, exist_ok=True)
+        for version in ("ipv4", "ipv6"):
+            for data_dir in (report_data_dir, MIKROTIK_DATA_DIR):
+                self._write_json(
+                    os.path.join(data_dir, f"address_list_{version}.json"),
+                    state["address_lists"][version])
+                self._write_json(
+                    os.path.join(data_dir, f"firewall_{version}.json"),
+                    state["firewall_rules"][version])
+
+        # Persist a SQLite snapshot (authoritative audit trail, independent of
+        # git). Best-effort: never breaks a router sync if the DB is down.
+        self._persist_snapshot_to_db(state)
+
+        timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+        report_path = os.path.join(report_dir, "mikrotik", f"{timestamp}.json")
+        self._write_json(report_path, state)
+        return report_path
+
+    @staticmethod
+    def _persist_snapshot_to_db(state: dict) -> None:
+        """Write the firewall snapshot into the SQLite store (best-effort)."""
+        try:
+            from .db import get_connection
+            from .db import repositories as repo
+            repo.save_firewall_snapshot(
+                get_connection(),
+                address_lists_v4=state["address_lists"]["ipv4"],
+                address_lists_v6=state["address_lists"]["ipv6"],
+                rules_v4=state["firewall_rules"]["ipv4"],
+                rules_v6=state["firewall_rules"]["ipv6"],
+            )
+        except Exception as exc:
+            print(f"WARN: could not persist firewall snapshot to db ({exc})")
+
+    def sync_blocklist(self, blocked_ipv4: Iterable[str],
+                       blocked_ipv6: Iterable[str],
+                       dry_run: bool = False,
+                       report_dir: str = REPORT_DIR) -> dict[str, list[str]]:
+        """Synchronize the Blocked list cumulatively and audit the result.
+
+        The blocklist is grow-only: targets are added when present in the
+        complete cumulative set (git history of the blocklist files + the
+        current working files + this scan's targets) and never removed
+        automatically. Only router entries absent from the complete set — i.e.
+        never blocked by the system — are pruned. The history now lives in the
+        SQLite blocklist ledger (``app.db`` ``blocklist_entries``); git history
+        remains a read-only fallback. See ``app.blocklist_history``.
+        """
+
+        desired_ipv4 = set(blocked_ipv4)
+        desired_ipv6 = set(blocked_ipv6)
+        current_ipv4 = self.get_address_list_ipv4(BLOCKLIST_NAME)
+        current_ipv6 = self.get_address_list_ipv6(BLOCKLIST_NAME)
+        whitelisted_ipv4 = set()
+        whitelisted_ipv6 = set()
+        for name in WHITELIST_NAMES:
+            whitelisted_ipv4.update(self.get_address_list_ipv4(name))
+            whitelisted_ipv6.update(self.get_address_list_ipv6(name))
+
+        v4_path = os.path.join(
+            report_dir, "app", "data", "blocklist_ipv4.txt")
+        v6_path = os.path.join(
+            report_dir, "app", "data", "blocklist_ipv6.txt")
+        complete_v4, complete_v6 = cumulative_blocklist(
+            v4_path, v6_path, desired_ipv4, desired_ipv6)
+
+        changes = compute_changes(
+            current_ipv4, current_ipv6,
+            complete_v4, complete_v6,
+            whitelisted_ipv4, whitelisted_ipv6)
+        changes["dry_run"] = dry_run
+
+        if not dry_run:
+            for ip in changes["removed_ipv4"]:
+                self.execute_command(
+                    f"/ip firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+            for ip in changes["removed_ipv6"]:
+                self.execute_command(
+                    f"/ipv6 firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+            self.write_address_list_ipv4(BLOCKLIST_NAME, changes["added_ipv4"])
+            self.write_address_list_ipv6(BLOCKLIST_NAME, changes["added_ipv6"])
+        self.save_state_report(report_dir, changes)
+        return changes
+
+    def write_address_list_ipv4(self, name: str, ips: Iterable[str]) -> None:
+        """Write a list of IPv4 addresses to the Mikrotik router's address list."""
+        for ip in ips:
+            if is_whitelisted(ip):
+                continue  # Skip whitelisted IPs
+            self.execute_command(
+                f"/ip firewall address-list add list={name} address={ip}")
+
+    def write_address_list_ipv6(self, name: str, ips: Iterable[str]) -> None:
+        """Write a list of IPv6 addresses to the Mikrotik router's address list."""
+        for ip in ips:
+            if is_whitelisted(ip):
+                continue  # Skip whitelisted IPs
+            self.execute_command(
+                f"/ipv6 firewall address-list add list={name} address={ip}")
+
+
+def write_blocklist_files(blocked_ipv4: Iterable[str],
+                          blocked_ipv6: Iterable[str],
+                          report_dir: str = REPORT_DIR) -> tuple[str, str]:
+    """Write the final consolidated blocklist to ``<report_dir>/app/data/``.
+
+    Returns the paths of the IPv4 and IPv6 blocklist files
+    (``blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``).
+    """
+    data_dir = os.path.join(report_dir, "app", "data")
+    os.makedirs(data_dir, exist_ok=True)
+    v4_path = os.path.join(data_dir, "blocklist_ipv4.txt")
+    v6_path = os.path.join(data_dir, "blocklist_ipv6.txt")
+    write_lines(v4_path, sorted(blocked_ipv4))
+    write_lines(v6_path, sorted(blocked_ipv6))
+    return v4_path, v6_path
+
+
+def get_scan_report(report_dir: str = REPORT_DIR) -> dict[str, list[str]]:
+    """Read the latest scan report `mikrotik-block.rsc` from the report directory and return the list of blocked IPs."""
+    report_path = os.path.join(report_dir, "mikrotik-block.rsc")
+    if not os.path.exists(report_path):
+        raise FileNotFoundError(f"Scan report not found: {report_path}")
+
+    # add sections, they start with `/ip firewall address-list` or `/ipv6 firewall address-list` and end with a blank line or EOF
+    sections = {
+        "/ip firewall address-list": [],
+        "/ipv6 firewall address-list": [],
+    }
+    # add list=Cloudflare or list=Init7 to whitelist sections, list=Blocked to blocked section
+    with open(report_path, "r", encoding="utf-8") as f:
+        current_section = None
+        for line in f:
+            line = line.strip()
+            if not line:
+                current_section = None
+                continue
+            if line.startswith("/ip firewall address-list"):
+                current_section = "/ip firewall address-list"
+                continue
+            if line.startswith("/ipv6 firewall address-list"):
+                current_section = "/ipv6 firewall address-list"
+                continue
+            if current_section:
+                sections[current_section].append(line)
+
+    return sections
+
+
+def main():
+    """Legacy entry point for applying the latest generated RouterOS report."""
+    # Read the latest scan report
+    scan_report = get_scan_report(REPORT_DIR)
+
+    # Connect to the Mikrotik router
+    firewall = MikrotikFirewall()
+    firewall.connect()
+
+    # read current blocked IPs from the router
+    current_blocked_ipv4 = firewall.get_address_list_ipv4(BLOCKLIST_NAME)
+    current_blocked_ipv6 = firewall.get_address_list_ipv6(BLOCKLIST_NAME)
+
+    # read whitelisted IPs from the router
+    whitelisted_ipv4 = set()
+    whitelisted_ipv6 = set()
+    for wl_name in WHITELIST_NAMES:
+        whitelisted_ipv4.update(firewall.get_address_list_ipv4(wl_name))
+        whitelisted_ipv6.update(firewall.get_address_list_ipv6(wl_name))
+
+    # read blocked IPs from the scan report
+    blocked_ipv4 = set()
+    blocked_ipv6 = set()
+    for line in scan_report["/ip firewall address-list"]:
+        if "list=Blocked" in line:
+            parts = line.split()
+            for part in parts:
+                if part.startswith("address="):
+                    ip = part.split("=")[1]
+                    blocked_ipv4.add(ip)
+    for line in scan_report["/ipv6 firewall address-list"]:
+        if "list=Blocked" in line:
+            parts = line.split()
+            for part in parts:
+                if part.startswith("address="):
+                    ip = part.split("=")[1]
+                    blocked_ipv6.add(ip)
+
+    # Determine which IPs to add and remove
+    to_add_ipv4 = blocked_ipv4 - current_blocked_ipv4 - whitelisted_ipv4
+    to_add_ipv6 = blocked_ipv6 - current_blocked_ipv6 - whitelisted_ipv6
+    to_remove_ipv4 = current_blocked_ipv4 - blocked_ipv4
+    to_remove_ipv6 = current_blocked_ipv6 - blocked_ipv6
+
+    # Remove old blocked IPs
+    for ip in to_remove_ipv4:
+        firewall.execute_command(
+            f"/ip firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+    for ip in to_remove_ipv6:
+        firewall.execute_command(
+            f"/ipv6 firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+
+    # Add new blocked IPs
+    firewall.write_address_list_ipv4(BLOCKLIST_NAME, to_add_ipv4)
+    firewall.write_address_list_ipv6(BLOCKLIST_NAME, to_add_ipv6)
+
+    firewall.save_state_report(REPORT_DIR)
+
+    # Disconnect from the router
+    firewall.disconnect()
+
+    # Print summary
+    print(
+        f"Removed {len(to_remove_ipv4)} IPv4 and {len(to_remove_ipv6)} IPv6 addresses from the blocklist.")
+    print(
+        f"Added {len(to_add_ipv4)} IPv4 and {len(to_add_ipv6)} IPv6 addresses to the blocklist.")
+
+
+if __name__ == "__main__":
+    main()

app/flags.py

@@ -1,47 +1,47 @@
-#!/usr/bin/env python3
-"""Flagging rules for the intrusion-detection system.
-
-Detection functions that identify hostile IPs based on aggregated stats.
-"""
-
-from __future__ import annotations
-
-from . import bad_patterns
-from .constants import DEFAULT_ERROR_RATIO, DEFAULT_MIN_DENIED, DEFAULT_MIN_ERRORS
-from .models import IPStats
-
-
-def flag_scanners(stats: dict[str, IPStats], min_errors: int = DEFAULT_MIN_ERRORS,
-                  min_ratio: float = DEFAULT_ERROR_RATIO) -> dict[str, IPStats]:
-    """IPs with many errors AND a high error ratio."""
-    return {
-        ip: s for ip, s in stats.items()
-        if s.errors >= min_errors and s.error_ratio >= min_ratio
-    }
-
-
-def flag_denied_only(stats: dict[str, IPStats],
-                     min_denied: int = DEFAULT_MIN_DENIED) -> dict[str, IPStats]:
-    """IPs that only ever hit denied paths and never authenticated."""
-    return {
-        ip: s for ip, s in stats.items()
-        if s.denied >= min_denied and not s.authenticated
-    }
-
-
-def flag_malicious_patterns(stats: dict[str, IPStats],
-                            min_severity: int = 2,
-                            min_hits: int = 1) -> dict[str, IPStats]:
-    """IPs with hits against malicious path patterns at or above severity threshold."""
-    return {
-        ip: s for ip, s in stats.items()
-        if s.malicious_hits >= min_hits and any(
-            bad_patterns.PATTERNS_BY_NAME[name].severity >= min_severity
-            for name in s.malicious_patterns
-        )
-    }
-
-
-def by_total(stats: dict[str, IPStats]) -> list[IPStats]:
-    """Stats sorted by request count, descending."""
-    return sorted(stats.values(), key=lambda s: (-s.total, s.ip))
+#!/usr/bin/env python3
+"""Flagging rules for the intrusion-detection system.
+
+Detection functions that identify hostile IPs based on aggregated stats.
+"""
+
+from __future__ import annotations
+
+from . import bad_patterns
+from .utils.constants import DEFAULT_ERROR_RATIO, DEFAULT_MIN_DENIED, DEFAULT_MIN_ERRORS
+from .utils.models import IPStats
+
+
+def flag_scanners(stats: dict[str, IPStats], min_errors: int = DEFAULT_MIN_ERRORS,
+                  min_ratio: float = DEFAULT_ERROR_RATIO) -> dict[str, IPStats]:
+    """IPs with many errors AND a high error ratio."""
+    return {
+        ip: s for ip, s in stats.items()
+        if s.errors >= min_errors and s.error_ratio >= min_ratio
+    }
+
+
+def flag_denied_only(stats: dict[str, IPStats],
+                     min_denied: int = DEFAULT_MIN_DENIED) -> dict[str, IPStats]:
+    """IPs that only ever hit denied paths and never authenticated."""
+    return {
+        ip: s for ip, s in stats.items()
+        if s.denied >= min_denied and not s.authenticated
+    }
+
+
+def flag_malicious_patterns(stats: dict[str, IPStats],
+                            min_severity: int = 2,
+                            min_hits: int = 1) -> dict[str, IPStats]:
+    """IPs with hits against malicious path patterns at or above severity threshold."""
+    return {
+        ip: s for ip, s in stats.items()
+        if s.malicious_hits >= min_hits and any(
+            bad_patterns.PATTERNS_BY_NAME[name].severity >= min_severity
+            for name in s.malicious_patterns
+        )
+    }
+
+
+def by_total(stats: dict[str, IPStats]) -> list[IPStats]:
+    """Stats sorted by request count, descending."""
+    return sorted(stats.values(), key=lambda s: (-s.total, s.ip))

app/mikrotik.py

@@ -1,111 +1,111 @@
-#!/usr/bin/env python3
-"""MikroTik RouterOS script generation."""
-
-from __future__ import annotations
-
-import ipaddress
-from datetime import datetime
-from typing import Iterable
-
-from .ip_utils import is_ipv4, is_ipv6, is_whitelisted, parse_ip
-from .whitelists import WHITELISTS
-from .constants import BLOCKLIST_NAME
-
-
-def split_cidr_versions(targets: Iterable[str]) -> tuple[list[str], list[str]]:
-    """Split target CIDRs/IPs into (sorted IPv4 list, sorted IPv6 list)."""
-    v4 = []
-    v6 = []
-    for t in targets:
-        ip = t.split("/")[0]
-        if is_ipv4(ip):
-            v4.append(t)
-        elif is_ipv6(ip):
-            v6.append(t)
-
-    def key_cidr(cidr: str):
-        ip = cidr.split("/")[0]
-        addr = parse_ip(ip)
-        if addr is None:
-            return (9, b"", cidr)
-        return (addr.version, addr.packed, cidr)
-
-    return sorted(v4, key=key_cidr), sorted(v6, key=key_cidr)
-
-
-def mikrotik_script(tag: str, blocked_ips_or_subnets: Iterable[str],
-                    whitelists: dict[str, list[str]] | None = None,
-                    generated: str | None = None) -> tuple[str, int, int]:
-    """Build a RouterOS .rsc script.
-
-    Whitelisted ranges become `accept` rules placed above the `drop` rules.
-    Blocked targets can be individual IPs (e.g. 1.2.3.4/32) or CIDR blocks
-    (e.g. 1.2.3.0/24).
-
-    Returns (script_text, blocked_count, whitelisted_cidr_count).
-    """
-    whitelists = WHITELISTS if whitelists is None else whitelists
-    generated = generated or datetime.now().isoformat(timespec="seconds")
-
-    def clean_blocked(targets: Iterable[str]) -> list[str]:
-        cleaned = []
-        for t in targets:
-            ip = t.split("/")[0]
-            if not is_whitelisted(ip):
-                cleaned.append(t)
-        return cleaned
-
-    blocked = clean_blocked(set(blocked_ips_or_subnets))
-    block_v4, block_v6 = split_cidr_versions(blocked)
-
-    lines: list[str] = [f"# {tag} firewall export - generated {generated}", ""]
-    wl_count = 0
-
-    for name, cidrs in whitelists.items():
-        nets = [ipaddress.ip_network(c) for c in cidrs]
-        wl_count += len(nets)
-        for version, cmd in ((4, "/ip firewall address-list"),
-                             (6, "/ipv6 firewall address-list")):
-            chunk = [n for n in nets if n.version == version]
-            if not chunk:
-                continue
-            lines.append(f"# {name} IPv{version} ranges (whitelisted)")
-            lines.append(cmd)
-            lines += [f"add list={name} address={n}" for n in chunk]
-            lines.append("")
-
-    if block_v4:
-        lines.append(f"# Blocked IPv4 ({len(block_v4)}, non-whitelisted)")
-        lines.append("/ip firewall address-list")
-        lines += [f"add list={BLOCKLIST_NAME} address={t}" for t in block_v4]
-        lines.append("")
-    if block_v6:
-        lines.append(f"# Blocked IPv6 ({len(block_v6)}, non-whitelisted)")
-        lines.append("/ipv6 firewall address-list")
-        lines += [f"add list={BLOCKLIST_NAME} address={t}" for t in block_v6]
-        lines.append("")
-
-    for version, cmd, blocks in ((4, "/ip firewall filter", block_v4),
-                                 (6, "/ipv6 firewall filter", block_v6)):
-        lines.append(cmd)
-        for name in whitelists:
-            lines.append(
-                f"add chain=input action=accept src-address-list={name} "
-                f'comment="{tag} {name.lower()} whitelist (v{version})"')
-        if blocks:
-            for chain in ("input", "forward"):
-                lines.append(
-                    f"add chain={chain} action=drop src-address-list={BLOCKLIST_NAME} "
-                    f'comment="{tag} blocked (v{version})"')
-        lines.append("")
-
-    return "\n".join(lines), len(blocked), wl_count
-
-
-def write_mikrotik(path: str, tag: str,
-                   blocked_ips_or_subnets: Iterable[str]) -> tuple[str, int, int]:
-    text, n_blocked, n_whitelisted = mikrotik_script(
-        tag, blocked_ips_or_subnets)
-    with open(path, "w", encoding="utf-8") as f:
-        f.write(text)
-    return path, n_blocked, n_whitelisted
+#!/usr/bin/env python3
+"""MikroTik RouterOS script generation."""
+
+from __future__ import annotations
+
+import ipaddress
+from datetime import datetime
+from typing import Iterable
+
+from .utils.ip_utils import is_ipv4, is_ipv6, is_whitelisted, parse_ip
+from .utils.whitelists import WHITELISTS
+from .utils.constants import BLOCKLIST_NAME
+
+
+def split_cidr_versions(targets: Iterable[str]) -> tuple[list[str], list[str]]:
+    """Split target CIDRs/IPs into (sorted IPv4 list, sorted IPv6 list)."""
+    v4 = []
+    v6 = []
+    for t in targets:
+        ip = t.split("/")[0]
+        if is_ipv4(ip):
+            v4.append(t)
+        elif is_ipv6(ip):
+            v6.append(t)
+
+    def key_cidr(cidr: str):
+        ip = cidr.split("/")[0]
+        addr = parse_ip(ip)
+        if addr is None:
+            return (9, b"", cidr)
+        return (addr.version, addr.packed, cidr)
+
+    return sorted(v4, key=key_cidr), sorted(v6, key=key_cidr)
+
+
+def mikrotik_script(tag: str, blocked_ips_or_subnets: Iterable[str],
+                    whitelists: dict[str, list[str]] | None = None,
+                    generated: str | None = None) -> tuple[str, int, int]:
+    """Build a RouterOS .rsc script.
+
+    Whitelisted ranges become `accept` rules placed above the `drop` rules.
+    Blocked targets can be individual IPs (e.g. 1.2.3.4/32) or CIDR blocks
+    (e.g. 1.2.3.0/24).
+
+    Returns (script_text, blocked_count, whitelisted_cidr_count).
+    """
+    whitelists = WHITELISTS if whitelists is None else whitelists
+    generated = generated or datetime.now().isoformat(timespec="seconds")
+
+    def clean_blocked(targets: Iterable[str]) -> list[str]:
+        cleaned = []
+        for t in targets:
+            ip = t.split("/")[0]
+            if not is_whitelisted(ip):
+                cleaned.append(t)
+        return cleaned
+
+    blocked = clean_blocked(set(blocked_ips_or_subnets))
+    block_v4, block_v6 = split_cidr_versions(blocked)
+
+    lines: list[str] = [f"# {tag} firewall export - generated {generated}", ""]
+    wl_count = 0
+
+    for name, cidrs in whitelists.items():
+        nets = [ipaddress.ip_network(c) for c in cidrs]
+        wl_count += len(nets)
+        for version, cmd in ((4, "/ip firewall address-list"),
+                             (6, "/ipv6 firewall address-list")):
+            chunk = [n for n in nets if n.version == version]
+            if not chunk:
+                continue
+            lines.append(f"# {name} IPv{version} ranges (whitelisted)")
+            lines.append(cmd)
+            lines += [f"add list={name} address={n}" for n in chunk]
+            lines.append("")
+
+    if block_v4:
+        lines.append(f"# Blocked IPv4 ({len(block_v4)}, non-whitelisted)")
+        lines.append("/ip firewall address-list")
+        lines += [f"add list={BLOCKLIST_NAME} address={t}" for t in block_v4]
+        lines.append("")
+    if block_v6:
+        lines.append(f"# Blocked IPv6 ({len(block_v6)}, non-whitelisted)")
+        lines.append("/ipv6 firewall address-list")
+        lines += [f"add list={BLOCKLIST_NAME} address={t}" for t in block_v6]
+        lines.append("")
+
+    for version, cmd, blocks in ((4, "/ip firewall filter", block_v4),
+                                 (6, "/ipv6 firewall filter", block_v6)):
+        lines.append(cmd)
+        for name in whitelists:
+            lines.append(
+                f"add chain=input action=accept src-address-list={name} "
+                f'comment="{tag} {name.lower()} whitelist (v{version})"')
+        if blocks:
+            for chain in ("input", "forward"):
+                lines.append(
+                    f"add chain={chain} action=drop src-address-list={BLOCKLIST_NAME} "
+                    f'comment="{tag} blocked (v{version})"')
+        lines.append("")
+
+    return "\n".join(lines), len(blocked), wl_count
+
+
+def write_mikrotik(path: str, tag: str,
+                   blocked_ips_or_subnets: Iterable[str]) -> tuple[str, int, int]:
+    text, n_blocked, n_whitelisted = mikrotik_script(
+        tag, blocked_ips_or_subnets)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(text)
+    return path, n_blocked, n_whitelisted

app/npm.py

@@ -1,170 +1,170 @@
-#!/usr/bin/env python3
-"""Nginx Proxy Manager access-log analyzer.
-
-Parses NPM proxy access logs (plain + gzipped), aggregates per-IP
-behaviour and flags scanner IPs: clients with many error responses and a
-high error ratio. Flagged IPs become the MikroTik block list.
-"""
-
-from __future__ import annotations
-
-import os
-import re
-from collections import Counter
-
-from .analyzer import Analyzer
-from .constants import (
-    DEFAULT_ERROR_RATIO,
-    DEFAULT_MIN_ERRORS,
-    NPM_LOG_PATH,
-    NPM_REPORT_DIR,
-)
-from .files import write_csv
-from .flags import flag_scanners
-from .models import Request
-from .utils import arg_parser
-
-# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path"
-# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer"
-LINE_RE = re.compile(
-    r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+"  # [datetime]
-    r"-\s+"
-    r"(\d+|-)\s+"                                                  # upstream
-    r"(\d+)\s+"                                                    # status
-    r"-\s+"
-    r"(\S+)\s+"                                                    # method
-    r"(\S+)\s+"                                                    # scheme
-    r"(\S+)\s+"                                                    # host
-    r'"([^"]*)"\s+'                                                # "path"
-    r"\[Client\s+(\S+)\]\s+"
-    r"\[Length\s+(\d+|-)\]\s+"
-    r"\[Gzip\s+(\S+)\]\s+"
-    r"\[Sent-to\s+(\S+)\]\s+"
-    r'"([^"]*)"\s+'                                                # "UA"
-    r'"([^"]*)"'                                                   # "Referer"
-)
-
-
-class NpmAnalyzer(Analyzer):
-    name = "npm"
-    title = "Nginx Proxy Manager log analysis report"
-    log_suffixes = (".log", ".gz")
-    exclude_prefixes = ("letsencrypt",)
-    paths_file = "error_paths.csv"
-    paths_label = "error paths"
-    flag_files = ("scanner_ips.txt", "scanner_ipv6.txt")
-    flag_label = "Scanner IPs"
-    flag_note = "high error rate -- probable scanners/attackers"
-
-    ip_columns = (
-        ("ip", lambda s: s.ip),
-        ("total", lambda s: s.total),
-        ("4xx", lambda s: s.c4xx),
-        ("5xx", lambda s: s.c5xx),
-        ("ratio", lambda s: s.error_ratio),
-        ("first", lambda s: s.first),
-        ("last", lambda s: s.last),
-        ("methods", lambda s: s.methods_str),
-        ("whitelist", lambda s: s.whitelist),
-    )
-
-    request_columns = (
-        ("datetime", lambda r: r.ts),
-        ("ip", lambda r: r.ip),
-        ("method", lambda r: r.method),
-        ("scheme", lambda r: r.extra.get("scheme")),
-        ("host", lambda r: r.host),
-        ("path", lambda r: r.path),
-        ("status", lambda r: r.status),
-        ("upstream_status", lambda r: r.upstream_status),
-        ("length", lambda r: r.extra.get("length")),
-        ("gzip", lambda r: r.extra.get("gzip")),
-        ("sent_to", lambda r: r.extra.get("sent_to")),
-        ("referer", lambda r: r.referer),
-        ("ua", lambda r: r.ua),
-    )
-
-    def __init__(self, *args, min_errors: int = DEFAULT_MIN_ERRORS,
-                 error_ratio: float = DEFAULT_ERROR_RATIO, **kwargs):
-        super().__init__(*args, **kwargs)
-        self.min_errors = min_errors
-        self.min_ratio = error_ratio
-
-    # --- parsing -----------------------------------------------------------
-
-    def parse_line(self, line: str) -> Request | None:
-        if not line:
-            return None
-        m = LINE_RE.match(line)
-        if not m:
-            return None
-        (ts, upstream, status, method, scheme, host, path, ip,
-         length, gz, sent_to, ua, referer) = m.groups()
-        return Request(
-            ts=ts,
-            ip=ip,
-            method=method,
-            path=path,
-            status=int(status),
-            upstream_status=int(upstream) if upstream.isdigit() else None,
-            host=host,
-            ua=ua,
-            referer=referer,
-            extra={"scheme": scheme, "length": length,
-                   "gzip": gz, "sent_to": sent_to},
-        )
-
-    # --- flagging ----------------------------------------------------------
-
-    def flag(self, stats):
-        return flag_scanners(stats, min_errors=self.min_errors,
-                             min_ratio=self.min_ratio)
-
-    # --- extras ------------------------------------------------------------
-
-    def extra_summary_rows(self):
-        c4 = sum(1 for r in self.requests if r.is_client_error)
-        c5 = sum(1 for r in self.requests if r.is_server_error)
-        return [
-            ("4xx / 5xx", f"{c4} / {c5}"),
-            ("hosts seen", len({r.host for r in self.requests if r.host})),
-            ("scanner threshold",
-             f">= {self.min_errors} errors and ratio >= {self.min_ratio}"),
-        ]
-
-    def write_extra(self, data_dir: str) -> list[str]:
-        uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-")
-        hosts = Counter(r.host for r in self.requests if r.host)
-        return [
-            write_csv(os.path.join(data_dir, "ua_summary.csv"),
-                      ["user_agent", "count"], uas.most_common()),
-            write_csv(os.path.join(data_dir, "host_summary.csv"),
-                      ["host", "count"], hosts.most_common()),
-        ]
-
-
-def main() -> None:
-    parser = arg_parser(
-        "analyze NPM proxy logs, flag scanner IPs, export MikroTik rules",
-        NPM_LOG_PATH, NPM_REPORT_DIR)
-    parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
-                        help=f"minimum error responses to flag (default: {DEFAULT_MIN_ERRORS})")
-    parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
-                        help=f"minimum error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
-    parser.add_argument("--full", action="store_true",
-                        help="rescan all logs from the start (ignore incremental state)")
-    args = parser.parse_args()
-
-    NpmAnalyzer(
-        log_path=args.log_path,
-        report_dir=args.report_dir,
-        top=args.top,
-        quiet=args.quiet,
-        full=args.full,
-        min_errors=args.min_errors,
-        error_ratio=args.error_ratio,
-    ).run()
-
-
-if __name__ == "__main__":
-    main()
+#!/usr/bin/env python3
+"""Nginx Proxy Manager access-log analyzer.
+
+Parses NPM proxy access logs (plain + gzipped), aggregates per-IP
+behaviour and flags scanner IPs: clients with many error responses and a
+high error ratio. Flagged IPs become the MikroTik block list.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections import Counter
+
+from .analyzer import Analyzer
+from .utils.constants import (
+    DEFAULT_ERROR_RATIO,
+    DEFAULT_MIN_ERRORS,
+    NPM_LOG_PATH,
+    NPM_REPORT_DIR,
+)
+from .utils.files import write_csv
+from .flags import flag_scanners
+from .utils.models import Request
+from .utils.utils import arg_parser
+
+# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path"
+# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer"
+LINE_RE = re.compile(
+    r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+"  # [datetime]
+    r"-\s+"
+    r"(\d+|-)\s+"                                                  # upstream
+    r"(\d+)\s+"                                                    # status
+    r"-\s+"
+    r"(\S+)\s+"                                                    # method
+    r"(\S+)\s+"                                                    # scheme
+    r"(\S+)\s+"                                                    # host
+    r'"([^"]*)"\s+'                                                # "path"
+    r"\[Client\s+(\S+)\]\s+"
+    r"\[Length\s+(\d+|-)\]\s+"
+    r"\[Gzip\s+(\S+)\]\s+"
+    r"\[Sent-to\s+(\S+)\]\s+"
+    r'"([^"]*)"\s+'                                                # "UA"
+    r'"([^"]*)"'                                                   # "Referer"
+)
+
+
+class NpmAnalyzer(Analyzer):
+    name = "npm"
+    title = "Nginx Proxy Manager log analysis report"
+    log_suffixes = (".log", ".gz")
+    exclude_prefixes = ("letsencrypt",)
+    paths_file = "error_paths.csv"
+    paths_label = "error paths"
+    flag_files = ("scanner_ips.txt", "scanner_ipv6.txt")
+    flag_label = "Scanner IPs"
+    flag_note = "high error rate -- probable scanners/attackers"
+
+    ip_columns = (
+        ("ip", lambda s: s.ip),
+        ("total", lambda s: s.total),
+        ("4xx", lambda s: s.c4xx),
+        ("5xx", lambda s: s.c5xx),
+        ("ratio", lambda s: s.error_ratio),
+        ("first", lambda s: s.first),
+        ("last", lambda s: s.last),
+        ("methods", lambda s: s.methods_str),
+        ("whitelist", lambda s: s.whitelist),
+    )
+
+    request_columns = (
+        ("datetime", lambda r: r.ts),
+        ("ip", lambda r: r.ip),
+        ("method", lambda r: r.method),
+        ("scheme", lambda r: r.extra.get("scheme")),
+        ("host", lambda r: r.host),
+        ("path", lambda r: r.path),
+        ("status", lambda r: r.status),
+        ("upstream_status", lambda r: r.upstream_status),
+        ("length", lambda r: r.extra.get("length")),
+        ("gzip", lambda r: r.extra.get("gzip")),
+        ("sent_to", lambda r: r.extra.get("sent_to")),
+        ("referer", lambda r: r.referer),
+        ("ua", lambda r: r.ua),
+    )
+
+    def __init__(self, *args, min_errors: int = DEFAULT_MIN_ERRORS,
+                 error_ratio: float = DEFAULT_ERROR_RATIO, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.min_errors = min_errors
+        self.min_ratio = error_ratio
+
+    # --- parsing -----------------------------------------------------------
+
+    def parse_line(self, line: str) -> Request | None:
+        if not line:
+            return None
+        m = LINE_RE.match(line)
+        if not m:
+            return None
+        (ts, upstream, status, method, scheme, host, path, ip,
+         length, gz, sent_to, ua, referer) = m.groups()
+        return Request(
+            ts=ts,
+            ip=ip,
+            method=method,
+            path=path,
+            status=int(status),
+            upstream_status=int(upstream) if upstream.isdigit() else None,
+            host=host,
+            ua=ua,
+            referer=referer,
+            extra={"scheme": scheme, "length": length,
+                   "gzip": gz, "sent_to": sent_to},
+        )
+
+    # --- flagging ----------------------------------------------------------
+
+    def flag(self, stats):
+        return flag_scanners(stats, min_errors=self.min_errors,
+                             min_ratio=self.min_ratio)
+
+    # --- extras ------------------------------------------------------------
+
+    def extra_summary_rows(self):
+        c4 = sum(1 for r in self.requests if r.is_client_error)
+        c5 = sum(1 for r in self.requests if r.is_server_error)
+        return [
+            ("4xx / 5xx", f"{c4} / {c5}"),
+            ("hosts seen", len({r.host for r in self.requests if r.host})),
+            ("scanner threshold",
+             f">= {self.min_errors} errors and ratio >= {self.min_ratio}"),
+        ]
+
+    def write_extra(self, data_dir: str) -> list[str]:
+        uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-")
+        hosts = Counter(r.host for r in self.requests if r.host)
+        return [
+            write_csv(os.path.join(data_dir, "ua_summary.csv"),
+                      ["user_agent", "count"], uas.most_common()),
+            write_csv(os.path.join(data_dir, "host_summary.csv"),
+                      ["host", "count"], hosts.most_common()),
+        ]
+
+
+def main() -> None:
+    parser = arg_parser(
+        "analyze NPM proxy logs, flag scanner IPs, export MikroTik rules",
+        NPM_LOG_PATH, NPM_REPORT_DIR)
+    parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
+                        help=f"minimum error responses to flag (default: {DEFAULT_MIN_ERRORS})")
+    parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
+                        help=f"minimum error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
+    parser.add_argument("--full", action="store_true",
+                        help="rescan all logs from the start (ignore incremental state)")
+    args = parser.parse_args()
+
+    NpmAnalyzer(
+        log_path=args.log_path,
+        report_dir=args.report_dir,
+        top=args.top,
+        quiet=args.quiet,
+        full=args.full,
+        min_errors=args.min_errors,
+        error_ratio=args.error_ratio,
+    ).run()
+
+
+if __name__ == "__main__":
+    main()

app/scan/__init__.py

@@ -8,9 +8,16 @@ and outputs a single consolidated MikroTik RouterOS blocklist.
 from __future__ import annotations
 
 import argparse
+import os
 from dataclasses import dataclass, field
 
-from .constants import (
+from ..utils.blocklist_history import (
+    committed_lines,
+    cumulative_blocklist,
+    current_file_lines,
+    write_blocklist_report,
+)
+from ..utils.constants import (
     COPYPARTY_LOG_PATH,
     COPYPARTY_REPORT_DIR,
     DEFAULT_ERROR_RATIO,
@@ -20,14 +27,64 @@ from .constants import (
     NPM_LOG_PATH,
     NPM_REPORT_DIR,
     REPORT_DIR,
+    REPORT_RETENTION_DAYS,
+    REPORT_RETENTION_MAX_FILES,
     SUBNET_THRESHOLD,
 )
-from .firewall import MikrotikFirewall, write_blocklist_files
-from .copyparty import CopypartyAnalyzer
-from .npm import NpmAnalyzer
-from .ip_utils import is_whitelisted
-from .mikrotik import split_cidr_versions
-from .whois import consolidate_by_subnet
+from ..firewall import MikrotikFirewall, write_blocklist_files
+from ..utils.retention import prune_report_dirs
+from ..copyparty import CopypartyAnalyzer
+from ..npm import NpmAnalyzer
+from ..utils.ip_utils import is_whitelisted
+from ..mikrotik import split_cidr_versions
+from ..utils.whois import consolidate_by_subnet
+
+
+def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
+                          desired_v6: list[str],
+                          added_by: str | None = None) -> dict | None:
+    """Best-effort grow-only sync of block targets into the SQLite ledger.
+
+    Uses ``record_blocklist`` (add-only) so the ledger is the authoritative
+    cumulative record: once an IP is ever blocked it stays active even after it
+    leaves the scan window, matching the router's cumulative model. Returns the
+    added delta or None if the DB is unavailable.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        conn = get_connection()
+        result = repo.record_blocklist(
+            conn, list(desired_v4) + list(desired_v6), added_by=added_by)
+        # Retention policy: expire stale single-host entries after 30 days
+        # while keeping CIDR ranges/subnets intact.
+        pruned = repo.prune_old_single_ip_blocks(conn, older_than_days=30)
+        result["pruned_single_ips"] = pruned
+        return result
+    except Exception as exc:  # never break the scan for a DB issue
+        print(f"WARN: could not sync blocklist to db ({exc})")
+        return None
+
+
+def _db_complete_blocklist(desired_v4: list[str], desired_v6: list[str]):
+    """Complete cumulative blocklist from the DB ledger (B001).
+
+    Returns ``(complete_v4, complete_v6)`` as **sets** (same contract as
+    ``cumulative_blocklist``) derived from the active blocklist ledger plus
+    this scan's newly desired targets (in case the DB write for this scan has
+    not run yet). Falls back to the git-history reconstruction when the DB is
+    unavailable, so nothing regresses.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        conn = get_connection()
+        active = [e["cidr"] for e in repo.active_blocklist(conn)]
+    except Exception:
+        active = []
+    union = set(active) | set(desired_v4) | set(desired_v6)
+    v4, v6 = split_cidr_versions(union)
+    return set(v4), set(v6)
 
 
 @dataclass
@@ -70,6 +127,7 @@ class ScanResult:
     blocklist_v6_path: str | None = None
     firewall_connected: bool = False
     firewall_changes: dict | None = None
+    blocklist_report_path: str | None = None
     errors: list[str] = field(default_factory=list)
 
 
@@ -82,6 +140,24 @@ def run_scan(params: ScanParams) -> ScanResult:
     """
     errors: list[str] = []
 
+    # --- Step 0: open the SQLite store for this consolidated run -------------
+    db_scan_id: int | None = None
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        from ..utils.whitelists import sync_whitelists_to_db
+        conn = get_connection()
+        sync_whitelists_to_db()
+        params_map = {
+            "full": params.full, "dry_run": params.dry_run,
+            "update_firewall": params.update_firewall,
+            "quiet": params.quiet, "top": params.top,
+        }
+        db_scan_id = repo.start_scan(conn, "ids", params=params_map,
+                                     full=params.full, dry_run=params.dry_run)
+    except Exception as exc:
+        print(f"WARN: sqlite store unavailable ({exc})")
+
     # --- Step 1-3: connect to MikroTik RouterOS and capture current state ---
     # If the router is unreachable we fall back to scan-only mode: still
     # generate reports and blocklists, but skip the firewall sync.
@@ -123,7 +199,7 @@ def run_scan(params: ScanParams) -> ScanResult:
     print("=== Running copyparty scan ===")
     cp_exists = True
     try:
-        cp_analyzer.run()
+        cp_analyzer.run(db_scan_id=db_scan_id)
     except Exception as e:
         errors.append(f"copyparty scan failed: {e}")
         print(f"Error running copyparty scan: {e}")
@@ -132,7 +208,7 @@ def run_scan(params: ScanParams) -> ScanResult:
     print("\n=== Running NPM scan ===")
     npm_exists = True
     try:
-        npm_analyzer.run()
+        npm_analyzer.run(db_scan_id=db_scan_id)
     except Exception as e:
         errors.append(f"NPM scan failed: {e}")
         print(f"Error running NPM scan: {e}")
@@ -144,6 +220,14 @@ def run_scan(params: ScanParams) -> ScanResult:
         print(f"\n{message}")
         if firewall is not None:
             firewall.disconnect()
+        if db_scan_id is not None:
+            try:
+                from ..db import get_connection
+                from ..db import repositories as repo
+                repo.finish_scan(get_connection(), db_scan_id, status="error",
+                                 error="; ".join(errors))
+            except Exception:
+                pass
         return ScanResult(ok=False, errors=errors)
 
     # Consolidate block candidates
@@ -173,12 +257,22 @@ def run_scan(params: ScanParams) -> ScanResult:
     print(f"Total target blocks/subnets: {len(final_blocked_targets)}")
 
     # --- Step 6: write consolidated blocklists to reports/app/data ----------
-    blocklist_v4_path, blocklist_v6_path = write_blocklist_files(
-        blocked_ipv4, blocked_ipv6, params.report_dir)
+    # Capture the previous blocklist state BEFORE the files are overwritten so
+    # the change report can compute the complete-set delta.
+    app_data_dir = os.path.join(params.report_dir, "app", "data")
+    blocklist_v4_path = os.path.join(app_data_dir, "blocklist_ipv4.txt")
+    blocklist_v6_path = os.path.join(app_data_dir, "blocklist_ipv6.txt")
+    prev_v4 = current_file_lines(blocklist_v4_path)
+    prev_v6 = current_file_lines(blocklist_v6_path)
+    history_v4 = committed_lines(blocklist_v4_path)
+    history_v6 = committed_lines(blocklist_v6_path)
+
+    write_blocklist_files(blocked_ipv4, blocked_ipv6, params.report_dir)
     print(f"blocklist files:\n  {blocklist_v4_path}\n  {blocklist_v6_path}")
 
     # --- Steps 5+7: compare current state with candidates and apply ---------
     firewall_changes: dict | None = None
+    blocklist_report_path: str | None = None
     if firewall is not None:
         try:
             firewall_changes = firewall.sync_blocklist(
@@ -192,6 +286,69 @@ def run_scan(params: ScanParams) -> ScanResult:
               f"remove {len(firewall_changes['removed_ipv4'])} IPv4 and "
               f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.")
 
+        # Timestamped change report — only when the router was reachable.
+        # Source of truth for the cumulative set: the SQLite |ledger| first
+        # (B001), falling back to the git-history reconstruction only if the
+        # DB is unavailable.
+        complete_v4, complete_v6 = _db_complete_blocklist(
+            blocked_ipv4, blocked_ipv6)
+        if not complete_v4 and not complete_v6:
+            complete_v4, complete_v6 = cumulative_blocklist(
+                blocklist_v4_path, blocklist_v6_path,
+                blocked_ipv4, blocked_ipv6)
+        blocklist_report_path = write_blocklist_report(
+            params.report_dir,
+            history_v4=history_v4, history_v6=history_v6,
+            prev_v4=prev_v4, prev_v6=prev_v6,
+            complete_v4=complete_v4, complete_v6=complete_v6,
+            firewall_changes=firewall_changes,
+            dry_run=params.dry_run,
+            errors=errors,
+            scan_id=db_scan_id)
+        print(f"blocklist report: {blocklist_report_path}")
+
+    # --- Step 8: prune old reports (retention) ------------------------------
+    # Each report subdirectory keeps at most REPORT_RETENTION_MAX_FILES files,
+    # all newer than REPORT_RETENTION_DAYS days. The blocklist files in
+    # ``app/data`` are always protected. ``--dry-run`` only previews.
+    pruned = prune_report_dirs(
+        params.report_dir, params.copyparty_report_dir, params.npm_report_dir,
+        dry_run=params.dry_run)
+    total_pruned = sum(len(paths) for paths in pruned.values())
+    action = "Would prune" if params.dry_run else "Pruned"
+    print(f"{action} {total_pruned} report file(s) "
+          f"(keep {REPORT_RETENTION_DAYS}d / "
+          f"max {REPORT_RETENTION_MAX_FILES} per dir)")
+
+    # --- Step 9: persist consolidated results to SQLite ---------------------
+    # Sync the blocklist ledger and close out the scan record regardless of
+    # whether the router sync was attempted.
+    _sync_blocklist_to_db(
+        firewall is not None, blocked_ipv4, blocked_ipv6,
+        added_by=("scan" if db_scan_id is None else f"scan:{db_scan_id}"))
+    if db_scan_id is not None:
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            conn = get_connection()
+            result_payload = {
+                "ok": True,
+                "cp_candidates": cp_candidates,
+                "npm_candidates": npm_candidates,
+                "blocked_ips": len(final_blocked_ips),
+                "blocked_targets": len(final_blocked_targets),
+                "firewall_connected": firewall is not None,
+            }
+            repo.finish_scan(
+                conn, db_scan_id, status="done",
+                result=result_payload, log_files=0,
+                request_count=(cp_analyzer and len(cp_analyzer.requests) or 0)
+                + (npm_analyzer and len(npm_analyzer.requests) or 0),
+                error="; ".join(errors) or None,
+            )
+        except Exception as exc:
+            print(f"WARN: could not finalize scan in db ({exc})")
+
     return ScanResult(
         ok=True,
         cp_exists=cp_exists,
@@ -206,6 +363,7 @@ def run_scan(params: ScanParams) -> ScanResult:
         blocklist_v6_path=blocklist_v6_path,
         firewall_connected=firewall is not None,
         firewall_changes=firewall_changes,
+        blocklist_report_path=blocklist_report_path,
         errors=errors,
     )
 

app/scan/scan_state.py

@@ -1,33 +1,33 @@
-#!/usr/bin/env python3
-"""Persistent state for incremental log scanning.
-
-Tracks the last processed byte offset per absolute log file path so a scan
-only reads new entries instead of re-reading the whole file. The state file
-is written atomically (temp file + rename) to avoid corruption.
-"""
-
-from __future__ import annotations
-
-import json
-import os
-
-from .constants import SCAN_STATE_PATH
-
-
-def load_state() -> dict[str, int]:
-    """Return ``{absolute log path: byte offset}`` from the scan-state file."""
-    try:
-        with open(SCAN_STATE_PATH, "r", encoding="utf-8") as f:
-            data = json.load(f)
-        return {k: int(v) for k, v in data.items()}
-    except (OSError, ValueError, TypeError):
-        return {}
-
-
-def save_state(state: dict[str, int]) -> None:
-    """Write the scan-state file atomically (temp file + rename)."""
-    os.makedirs(os.path.dirname(SCAN_STATE_PATH), exist_ok=True)
-    tmp = SCAN_STATE_PATH + ".tmp"
-    with open(tmp, "w", encoding="utf-8") as f:
-        json.dump(state, f, indent=2, sort_keys=True)
-    os.replace(tmp, SCAN_STATE_PATH)
+#!/usr/bin/env python3
+"""Persistent state for incremental log scanning.
+
+Tracks the last processed byte offset per absolute log file path so a scan
+only reads new entries instead of re-reading the whole file. The state file
+is written atomically (temp file + rename) to avoid corruption.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from ..utils.constants import SCAN_STATE_PATH
+
+
+def load_state() -> dict[str, int]:
+    """Return ``{absolute log path: byte offset}`` from the scan-state file."""
+    try:
+        with open(SCAN_STATE_PATH, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        return {k: int(v) for k, v in data.items()}
+    except (OSError, ValueError, TypeError):
+        return {}
+
+
+def save_state(state: dict[str, int]) -> None:
+    """Write the scan-state file atomically (temp file + rename)."""
+    os.makedirs(os.path.dirname(SCAN_STATE_PATH), exist_ok=True)
+    tmp = SCAN_STATE_PATH + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(state, f, indent=2, sort_keys=True)
+    os.replace(tmp, SCAN_STATE_PATH)

app/static/app.js

@@ -1,83 +1,83 @@
-// Shared helpers for the Intrusion Detection web UI.
-
-async function api(path, options = {}) {
-  const opts = { headers: {}, ...options };
-  if (opts.body && typeof opts.body !== "string") {
-    opts.headers["Content-Type"] = "application/json";
-    opts.body = JSON.stringify(opts.body);
-  }
-  const res = await fetch(path, opts);
-  let data = null;
-  try {
-    data = await res.json();
-  } catch (e) {
-    /* non-JSON */
-  }
-  if (!res.ok) {
-    const msg = (data && (data.error || data.message)) || `HTTP ${res.status}`;
-    throw new Error(msg);
-  }
-  return data;
-}
-
-function escapeHtml(s) {
-  return String(s ?? "").replace(
-    /[&<>"']/g,
-    (c) =>
-      ({
-        "&": "&amp;",
-        "<": "&lt;",
-        ">": "&gt;",
-        '"': "&quot;",
-        "'": "&#39;",
-      })[c],
-  );
-}
-
-function toast(message, type = "info") {
-  const box = document.getElementById("toasts");
-  if (!box) return;
-  const node = document.createElement("div");
-  node.className = `toast toast-${type}`;
-  node.textContent = message;
-  box.appendChild(node);
-  setTimeout(() => node.remove(), 4000);
-}
-
-function fmtTime(s) {
-  if (!s) return "—";
-  const d = new Date(s);
-  return isNaN(d) ? s : d.toLocaleString();
-}
-
-function badge(state) {
-  const node = document.createElement("span");
-  node.className = `badge ${state}`;
-  node.textContent = state;
-  return node;
-}
-
-function el(tag, attrs = {}, ...children) {
-  const node = document.createElement(tag);
-  // Allow el("tr", child1, child2, ...): if the 2nd argument is a node or an
-  // array, treat it as the first child instead of an attributes object.
-  if (attrs && (attrs.nodeType === 1 || Array.isArray(attrs))) {
-    children.unshift(attrs);
-    attrs = {};
-  }
-  for (const [k, v] of Object.entries(attrs || {})) {
-    if (k === "class") node.className = v;
-    else if (k === "text") node.textContent = v;
-    else if (k === "html") node.innerHTML = v;
-    else if (k.startsWith("on")) node.addEventListener(k.slice(2), v);
-    else node.setAttribute(k, v);
-  }
-  for (const c of children) {
-    node.append(c == null ? "" : c);
-  }
-  return node;
-}
-
-async function startScan(kind, params) {
-  return api(`/api/scan/${kind}`, { method: "POST", body: params });
-}
+// Shared helpers for the Intrusion Detection web UI.
+
+async function api(path, options = {}) {
+  const opts = { headers: {}, ...options };
+  if (opts.body && typeof opts.body !== "string") {
+    opts.headers["Content-Type"] = "application/json";
+    opts.body = JSON.stringify(opts.body);
+  }
+  const res = await fetch(path, opts);
+  let data = null;
+  try {
+    data = await res.json();
+  } catch (e) {
+    /* non-JSON */
+  }
+  if (!res.ok) {
+    const msg = (data && (data.error || data.message)) || `HTTP ${res.status}`;
+    throw new Error(msg);
+  }
+  return data;
+}
+
+function escapeHtml(s) {
+  return String(s ?? "").replace(
+    /[&<>"']/g,
+    (c) =>
+      ({
+        "&": "&amp;",
+        "<": "&lt;",
+        ">": "&gt;",
+        '"': "&quot;",
+        "'": "&#39;",
+      })[c],
+  );
+}
+
+function toast(message, type = "info") {
+  const box = document.getElementById("toasts");
+  if (!box) return;
+  const node = document.createElement("div");
+  node.className = `toast toast-${type}`;
+  node.textContent = message;
+  box.appendChild(node);
+  setTimeout(() => node.remove(), 4000);
+}
+
+function fmtTime(s) {
+  if (!s) return "—";
+  const d = new Date(s);
+  return isNaN(d) ? s : d.toLocaleString();
+}
+
+function badge(state) {
+  const node = document.createElement("span");
+  node.className = `badge ${state}`;
+  node.textContent = state;
+  return node;
+}
+
+function el(tag, attrs = {}, ...children) {
+  const node = document.createElement(tag);
+  // Allow el("tr", child1, child2, ...): if the 2nd argument is a node or an
+  // array, treat it as the first child instead of an attributes object.
+  if (attrs && (attrs.nodeType === 1 || Array.isArray(attrs))) {
+    children.unshift(attrs);
+    attrs = {};
+  }
+  for (const [k, v] of Object.entries(attrs || {})) {
+    if (k === "class") node.className = v;
+    else if (k === "text") node.textContent = v;
+    else if (k === "html") node.innerHTML = v;
+    else if (k.startsWith("on")) node.addEventListener(k.slice(2), v);
+    else node.setAttribute(k, v);
+  }
+  for (const c of children) {
+    node.append(c == null ? "" : c);
+  }
+  return node;
+}
+
+async function startScan(kind, params) {
+  return api(`/api/scan/${kind}`, { method: "POST", body: params });
+}

app/static/style.css

@@ -1,499 +1,499 @@
-:root {
-  --bg: #0f172a;
-  --surface: #1e293b;
-  --surface-2: #263449;
-  --border: #334155;
-  --text: #e2e8f0;
-  --muted: #94a3b8;
-  --accent: #22d3ee;
-  --ok: #34d399;
-  --warn: #fbbf24;
-  --danger: #f87171;
-  --mono: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
-}
-
-* {
-  box-sizing: border-box;
-}
-
-body {
-  margin: 0;
-  font-family:
-    system-ui,
-    -apple-system,
-    "Segoe UI",
-    Roboto,
-    sans-serif;
-  background: var(--bg);
-  color: var(--text);
-  line-height: 1.5;
-}
-
-.topbar {
-  display: flex;
-  align-items: center;
-  gap: 1.5rem;
-  padding: 0.75rem 1.5rem;
-  background: var(--surface);
-  border-bottom: 1px solid var(--border);
-  position: sticky;
-  top: 0;
-  z-index: 10;
-}
-
-.brand {
-  font-weight: 700;
-  letter-spacing: 0.02em;
-  white-space: nowrap;
-}
-
-.topbar nav {
-  display: flex;
-  gap: 0.25rem;
-  flex-wrap: wrap;
-}
-
-.topbar nav a {
-  color: var(--muted);
-  text-decoration: none;
-  padding: 0.4rem 0.75rem;
-  border-radius: 8px;
-  font-size: 0.9rem;
-}
-
-.topbar nav a:hover {
-  color: var(--text);
-  background: var(--surface-2);
-}
-.topbar nav a.active {
-  color: var(--accent);
-  background: var(--surface-2);
-}
-
-.container {
-  max-width: 1100px;
-  margin: 0 auto;
-  padding: 1.5rem;
-}
-
-h1 {
-  font-size: 1.4rem;
-  margin: 0 0 0.25rem;
-}
-h2 {
-  font-size: 1.05rem;
-  margin: 0;
-}
-.muted {
-  color: var(--muted);
-}
-.mono {
-  font-family: var(--mono);
-  font-size: 0.85rem;
-}
-
-.panel {
-  background: var(--surface);
-  border: 1px solid var(--border);
-  border-radius: 12px;
-  padding: 1rem 1.25rem;
-  margin: 1rem 0;
-}
-
-.panel-head {
-  display: flex;
-  justify-content: space-between;
-  align-items: center;
-  margin-bottom: 0.75rem;
-}
-
-.cards {
-  display: grid;
-  grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
-  gap: 1rem;
-  margin: 1rem 0;
-}
-
-.card {
-  background: var(--surface);
-  border: 1px solid var(--border);
-  border-radius: 12px;
-  padding: 1rem 1.25rem;
-}
-
-.card-label {
-  font-size: 0.78rem;
-  text-transform: uppercase;
-  letter-spacing: 0.05em;
-  color: var(--muted);
-}
-.card-value {
-  font-size: 1.5rem;
-  font-weight: 700;
-  margin-top: 0.25rem;
-}
-.card-value.ok {
-  color: var(--ok);
-}
-.card-value.warn {
-  color: var(--warn);
-}
-.card-value.danger {
-  color: var(--danger);
-}
-.card-value.muted {
-  color: var(--muted);
-}
-
-.table {
-  width: 100%;
-  border-collapse: collapse;
-  font-size: 0.88rem;
-}
-
-.table th {
-  text-align: left;
-  color: var(--muted);
-  font-weight: 600;
-  border-bottom: 1px solid var(--border);
-  padding: 0.5rem 0.6rem;
-  font-size: 0.78rem;
-  text-transform: uppercase;
-  letter-spacing: 0.04em;
-}
-
-.table td {
-  padding: 0.5rem 0.6rem;
-  border-bottom: 1px solid var(--border);
-}
-.table tr:last-child td {
-  border-bottom: none;
-}
-.table .num {
-  text-align: right;
-  font-variant-numeric: tabular-nums;
-}
-
-.badge {
-  display: inline-block;
-  padding: 0.1rem 0.5rem;
-  border-radius: 999px;
-  font-size: 0.72rem;
-  font-weight: 600;
-  text-transform: uppercase;
-  letter-spacing: 0.03em;
-  background: var(--surface-2);
-  color: var(--muted);
-}
-
-.badge.ok,
-.badge.done {
-  background: rgba(52, 211, 153, 0.15);
-  color: var(--ok);
-}
-.badge.warn,
-.badge.running,
-.badge.queued {
-  background: rgba(251, 191, 36, 0.15);
-  color: var(--warn);
-}
-.badge.danger,
-.badge.error {
-  background: rgba(248, 113, 113, 0.15);
-  color: var(--danger);
-}
-
-.btn {
-  display: inline-flex;
-  align-items: center;
-  gap: 0.4rem;
-  border: 1px solid var(--border);
-  background: var(--surface-2);
-  color: var(--text);
-  padding: 0.45rem 0.9rem;
-  border-radius: 8px;
-  font-size: 0.88rem;
-  cursor: pointer;
-  text-decoration: none;
-}
-
-.btn:hover {
-  border-color: var(--accent);
-}
-.btn.primary {
-  background: var(--accent);
-  border-color: var(--accent);
-  color: #062a35;
-  font-weight: 600;
-}
-.btn.primary:hover {
-  filter: brightness(1.08);
-}
-.btn.danger {
-  border-color: rgba(248, 113, 113, 0.5);
-  color: var(--danger);
-}
-.btn.danger:hover {
-  background: rgba(248, 113, 113, 0.12);
-}
-.btn.ghost {
-  background: transparent;
-}
-.btn:disabled {
-  opacity: 0.5;
-  cursor: not-allowed;
-}
-
-.panel-actions {
-  display: flex;
-  align-items: center;
-  gap: 1rem;
-  margin-top: 1rem;
-  flex-wrap: wrap;
-}
-
-.form-grid {
-  display: grid;
-  grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
-  gap: 1rem;
-}
-
-label {
-  display: flex;
-  flex-direction: column;
-  gap: 0.3rem;
-  font-size: 0.82rem;
-  color: var(--muted);
-}
-label.check {
-  flex-direction: row;
-  align-items: center;
-  gap: 0.5rem;
-  font-size: 0.88rem;
-  color: var(--text);
-  margin-top: 1.4rem;
-}
-
-input,
-select {
-  background: var(--bg);
-  border: 1px solid var(--border);
-  color: var(--text);
-  border-radius: 8px;
-  padding: 0.45rem 0.6rem;
-  font-size: 0.88rem;
-}
-
-input:focus,
-select:focus {
-  outline: none;
-  border-color: var(--accent);
-}
-
-.job-status {
-  margin-top: 1rem;
-  font-family: var(--mono);
-  font-size: 0.85rem;
-}
-
-#toasts {
-  position: fixed;
-  bottom: 1rem;
-  right: 1rem;
-  display: flex;
-  flex-direction: column;
-  gap: 0.5rem;
-  z-index: 100;
-}
-
-.toast {
-  padding: 0.6rem 1rem;
-  border-radius: 8px;
-  border: 1px solid var(--border);
-  background: var(--surface);
-  box-shadow: 0 6px 20px rgba(0, 0, 0, 0.4);
-  font-size: 0.88rem;
-  max-width: 360px;
-}
-
-.toast-error {
-  border-color: var(--danger);
-  color: var(--danger);
-}
-.toast-ok {
-  border-color: var(--ok);
-  color: var(--ok);
-}
-.toast-warn {
-  border-color: var(--warn);
-  color: var(--warn);
-}
-
-pre.content {
-  background: var(--bg);
-  border: 1px solid var(--border);
-  border-radius: 8px;
-  padding: 1rem;
-  overflow: auto;
-  max-height: 480px;
-  font-family: var(--mono);
-  font-size: 0.8rem;
-  white-space: pre-wrap;
-}
-
-.list {
-  display: flex;
-  flex-direction: column;
-  gap: 0.25rem;
-  max-height: 300px;
-  overflow: auto;
-  padding: 0.25rem;
-}
-.list a {
-  color: var(--accent);
-  text-decoration: none;
-  font-family: var(--mono);
-  font-size: 0.85rem;
-  padding: 0.2rem 0.4rem;
-  border-radius: 6px;
-}
-.list a:hover {
-  background: var(--surface-2);
-}
-
-/* Blocklist display */
-
-.table tbody[id="v4"] td {
-  padding: 0;
-}
-.table tbody[id="v4"] td .btn {
-  border-radius: 0;
-  border: none;
-  padding: 0.25rem 0.5rem;
-}
-
-/* IPv4 graph page */
-
-.graph-controls {
-  display: flex;
-  align-items: center;
-  gap: 1.25rem;
-  flex-wrap: wrap;
-}
-
-.legend {
-  display: inline-flex;
-  align-items: center;
-  gap: 0.5rem;
-  font-size: 0.82rem;
-  color: var(--muted);
-}
-
-.sw {
-  display: inline-block;
-  width: 0.8rem;
-  height: 0.8rem;
-  border-radius: 3px;
-  border: 1px solid var(--border);
-}
-.sw.red {
-  background: #e53935;
-}
-.sw.green {
-  background: #43a047;
-}
-.sw.plain {
-  background: #fff;
-}
-
-.graph-wrap {
-  position: relative;
-  width: 100%;
-  max-width: 920px;
-  margin: 0 auto;
-}
-
-#graph-canvas {
-  display: block;
-  width: 100%;
-  height: auto;
-  aspect-ratio: 1 / 1;
-  background: #fff;
-  border: 1px solid var(--border);
-  border-radius: 8px;
-  image-rendering: pixelated;
-  cursor: crosshair;
-}
-
-.graph-overlay {
-  position: absolute;
-  inset: 0;
-  pointer-events: none;
-}
-
-.g-line {
-  position: absolute;
-  background: rgba(148, 163, 184, 0.55);
-}
-.g-v {
-  top: 0;
-  bottom: 0;
-  width: 1px;
-}
-.g-h {
-  left: 0;
-  right: 0;
-  height: 1px;
-}
-
-.g-label {
-  position: absolute;
-  transform: translate(6px, 4px);
-  font-size: 0.55rem;
-  line-height: 1;
-  color: #475569;
-  white-space: nowrap;
-  user-select: none;
-  text-shadow: 0 0 2px #fff, 0 0 2px #fff;
-}
-
-.graph-tooltip {
-  position: absolute;
-  z-index: 5;
-  background: var(--surface-2);
-  border: 1px solid var(--border);
-  border-radius: 8px;
-  padding: 0.5rem 0.65rem;
-  pointer-events: none;
-  box-shadow: 0 6px 20px rgba(0, 0, 0, 0.45);
-  max-width: 260px;
-}
-.graph-tooltip.hidden {
-  display: none;
-}
-
-.tt-head {
-  font-weight: 600;
-  margin-bottom: 0.2rem;
-}
-.tt-status {
-  font-size: 0.75rem;
-  text-transform: uppercase;
-  letter-spacing: 0.04em;
-  font-weight: 600;
-}
-.tt-status.blocked {
-  color: #e53935;
-}
-.tt-status.whitelisted {
-  color: #43a047;
-}
-.tt-status.unlisted {
-  color: var(--muted);
-}
-.tt-list {
-  margin-top: 0.35rem;
-  font-size: 0.72rem;
-  white-space: pre-line;
-  color: var(--text);
-}
+:root {
+  --bg: #0f172a;
+  --surface: #1e293b;
+  --surface-2: #263449;
+  --border: #334155;
+  --text: #e2e8f0;
+  --muted: #94a3b8;
+  --accent: #22d3ee;
+  --ok: #34d399;
+  --warn: #fbbf24;
+  --danger: #f87171;
+  --mono: "SFMono-Regular", Consolas, "Liberation Mono", Menlo, monospace;
+}
+
+* {
+  box-sizing: border-box;
+}
+
+body {
+  margin: 0;
+  font-family:
+    system-ui,
+    -apple-system,
+    "Segoe UI",
+    Roboto,
+    sans-serif;
+  background: var(--bg);
+  color: var(--text);
+  line-height: 1.5;
+}
+
+.topbar {
+  display: flex;
+  align-items: center;
+  gap: 1.5rem;
+  padding: 0.75rem 1.5rem;
+  background: var(--surface);
+  border-bottom: 1px solid var(--border);
+  position: sticky;
+  top: 0;
+  z-index: 10;
+}
+
+.brand {
+  font-weight: 700;
+  letter-spacing: 0.02em;
+  white-space: nowrap;
+}
+
+.topbar nav {
+  display: flex;
+  gap: 0.25rem;
+  flex-wrap: wrap;
+}
+
+.topbar nav a {
+  color: var(--muted);
+  text-decoration: none;
+  padding: 0.4rem 0.75rem;
+  border-radius: 8px;
+  font-size: 0.9rem;
+}
+
+.topbar nav a:hover {
+  color: var(--text);
+  background: var(--surface-2);
+}
+.topbar nav a.active {
+  color: var(--accent);
+  background: var(--surface-2);
+}
+
+.container {
+  max-width: 1100px;
+  margin: 0 auto;
+  padding: 1.5rem;
+}
+
+h1 {
+  font-size: 1.4rem;
+  margin: 0 0 0.25rem;
+}
+h2 {
+  font-size: 1.05rem;
+  margin: 0;
+}
+.muted {
+  color: var(--muted);
+}
+.mono {
+  font-family: var(--mono);
+  font-size: 0.85rem;
+}
+
+.panel {
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: 12px;
+  padding: 1rem 1.25rem;
+  margin: 1rem 0;
+}
+
+.panel-head {
+  display: flex;
+  justify-content: space-between;
+  align-items: center;
+  margin-bottom: 0.75rem;
+}
+
+.cards {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+  gap: 1rem;
+  margin: 1rem 0;
+}
+
+.card {
+  background: var(--surface);
+  border: 1px solid var(--border);
+  border-radius: 12px;
+  padding: 1rem 1.25rem;
+}
+
+.card-label {
+  font-size: 0.78rem;
+  text-transform: uppercase;
+  letter-spacing: 0.05em;
+  color: var(--muted);
+}
+.card-value {
+  font-size: 1.5rem;
+  font-weight: 700;
+  margin-top: 0.25rem;
+}
+.card-value.ok {
+  color: var(--ok);
+}
+.card-value.warn {
+  color: var(--warn);
+}
+.card-value.danger {
+  color: var(--danger);
+}
+.card-value.muted {
+  color: var(--muted);
+}
+
+.table {
+  width: 100%;
+  border-collapse: collapse;
+  font-size: 0.88rem;
+}
+
+.table th {
+  text-align: left;
+  color: var(--muted);
+  font-weight: 600;
+  border-bottom: 1px solid var(--border);
+  padding: 0.5rem 0.6rem;
+  font-size: 0.78rem;
+  text-transform: uppercase;
+  letter-spacing: 0.04em;
+}
+
+.table td {
+  padding: 0.5rem 0.6rem;
+  border-bottom: 1px solid var(--border);
+}
+.table tr:last-child td {
+  border-bottom: none;
+}
+.table .num {
+  text-align: right;
+  font-variant-numeric: tabular-nums;
+}
+
+.badge {
+  display: inline-block;
+  padding: 0.1rem 0.5rem;
+  border-radius: 999px;
+  font-size: 0.72rem;
+  font-weight: 600;
+  text-transform: uppercase;
+  letter-spacing: 0.03em;
+  background: var(--surface-2);
+  color: var(--muted);
+}
+
+.badge.ok,
+.badge.done {
+  background: rgba(52, 211, 153, 0.15);
+  color: var(--ok);
+}
+.badge.warn,
+.badge.running,
+.badge.queued {
+  background: rgba(251, 191, 36, 0.15);
+  color: var(--warn);
+}
+.badge.danger,
+.badge.error {
+  background: rgba(248, 113, 113, 0.15);
+  color: var(--danger);
+}
+
+.btn {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.4rem;
+  border: 1px solid var(--border);
+  background: var(--surface-2);
+  color: var(--text);
+  padding: 0.45rem 0.9rem;
+  border-radius: 8px;
+  font-size: 0.88rem;
+  cursor: pointer;
+  text-decoration: none;
+}
+
+.btn:hover {
+  border-color: var(--accent);
+}
+.btn.primary {
+  background: var(--accent);
+  border-color: var(--accent);
+  color: #062a35;
+  font-weight: 600;
+}
+.btn.primary:hover {
+  filter: brightness(1.08);
+}
+.btn.danger {
+  border-color: rgba(248, 113, 113, 0.5);
+  color: var(--danger);
+}
+.btn.danger:hover {
+  background: rgba(248, 113, 113, 0.12);
+}
+.btn.ghost {
+  background: transparent;
+}
+.btn:disabled {
+  opacity: 0.5;
+  cursor: not-allowed;
+}
+
+.panel-actions {
+  display: flex;
+  align-items: center;
+  gap: 1rem;
+  margin-top: 1rem;
+  flex-wrap: wrap;
+}
+
+.form-grid {
+  display: grid;
+  grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
+  gap: 1rem;
+}
+
+label {
+  display: flex;
+  flex-direction: column;
+  gap: 0.3rem;
+  font-size: 0.82rem;
+  color: var(--muted);
+}
+label.check {
+  flex-direction: row;
+  align-items: center;
+  gap: 0.5rem;
+  font-size: 0.88rem;
+  color: var(--text);
+  margin-top: 1.4rem;
+}
+
+input,
+select {
+  background: var(--bg);
+  border: 1px solid var(--border);
+  color: var(--text);
+  border-radius: 8px;
+  padding: 0.45rem 0.6rem;
+  font-size: 0.88rem;
+}
+
+input:focus,
+select:focus {
+  outline: none;
+  border-color: var(--accent);
+}
+
+.job-status {
+  margin-top: 1rem;
+  font-family: var(--mono);
+  font-size: 0.85rem;
+}
+
+#toasts {
+  position: fixed;
+  bottom: 1rem;
+  right: 1rem;
+  display: flex;
+  flex-direction: column;
+  gap: 0.5rem;
+  z-index: 100;
+}
+
+.toast {
+  padding: 0.6rem 1rem;
+  border-radius: 8px;
+  border: 1px solid var(--border);
+  background: var(--surface);
+  box-shadow: 0 6px 20px rgba(0, 0, 0, 0.4);
+  font-size: 0.88rem;
+  max-width: 360px;
+}
+
+.toast-error {
+  border-color: var(--danger);
+  color: var(--danger);
+}
+.toast-ok {
+  border-color: var(--ok);
+  color: var(--ok);
+}
+.toast-warn {
+  border-color: var(--warn);
+  color: var(--warn);
+}
+
+pre.content {
+  background: var(--bg);
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  padding: 1rem;
+  overflow: auto;
+  max-height: 480px;
+  font-family: var(--mono);
+  font-size: 0.8rem;
+  white-space: pre-wrap;
+}
+
+.list {
+  display: flex;
+  flex-direction: column;
+  gap: 0.25rem;
+  max-height: 300px;
+  overflow: auto;
+  padding: 0.25rem;
+}
+.list a {
+  color: var(--accent);
+  text-decoration: none;
+  font-family: var(--mono);
+  font-size: 0.85rem;
+  padding: 0.2rem 0.4rem;
+  border-radius: 6px;
+}
+.list a:hover {
+  background: var(--surface-2);
+}
+
+/* Blocklist display */
+
+.table tbody[id="v4"] td {
+  padding: 0;
+}
+.table tbody[id="v4"] td .btn {
+  border-radius: 0;
+  border: none;
+  padding: 0.25rem 0.5rem;
+}
+
+/* IPv4 graph page */
+
+.graph-controls {
+  display: flex;
+  align-items: center;
+  gap: 1.25rem;
+  flex-wrap: wrap;
+}
+
+.legend {
+  display: inline-flex;
+  align-items: center;
+  gap: 0.5rem;
+  font-size: 0.82rem;
+  color: var(--muted);
+}
+
+.sw {
+  display: inline-block;
+  width: 0.8rem;
+  height: 0.8rem;
+  border-radius: 3px;
+  border: 1px solid var(--border);
+}
+.sw.red {
+  background: #e53935;
+}
+.sw.green {
+  background: #43a047;
+}
+.sw.plain {
+  background: #fff;
+}
+
+.graph-wrap {
+  position: relative;
+  width: 100%;
+  max-width: 920px;
+  margin: 0 auto;
+}
+
+#graph-canvas {
+  display: block;
+  width: 100%;
+  height: auto;
+  aspect-ratio: 1 / 1;
+  background: #fff;
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  image-rendering: pixelated;
+  cursor: crosshair;
+}
+
+.graph-overlay {
+  position: absolute;
+  inset: 0;
+  pointer-events: none;
+}
+
+.g-line {
+  position: absolute;
+  background: rgba(148, 163, 184, 0.55);
+}
+.g-v {
+  top: 0;
+  bottom: 0;
+  width: 1px;
+}
+.g-h {
+  left: 0;
+  right: 0;
+  height: 1px;
+}
+
+.g-label {
+  position: absolute;
+  transform: translate(6px, 4px);
+  font-size: 0.55rem;
+  line-height: 1;
+  color: #475569;
+  white-space: nowrap;
+  user-select: none;
+  text-shadow: 0 0 2px #fff, 0 0 2px #fff;
+}
+
+.graph-tooltip {
+  position: absolute;
+  z-index: 5;
+  background: var(--surface-2);
+  border: 1px solid var(--border);
+  border-radius: 8px;
+  padding: 0.5rem 0.65rem;
+  pointer-events: none;
+  box-shadow: 0 6px 20px rgba(0, 0, 0, 0.45);
+  max-width: 260px;
+}
+.graph-tooltip.hidden {
+  display: none;
+}
+
+.tt-head {
+  font-weight: 600;
+  margin-bottom: 0.2rem;
+}
+.tt-status {
+  font-size: 0.75rem;
+  text-transform: uppercase;
+  letter-spacing: 0.04em;
+  font-weight: 600;
+}
+.tt-status.blocked {
+  color: #e53935;
+}
+.tt-status.whitelisted {
+  color: #43a047;
+}
+.tt-status.unlisted {
+  color: var(--muted);
+}
+.tt-list {
+  margin-top: 0.35rem;
+  font-size: 0.72rem;
+  white-space: pre-line;
+  color: var(--text);
+}

app/templates/analysis.html

@@ -0,0 +1,191 @@
+{% extends "base.html" %} {% block title %}Log Analysis — Intrusion Detection{%
+endblock %} {% block content %}
+<h1>Log Analysis</h1>
+<p class="muted">
+  Detailed analysis of persisted log data from the SQLite store. Select an IP
+  to see its full request history across all log sources.
+</p>
+
+<div class="panel">
+  <div class="panel-head">
+    <h2>Top IPs</h2>
+    <span class="muted" id="top-disclaimer"></span>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>IP</th>
+        <th class="num">Requests</th>
+        <th class="num">Errors</th>
+        <th class="num">Malicious</th>
+        <th></th>
+      </tr>
+    </thead>
+    <tbody id="top-ips"></tbody>
+  </table>
+</div>
+
+<div class="panel">
+  <div class="panel-head">
+    <h2>Cross-source IPs</h2>
+    <span class="muted">Seen in more than one log source</span>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>IP</th>
+        <th class="num">Sources</th>
+        <th>Source list</th>
+        <th class="num">Total</th>
+      </tr>
+    </thead>
+    <tbody id="cross-source"></tbody>
+  </table>
+</div>
+
+<div class="panel" id="ip-panel" hidden>
+  <div class="panel-head">
+    <h2 id="ip-title">Request history</h2>
+    <a class="btn ghost" href="#" id="ip-close">✕ Close</a>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Time</th>
+        <th>Source</th>
+        <th>Method</th>
+        <th>Path</th>
+        <th class="num">Status</th>
+        <th>UA</th>
+      </tr>
+    </thead>
+    <tbody id="ip-timeline"></tbody>
+  </table>
+</div>
+{% endblock %} {% block scripts %}
+<script>
+  function showIp(ip) {
+    const panel = document.getElementById("ip-panel");
+    const body = document.getElementById("ip-timeline");
+    document.getElementById("ip-title").textContent = `Request history — ${ip}`;
+    panel.hidden = false;
+    body.innerHTML =
+      '<tr><td colspan="6" class="muted">Loading…</td></tr>';
+    api(`/api/analysis/ip/${encodeURIComponent(ip)}`)
+      .then((data) => {
+        body.innerHTML = "";
+        const tl = data.timeline || [];
+        if (!tl.length) {
+          body.innerHTML =
+            '<tr><td colspan="6" class="muted">No persisted requests for this IP.</td></tr>';
+          return;
+        }
+        for (const r of tl) {
+          const status = r.status ?? "—";
+          const row = el(
+            "tr",
+            el("td", {
+              class: "mono",
+              text: r.ts || "—",
+            }),
+            el("td", { text: r.source }),
+            el("td", { text: r.method || "—" }),
+            el("td", { class: "mono", text: r.path || "—" }),
+            el(
+              "td",
+              {
+                class: "num",
+                text: status,
+                style: status >= 400 ? "color:var(--danger)" : "",
+              },
+            ),
+            el("td", { class: "muted", text: r.ua || "—" }),
+          );
+          if (r.is_error) row.classList.add("row-error");
+          body.append(row);
+        }
+      })
+      .catch((e) => toast(e.message, "error"));
+    panel.scrollIntoView({ behavior: "smooth", block: "start" });
+  }
+
+  async function loadTop() {
+    const body = document.getElementById("top-ips");
+    body.innerHTML = "";
+    try {
+      const data = await api("/api/analysis/top-ips?limit=50");
+      body.innerHTML = "";
+      const ips = data.top_ips || [];
+      document.getElementById("top-disclaimer").textContent =
+        `showing top ${ips.length} IPs`;
+      if (!ips.length) {
+        body.innerHTML =
+          '<tr><td colspan="5" class="muted">No persisted requests yet — run a scan to populate the SQLite store.</td></tr>';
+        return;
+      }
+      for (const r of ips) {
+        body.append(
+          el(
+            "tr",
+            el("td", { class: "mono", text: r.ip }),
+            el("td", { class: "num", text: r.total }),
+            el("td", { class: "num", text: r.errors }),
+            el("td", { class: "num", text: r.malicious_hits }),
+            el(
+              "td",
+              el(
+                "button",
+                { class: "btn ghost", onclick: () => showIp(r.ip) },
+                "Detail",
+              ),
+            ),
+          ),
+        );
+      }
+    } catch (e) {
+      body.innerHTML =
+        '<tr><td colspan="5" class="muted b">SQLite analysis unavailable.</td></tr>';
+      toast(e.message, "error");
+    }
+  }
+
+  async function loadCrossSource() {
+    const body = document.getElementById("cross-source");
+    body.innerHTML = "";
+    try {
+      const data = await api("/api/analysis/cross-source?limit=100");
+      const ips = data.ips || [];
+      if (!ips.length) {
+        body.innerHTML =
+          '<tr><td colspan="4" class="muted">No IPs seen across multiple sources yet.</td></tr>';
+        return;
+      }
+      for (const r of ips) {
+        body.append(
+          el(
+            "tr",
+            el("td", { class: "mono", text: r.ip }),
+            el("td", { class: "num", text: r.sources }),
+            el("td", { text: r.source_list }),
+            el("td", { class: "num", text: r.total }),
+          ),
+        );
+      }
+    } catch (e) {
+      body.innerHTML =
+        '<tr><td colspan="4" class="muted b">SQLite analysis unavailable.</td></tr>';
+      toast(e.message, "error");
+    }
+  }
+
+  document
+    .getElementById("ip-close")
+    .addEventListener("click", (e) => {
+      e.preventDefault();
+      document.getElementById("ip-panel").hidden = true;
+    });
+
+  loadTop();
+  loadCrossSource();
+</script>
+{% endblock %}

app/templates/base.html

@@ -1,50 +1,55 @@
-<!doctype html>
-<html lang="en">
-  <head>
-    <meta charset="utf-8" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <title>{% block title %}Intrusion Detection{% endblock %}</title>
-    <link
-      rel="stylesheet"
-      href="{{ url_for('static', filename='style.css') }}"
-    />
-  </head>
-  <body>
-    <header class="topbar">
-      <div class="brand">🛡 Intrusion Detection</div>
-      <nav>
-        <a href="/" class="{{ 'active' if request.path == '/' }}">Dashboard</a>
-        <a href="/scan" class="{{ 'active' if request.path == '/scan' }}"
-          >Scan</a
-        >
-        <a href="/reports" class="{{ 'active' if request.path == '/reports' }}"
-          >Reports</a
-        >
-        <a
-          href="/blocklist"
-          class="{{ 'active' if request.path == '/blocklist' }}"
-          >Blocklist</a
-        >
-        <a
-          href="/firewall"
-          class="{{ 'active' if request.path == '/firewall' }}"
-          >Firewall</a
-        >
-        <a
-          href="/whitelist"
-          class="{{ 'active' if request.path == '/whitelist' }}"
-          >Whitelist</a
-        >
-        <a
-          href="/graph"
-          class="{{ 'active' if request.path == '/graph' }}"
-          >Graph</a
-        >
-      </nav>
-    </header>
-    <main class="container">{% block content %}{% endblock %}</main>
-    <div id="toasts"></div>
-    <script src="{{ url_for('static', filename='app.js') }}"></script>
-    {% block scripts %}{% endblock %}
-  </body>
-</html>
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>{% block title %}Intrusion Detection{% endblock %}</title>
+    <link
+      rel="stylesheet"
+      href="{{ url_for('static', filename='style.css') }}"
+    />
+  </head>
+  <body>
+    <header class="topbar">
+      <div class="brand">🛡 Intrusion Detection</div>
+      <nav>
+        <a href="/" class="{{ 'active' if request.path == '/' }}">Dashboard</a>
+        <a href="/scan" class="{{ 'active' if request.path == '/scan' }}"
+          >Scan</a
+        >
+        <a href="/reports" class="{{ 'active' if request.path == '/reports' }}"
+          >Reports</a
+        >
+        <a
+          href="/blocklist"
+          class="{{ 'active' if request.path == '/blocklist' }}"
+          >Blocklist</a
+        >
+        <a
+          href="/firewall"
+          class="{{ 'active' if request.path == '/firewall' }}"
+          >Firewall</a
+        >
+        <a
+          href="/whitelist"
+          class="{{ 'active' if request.path == '/whitelist' }}"
+          >Whitelist</a
+        >
+        <a
+          href="/graph"
+          class="{{ 'active' if request.path == '/graph' }}"
+          >Graph</a
+        >
+        <a
+          href="/analysis"
+          class="{{ 'active' if request.path == '/analysis' }}"
+          >Analysis</a
+        >
+      </nav>
+    </header>
+    <main class="container">{% block content %}{% endblock %}</main>
+    <div id="toasts"></div>
+    <script src="{{ url_for('static', filename='app.js') }}"></script>
+    {% block scripts %}{% endblock %}
+  </body>
+</html>

app/templates/blocklist.html

@@ -1,134 +1,134 @@
-{% extends "base.html" %} {% block title %}Blocklist — Intrusion Detection{%
-endblock %} {% block content %}
-<h1>Blocklist</h1>
-<p class="muted">
-  Current consolidated blocklist (from the latest scan). Adding/removing touches
-  the MikroTik router.
-</p>
-
-<div class="panel">
-  <div class="panel-head"><h2>Add / remove entry</h2></div>
-  <div class="panel-actions" style="margin-top: 0">
-    <input
-      id="ip"
-      placeholder="IP or CIDR, e.g. 1.2.3.4 or 1.2.0.0/16"
-      style="min-width: 260px"
-    />
-    <button class="btn primary" id="add-btn">Add to Blocked</button>
-    <button class="btn danger" id="remove-btn">Remove from Blocked</button>
-  </div>
-</div>
-
-<div class="panel">
-  <div class="panel-head">
-    <h2>IPv4 <span id="v4-count"></span></h2>
-  </div>
-  <table class="table">
-    <thead>
-      <tr>
-        <th>Address</th>
-        <th></th>
-      </tr>
-    </thead>
-    <tbody id="v4"></tbody>
-  </table>
-</div>
-
-<div class="panel">
-  <div class="panel-head">
-    <h2>IPv6 <span id="v6-count"></span></h2>
-  </div>
-  <table class="table">
-    <thead>
-      <tr>
-        <th>Address</th>
-        <th></th>
-      </tr>
-    </thead>
-    <tbody id="v6"></tbody>
-  </table>
-</div>
-{% endblock %} {% block scripts %}
-<script>
-  async function loadBlocklist() {
-    try {
-      const data = await api("/api/blocklist");
-      render("v4", data.ipv4 || [], "v4-count");
-      render("v6", data.ipv6 || [], "v6-count");
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  function render(tbodyId, entries, countId) {
-    const body = document.getElementById(tbodyId);
-    body.innerHTML = "";
-    document.getElementById(countId).textContent = `(${entries.length})`;
-    if (!entries.length) {
-      body.innerHTML =
-        '<tr><td colspan="2" class="muted">No entries.</td></tr>';
-      return;
-    }
-    for (const ip of entries) {
-      body.append(
-        el(
-          "tr",
-          el("td", { class: "mono", text: ip }),
-          el(
-            "td",
-            { style: "text-align:right" },
-            el("button", {
-              class: "btn danger",
-              text: "Remove",
-              onclick: () => remove(ip),
-            }),
-          ),
-        ),
-      );
-    }
-  }
-
-  async function act(action) {
-    const ip = document.getElementById("ip").value.trim();
-    if (!ip) {
-      toast("Enter an IP or CIDR", "warn");
-      return;
-    }
-    try {
-      const data = await api(`/api/blocklist/${action}`, {
-        method: "POST",
-        body: { ip },
-      });
-      toast(
-        `Removed ${ip}`.replace(
-          "Removed",
-          action === "add" ? "Added" : "Removed",
-        ),
-        "ok",
-      );
-      document.getElementById("ip").value = "";
-      loadBlocklist();
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  async function remove(ip) {
-    try {
-      await api("/api/blocklist/remove", { method: "POST", body: { ip } });
-      toast(`Removed ${ip}`, "ok");
-      loadBlocklist();
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  document
-    .getElementById("add-btn")
-    .addEventListener("click", () => act("add"));
-  document
-    .getElementById("remove-btn")
-    .addEventListener("click", () => act("remove"));
-  loadBlocklist();
-</script>
-{% endblock %}
+{% extends "base.html" %} {% block title %}Blocklist — Intrusion Detection{%
+endblock %} {% block content %}
+<h1>Blocklist</h1>
+<p class="muted">
+  Current consolidated blocklist (from the latest scan). Adding/removing touches
+  the MikroTik router.
+</p>
+
+<div class="panel">
+  <div class="panel-head"><h2>Add / remove entry</h2></div>
+  <div class="panel-actions" style="margin-top: 0">
+    <input
+      id="ip"
+      placeholder="IP or CIDR, e.g. 1.2.3.4 or 1.2.0.0/16"
+      style="min-width: 260px"
+    />
+    <button class="btn primary" id="add-btn">Add to Blocked</button>
+    <button class="btn danger" id="remove-btn">Remove from Blocked</button>
+  </div>
+</div>
+
+<div class="panel">
+  <div class="panel-head">
+    <h2>IPv4 <span id="v4-count"></span></h2>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Address</th>
+        <th></th>
+      </tr>
+    </thead>
+    <tbody id="v4"></tbody>
+  </table>
+</div>
+
+<div class="panel">
+  <div class="panel-head">
+    <h2>IPv6 <span id="v6-count"></span></h2>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Address</th>
+        <th></th>
+      </tr>
+    </thead>
+    <tbody id="v6"></tbody>
+  </table>
+</div>
+{% endblock %} {% block scripts %}
+<script>
+  async function loadBlocklist() {
+    try {
+      const data = await api("/api/blocklist");
+      render("v4", data.ipv4 || [], "v4-count");
+      render("v6", data.ipv6 || [], "v6-count");
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  function render(tbodyId, entries, countId) {
+    const body = document.getElementById(tbodyId);
+    body.innerHTML = "";
+    document.getElementById(countId).textContent = `(${entries.length})`;
+    if (!entries.length) {
+      body.innerHTML =
+        '<tr><td colspan="2" class="muted">No entries.</td></tr>';
+      return;
+    }
+    for (const ip of entries) {
+      body.append(
+        el(
+          "tr",
+          el("td", { class: "mono", text: ip }),
+          el(
+            "td",
+            { style: "text-align:right" },
+            el("button", {
+              class: "btn danger",
+              text: "Remove",
+              onclick: () => remove(ip),
+            }),
+          ),
+        ),
+      );
+    }
+  }
+
+  async function act(action) {
+    const ip = document.getElementById("ip").value.trim();
+    if (!ip) {
+      toast("Enter an IP or CIDR", "warn");
+      return;
+    }
+    try {
+      const data = await api(`/api/blocklist/${action}`, {
+        method: "POST",
+        body: { ip },
+      });
+      toast(
+        `Removed ${ip}`.replace(
+          "Removed",
+          action === "add" ? "Added" : "Removed",
+        ),
+        "ok",
+      );
+      document.getElementById("ip").value = "";
+      loadBlocklist();
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  async function remove(ip) {
+    try {
+      await api("/api/blocklist/remove", { method: "POST", body: { ip } });
+      toast(`Removed ${ip}`, "ok");
+      loadBlocklist();
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  document
+    .getElementById("add-btn")
+    .addEventListener("click", () => act("add"));
+  document
+    .getElementById("remove-btn")
+    .addEventListener("click", () => act("remove"));
+  loadBlocklist();
+</script>
+{% endblock %}

app/templates/dashboard.html

@@ -1,102 +1,109 @@
-{% extends "base.html" %} {% block title %}Dashboard — Intrusion Detection{%
-endblock %} {% block content %}
-<h1>Dashboard</h1>
-<p class="muted">
-  Localhost-only control panel for the intrusion detection scanner.
-</p>
-
-<div class="cards" id="status-cards"></div>
-
-<div class="panel">
-  <div class="panel-head">
-    <h2>Latest scans</h2>
-    <a class="btn ghost" href="/scan">Run a scan →</a>
-  </div>
-  <table class="table">
-    <thead>
-      <tr>
-        <th>Job</th>
-        <th>Kind</th>
-        <th>State</th>
-        <th>Started</th>
-        <th>Finished</th>
-        <th class="num">Blocked</th>
-      </tr>
-    </thead>
-    <tbody id="latest-scans"></tbody>
-  </table>
-</div>
-{% endblock %} {% block scripts %}
-<script>
-  async function loadDashboard() {
-    try {
-      const s = await api("/api/status");
-      renderCards(s);
-      renderScans(s.latest_scans || []);
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  function renderCards(s) {
-    const cards = [
-      {
-        label: "Scan running",
-        value: s.scan_running ? "Yes" : "No",
-        tone: s.scan_running ? "warn" : "ok",
-      },
-      { label: "Blocklist IPv4", value: s.blocklist_ipv4_count ?? "—" },
-      { label: "Blocklist IPv6", value: s.blocklist_ipv6_count ?? "—" },
-      {
-        label: "Firewall snapshot",
-        value:
-          s.firewall_snapshot && s.firewall_snapshot.exists
-            ? "Present"
-            : "None",
-        tone:
-          s.firewall_snapshot && s.firewall_snapshot.exists ? "ok" : "muted",
-      },
-    ];
-    const box = document.getElementById("status-cards");
-    box.innerHTML = "";
-    for (const c of cards) {
-      box.append(
-        el(
-          "div",
-          { class: "card" },
-          el("div", { class: "card-label", text: c.label }),
-          el("div", { class: `card-value ${c.tone || ""}`, text: c.value }),
-        ),
-      );
-    }
-  }
-
-  function renderScans(scans) {
-    const body = document.getElementById("latest-scans");
-    body.innerHTML = "";
-    if (!scans.length) {
-      body.innerHTML =
-        '<tr><td colspan="6" class="muted">No scans yet — run one from the Scan page.</td></tr>';
-      return;
-    }
-    for (const j of scans) {
-      const r = j.result || {};
-      body.append(
-        el(
-          "tr",
-          el("td", { class: "mono", text: j.job_id }),
-          el("td", { text: j.kind }),
-          el("td", {}, badge(j.state)),
-          el("td", { text: fmtTime(j.started) }),
-          el("td", { text: fmtTime(j.finished) }),
-          el("td", { class: "num", text: r.blocked_targets ?? "—" }),
-        ),
-      );
-    }
-  }
-
-  loadDashboard();
-  // Keep the latest-scans table fresh without hammering /api/scan/status.
-  setInterval(loadDashboard, 15000);
-</script>
-{% endblock %}
+{% extends "base.html" %} {% block title %}Dashboard — Intrusion Detection{%
+endblock %} {% block content %}
+<h1>Dashboard</h1>
+<p class="muted">
+  Localhost-only control panel for the intrusion detection scanner.
+</p>
+
+<div class="cards" id="status-cards"></div>
+
+<div class="panel">
+  <div class="panel-head">
+    <h2>Latest scans</h2>
+    <a class="btn ghost" href="/scan">Run a scan →</a>
+  </div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Job</th>
+        <th>Kind</th>
+        <th>State</th>
+        <th>Started</th>
+        <th>Finished</th>
+        <th class="num">Blocked</th>
+      </tr>
+    </thead>
+    <tbody id="latest-scans"></tbody>
+  </table>
+</div>
+{% endblock %} {% block scripts %}
+<script>
+  async function loadDashboard() {
+    try {
+      const s = await api("/api/status");
+      renderCards(s);
+      renderScans(s.latest_scans || []);
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  function renderCards(s) {
+    const cards = [
+      {
+        label: "Scan running",
+        value: s.scan_running ? "Yes" : "No",
+        tone: s.scan_running ? "warn" : "ok",
+      },
+      { label: "Blocklist IPv4", value: s.blocklist_ipv4_count ?? "—" },
+      { label: "Blocklist IPv6", value: s.blocklist_ipv6_count ?? "—" },
+      {
+        label: "Firewall snapshot",
+        value:
+          s.firewall_snapshot && s.firewall_snapshot.exists
+            ? "Present"
+            : "None",
+        tone:
+          s.firewall_snapshot && s.firewall_snapshot.exists ? "ok" : "muted",
+      },
+    ];
+    if (s.analysis && s.analysis.available) {
+      cards.push({
+        label: "SQLite stores",
+        value: `${s.analysis.request_count ?? 0} req / ${s.analysis.scan_count ?? 0} scans`,
+        tone: "ok",
+      });
+    }
+    const box = document.getElementById("status-cards");
+    box.innerHTML = "";
+    for (const c of cards) {
+      box.append(
+        el(
+          "div",
+          { class: "card" },
+          el("div", { class: "card-label", text: c.label }),
+          el("div", { class: `card-value ${c.tone || ""}`, text: c.value }),
+        ),
+      );
+    }
+  }
+
+  function renderScans(scans) {
+    const body = document.getElementById("latest-scans");
+    body.innerHTML = "";
+    if (!scans.length) {
+      body.innerHTML =
+        '<tr><td colspan="6" class="muted">No scans yet — run one from the Scan page.</td></tr>';
+      return;
+    }
+    for (const j of scans) {
+      const r = j.result || {};
+      body.append(
+        el(
+          "tr",
+          el("td", { class: "mono", text: j.job_id }),
+          el("td", { text: j.kind }),
+          el("td", {}, badge(j.state)),
+          el("td", { text: fmtTime(j.started) }),
+          el("td", { text: fmtTime(j.finished) }),
+          el("td", { class: "num", text: r.blocked_targets ?? "—" }),
+        ),
+      );
+    }
+  }
+
+  loadDashboard();
+  // Keep the latest-scans table fresh without hammering /api/scan/status.
+  setInterval(loadDashboard, 15000);
+</script>
+{% endblock %}

app/templates/firewall.html

@@ -1,176 +1,221 @@
-{% extends "base.html" %} {% block title %}Firewall — Intrusion Detection{%
-endblock %} {% block content %}
-<h1>Firewall</h1>
-<p class="muted">
-  Current MikroTik state (from the latest <code>data/mikrotik</code> snapshot).
-  Sync previews or applies the blocklist diff.
-</p>
-
-<div class="panel">
-  <div class="panel-actions" style="margin-top: 0">
-    <button class="btn" id="preview-btn">Preview sync (dry-run)</button>
-    <button class="btn danger" id="apply-btn">Apply sync</button>
-    <span class="muted" id="fw-result"></span>
-  </div>
-</div>
-
-<div class="panel">
-  <div class="panel-head"><h2>Address lists</h2></div>
-  <div id="address-lists"></div>
-</div>
-
-<div class="panel">
-  <div class="panel-head"><h2>Firewall rules</h2></div>
-  <div id="firewall-rules"></div>
-</div>
-{% endblock %} {% block scripts %}
-<script>
-  async function loadState() {
-    try {
-      const s = await api("/api/firewall/state");
-      renderAddressLists(s.address_lists || {});
-      renderRules("firewall-rules", s.firewall_rules || {});
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  function renderAddressLists(addr) {
-    const box = document.getElementById("address-lists");
-    box.innerHTML = "";
-    const byList = {};
-    for (const version of ["ipv4", "ipv6"]) {
-      for (const e of addr[version] || []) {
-        const name = e.list || "?";
-        (byList[name] = byList[name] || { ipv4: [], ipv6: [] })[version].push(
-          e.address,
-        );
-      }
-    }
-    const names = Object.keys(byList);
-    if (!names.length) {
-      box.innerHTML =
-        '<p class="muted">No address-list snapshot available. Run <code>app.py --update-firewall</code> first.</p>';
-      return;
-    }
-    for (const name of names) {
-      const rows = [];
-      for (const version of ["ipv4", "ipv6"]) {
-        const entries = (byList[name][version] || []).sort();
-        rows.push(
-          el(
-            "tr",
-            el("td", { text: version }),
-            el("td", { class: "mono", text: entries.join(", ") || "—" }),
-          ),
-        );
-      }
-      box.append(
-        el(
-          "div",
-          { style: "margin-bottom:1rem" },
-          el("div", {
-            style: "font-weight:600;margin-bottom:.4rem",
-            text: name,
-          }),
-          el(
-            "table",
-            { class: "table" },
-            el(
-              "thead",
-              {},
-              el(
-                "tr",
-                {},
-                el("th", { text: "Version" }),
-                el("th", { text: "Addresses" }),
-              ),
-            ),
-            el("tbody", {}, ...rows),
-          ),
-        ),
-      );
-    }
-  }
-
-  function renderRules(boxId, rules) {
-    const box = document.getElementById(boxId);
-    box.innerHTML = "";
-    for (const version of ["ipv4", "ipv6"]) {
-      const list = rules[version] || [];
-      const rows = list.map((r) =>
-        el(
-          "tr",
-          el("td", {
-            class: "mono",
-            text: r.address ? r.address : r["src-address-list"] || "",
-          }),
-          el("td", { text: r.chain || "—" }),
-          el("td", { text: r.action || "—" }),
-          el("td", { text: r.comment || "" }),
-        ),
-      );
-      box.append(
-        el(
-          "div",
-          { style: "margin-bottom:1rem" },
-          el("div", {
-            style: "font-weight:600;margin-bottom:.4rem",
-            text: `IPv${version} (${list.length})`,
-          }),
-          rows.length
-            ? el(
-                "table",
-                { class: "table" },
-                el(
-                  "thead",
-                  {},
-                  el(
-                    "tr",
-                    {},
-                    el("th", { text: "Address / List" }),
-                    el("th", { text: "Chain" }),
-                    el("th", { text: "Action" }),
-                    el("th", { text: "Comment" }),
-                  ),
-                ),
-                el("tbody", {}, ...rows),
-              )
-            : el("p", { class: "muted", text: "No rules in snapshot." }),
-        ),
-      );
-    }
-  }
-
-  async function sync(dryRun) {
-    const out = document.getElementById("fw-result");
-    out.textContent = "Working…";
-    try {
-      const changes = await api(`/api/firewall/${dryRun ? "sync" : "apply"}`, {
-        method: "POST",
-        body: {},
-      });
-      const verb = dryRun ? "Would" : "";
-      out.textContent =
-        `${verb} add ${(changes.added_ipv4 || []).length} IPv4 + ${(changes.added_ipv6 || []).length} IPv6; ` +
-        `remove ${(changes.removed_ipv4 || []).length} IPv4 + ${(changes.removed_ipv6 || []).length} IPv6.`;
-      toast(
-        (dryRun ? "Preview" : "Applied") + " firewall sync",
-        dryRun ? "warn" : "ok",
-      );
-      if (!dryRun) loadState();
-    } catch (e) {
-      out.textContent = "";
-      toast(e.message, "error");
-    }
-  }
-
-  document
-    .getElementById("preview-btn")
-    .addEventListener("click", () => sync(true));
-  document
-    .getElementById("apply-btn")
-    .addEventListener("click", () => sync(false));
-  loadState();
-</script>
-{% endblock %}
+{% extends "base.html" %} {% block title %}Firewall — Intrusion Detection{%
+endblock %} {% block content %}
+<h1>Firewall</h1>
+<p class="muted">
+  Current MikroTik state (from the latest <code>data/mikrotik</code> snapshot).
+  Sync previews or applies the blocklist diff.
+</p>
+
+<div class="panel">
+  <div class="panel-actions" style="margin-top: 0">
+    <button class="btn" id="preview-btn">Preview sync (dry-run)</button>
+    <button class="btn danger" id="apply-btn">Apply sync</button>
+    <span class="muted" id="fw-result"></span>
+  </div>
+</div>
+
+<div class="panel">
+  <div class="panel-head"><h2>Address lists</h2></div>
+  <div id="address-lists"></div>
+</div>
+
+<div class="panel">
+  <div class="panel-head"><h2>Firewall rules</h2></div>
+  <div id="firewall-rules"></div>
+</div>
+
+<div class="panel">
+  <div class="panel-head"><h2>Snapshot history (SQLite)</h2></div>
+  <div id="fw-snapshots"></div>
+</div>
+{% endblock %} {% block scripts %}
+<script>
+  async function loadState() {
+    try {
+      const s = await api("/api/firewall/state");
+      renderAddressLists(s.address_lists || {});
+      renderRules("firewall-rules", s.firewall_rules || {});
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  async function loadSnapshots() {
+    const box = document.getElementById("fw-snapshots");
+    try {
+      const d = await api("/api/firewall/snapshots?limit=10");
+      const snaps = d.snapshots || [];
+      if (!snaps.length) {
+        box.innerHTML =
+          '<p class="muted">No SQLite snapshots yet — they are recorded on each <code>--update-firewall</code> run.</p>';
+        return;
+      }
+      box.innerHTML = "";
+      const rows = snaps.map((s) =>
+        el(
+          "tr",
+          el("td", { class: "mono", text: `#${s.id}` }),
+          el("td", { text: fmtTime(s.captured_at) }),
+          el("td", { text: s.kind }),
+          el("td", { text: s.trigger }),
+          el("td", { class: "num", text: s.addr_entries }),
+          el("td", { class: "num", text: s.rule_entries }),
+        ),
+      );
+      box.append(
+        el(
+          "table",
+          { class: "table" },
+          el(
+            "thead",
+            el("tr", {}, el("th", { text: "ID" }), el("th", { text: "Captured" }), el("th", { text: "Kind" }), el("th", { text: "Trigger" }), el("th", { text: "Addr" }), el("th", { text: "Rules" })),
+          ),
+          el("tbody", {}, ...rows),
+        ),
+      );
+    } catch (e) {
+      box.innerHTML=
+        '<p class="muted">Snapshot history unavailable (SQLite).</p>';
+    }
+  }
+
+  function renderAddressLists(addr) {
+    const box = document.getElementById("address-lists");
+    box.innerHTML = "";
+    const byList = {};
+    for (const version of ["ipv4", "ipv6"]) {
+      for (const e of addr[version] || []) {
+        const name = e.list || "?";
+        (byList[name] = byList[name] || { ipv4: [], ipv6: [] })[version].push(
+          e.address,
+        );
+      }
+    }
+    const names = Object.keys(byList);
+    if (!names.length) {
+      box.innerHTML =
+        '<p class="muted">No address-list snapshot available. Run <code>app.py --update-firewall</code> first.</p>';
+      return;
+    }
+    for (const name of names) {
+      const rows = [];
+      for (const version of ["ipv4", "ipv6"]) {
+        const entries = (byList[name][version] || []).sort();
+        rows.push(
+          el(
+            "tr",
+            el("td", { text: version }),
+            el("td", { class: "mono", text: entries.join(", ") || "—" }),
+          ),
+        );
+      }
+      box.append(
+        el(
+          "div",
+          { style: "margin-bottom:1rem" },
+          el("div", {
+            style: "font-weight:600;margin-bottom:.4rem",
+            text: name,
+          }),
+          el(
+            "table",
+            { class: "table" },
+            el(
+              "thead",
+              {},
+              el(
+                "tr",
+                {},
+                el("th", { text: "Version" }),
+                el("th", { text: "Addresses" }),
+              ),
+            ),
+            el("tbody", {}, ...rows),
+          ),
+        ),
+      );
+    }
+  }
+
+  function renderRules(boxId, rules) {
+    const box = document.getElementById(boxId);
+    box.innerHTML = "";
+    for (const version of ["ipv4", "ipv6"]) {
+      const list = rules[version] || [];
+      const rows = list.map((r) =>
+        el(
+          "tr",
+          el("td", {
+            class: "mono",
+            text: r.address ? r.address : r["src-address-list"] || "",
+          }),
+          el("td", { text: r.chain || "—" }),
+          el("td", { text: r.action || "—" }),
+          el("td", { text: r.comment || "" }),
+        ),
+      );
+      box.append(
+        el(
+          "div",
+          { style: "margin-bottom:1rem" },
+          el("div", {
+            style: "font-weight:600;margin-bottom:.4rem",
+            text: `IPv${version} (${list.length})`,
+          }),
+          rows.length
+            ? el(
+                "table",
+                { class: "table" },
+                el(
+                  "thead",
+                  {},
+                  el(
+                    "tr",
+                    {},
+                    el("th", { text: "Address / List" }),
+                    el("th", { text: "Chain" }),
+                    el("th", { text: "Action" }),
+                    el("th", { text: "Comment" }),
+                  ),
+                ),
+                el("tbody", {}, ...rows),
+              )
+            : el("p", { class: "muted", text: "No rules in snapshot." }),
+        ),
+      );
+    }
+  }
+
+  async function sync(dryRun) {
+    const out = document.getElementById("fw-result");
+    out.textContent = "Working…";
+    try {
+      const changes = await api(`/api/firewall/${dryRun ? "sync" : "apply"}`, {
+        method: "POST",
+        body: {},
+      });
+      const verb = dryRun ? "Would" : "";
+      out.textContent =
+        `${verb} add ${(changes.added_ipv4 || []).length} IPv4 + ${(changes.added_ipv6 || []).length} IPv6; ` +
+        `remove ${(changes.removed_ipv4 || []).length} IPv4 + ${(changes.removed_ipv6 || []).length} IPv6.`;
+      toast(
+        (dryRun ? "Preview" : "Applied") + " firewall sync",
+        dryRun ? "warn" : "ok",
+      );
+      if (!dryRun) loadState();
+    } catch (e) {
+      out.textContent = "";
+      toast(e.message, "error");
+    }
+  }
+
+  document
+    .getElementById("preview-btn")
+    .addEventListener("click", () => sync(true));
+  document
+    .getElementById("apply-btn")
+    .addEventListener("click", () => sync(false));
+  loadState();
+  loadSnapshots();
+</script>
+{% endblock %}

app/templates/graph.html

@@ -1,9 +1,8 @@
-{% extends "base.html" %} {% block title %}Graph — Intrusion Detection{%
-endblock %} {% block content %}
+{% extends "base.html" %} {% block title %}Graph — Intrusion Detection{% endblock %} {% block content %}
 <h1>IPv4 address space map</h1>
 <p class="muted">
-  One pixel per /24 network, arranged as 16×16 tiles of /8 (first octet).
-  Blocked targets are red, whitelisted ranges are green.
+  One pixel per /24 network, arranged as 16×16 tiles of /8 (first octet). Blocked targets are red, whitelisted ranges
+  are green.
 </p>
 
 <div class="panel">
@@ -14,8 +13,7 @@ endblock %} {% block content %}
         Show whitelisted
       </label>
       <span class="legend">
-        <span class="sw red"></span> Blocked
-        <span class="sw green"></span> Whitelisted
+        <span class="sw red"></span> Blocked <span class="sw green"></span> Whitelisted
         <span class="sw plain"></span> Unlisted
       </span>
     </div>
@@ -113,7 +111,7 @@ endblock %} {% block content %}
 
   function render() {
     const showWl = document.getElementById("show-whitelisted").checked;
-    ctx.fillStyle = "#ffffff";
+    ctx.fillStyle = "#0c0c0c";
     ctx.fillRect(0, 0, TOTAL, TOTAL);
     for (const [idx, value] of marked) {
       if (!showWl && value === V_WHITELISTED) continue;
@@ -142,37 +140,24 @@ endblock %} {% block content %}
     const c = y % SIZE;
     const idx = (a << 16) | (b << 8) | c;
     const value = marked.get(idx) ?? V_UNLISTED;
-    const status =
-      value === V_BLOCKED
-        ? "blocked"
-        : value === V_WHITELISTED
-          ? "whitelisted"
-          : "unlisted";
-    const entries = matchesIn(blockedRanges, idx).concat(
-      matchesIn(whitelistedRanges, idx),
-    );
+    const status = value === V_BLOCKED ? "blocked" : value === V_WHITELISTED ? "whitelisted" : "unlisted";
+    const entries = matchesIn(blockedRanges, idx).concat(matchesIn(whitelistedRanges, idx));
     tooltip.innerHTML = "";
     tooltip.append(
       el("div", { class: "tt-head mono", text: `${a}.${b}.${c}.0/24` }),
       el("div", { class: `tt-status ${status}`, text: status }),
     );
     if (entries.length) {
-      tooltip.append(
-        el("div", { class: "tt-list mono", text: entries.join("\n") }),
-      );
+      tooltip.append(el("div", { class: "tt-list mono", text: entries.join("\n") }));
     } else {
-      tooltip.append(
-        el("div", { class: "tt-list muted", text: "no matching entry" }),
-      );
+      tooltip.append(el("div", { class: "tt-list muted", text: "no matching entry" }));
     }
     tooltip.classList.remove("hidden");
     const rect = canvas.getBoundingClientRect();
     let left = e.clientX - rect.left + 14;
     let top = e.clientY - rect.top + 14;
-    if (left + tooltip.offsetWidth > rect.width)
-      left = e.clientX - rect.left - tooltip.offsetWidth - 14;
-    if (top + tooltip.offsetHeight > rect.height)
-      top = e.clientY - rect.top - tooltip.offsetHeight - 14;
+    if (left + tooltip.offsetWidth > rect.width) left = e.clientX - rect.left - tooltip.offsetWidth - 14;
+    if (top + tooltip.offsetHeight > rect.height) top = e.clientY - rect.top - tooltip.offsetHeight - 14;
     tooltip.style.left = `${Math.max(0, left)}px`;
     tooltip.style.top = `${Math.max(0, top)}px`;
   }
@@ -217,9 +202,7 @@ endblock %} {% block content %}
     }
   }
 
-  document
-    .getElementById("show-whitelisted")
-    .addEventListener("change", render);
+  document.getElementById("show-whitelisted").addEventListener("change", render);
   document.getElementById("graph-refresh").addEventListener("click", loadGraph);
 
   loadGraph();

app/templates/reports.html

@@ -1,5 +1,4 @@
-{% extends "base.html" %} {% block title %}Reports — Intrusion Detection{%
-endblock %} {% block content %}
+{% extends "base.html" %} {% block title %}Reports — Intrusion Detection{% endblock %} {% block content %}
 <h1>Reports</h1>
 <p class="muted">Browse the timestamped reports written by each scanner.</p>
 
@@ -21,38 +20,87 @@ endblock %} {% block content %}
 
 <div class="panel">
   <div class="panel-head"><h2 id="content-title">Report content</h2></div>
-  <pre class="content" id="content">
-Select a report on the left to view it.</pre
-  >
+  <pre class="content" id="content">Select a report on the left to view it.</pre>
 </div>
 {% endblock %} {% block scripts %}
 <script>
-  async function loadFiles() {
+  let nextCursor = null;
+
+  function addReportLink(list, label, onClick) {
+    const link = document.createElement("a");
+    link.href = "#";
+    link.textContent = label;
+    link.addEventListener("click", (event) => {
+      event.preventDefault();
+      onClick();
+    });
+    list.append(link);
+  }
+
+  async function loadFiles(append = false) {
     const source = document.getElementById("source").value;
     const list = document.getElementById("files");
-    list.innerHTML = '<div class="muted">Loading…</div>';
+    if (!append) {
+      nextCursor = null;
+      list.innerHTML = '<div class="muted">Loading…</div>';
+    } else {
+      document.getElementById("load-more")?.remove();
+    }
+
+    let importedPaths = new Set();
     try {
-      const data = await api(`/api/reports/${source}`);
-      list.innerHTML = "";
-      const files = data.files || [];
-      if (!files.length) {
-        list.innerHTML = '<div class="muted">No reports in this source.</div>';
-        return;
-      }
-      for (const name of files) {
-        list.append(
-          el("a", {
-            href: "#",
-            text: name,
-            onclick: (ev) => {
-              ev.preventDefault();
-              openReport(name);
-            },
-          }),
+      const query = new URLSearchParams({ source, limit: "50" });
+      if (append && nextCursor) query.set("cursor", nextCursor);
+      const data = await api(`/api/reports?${query}`);
+      if (!append) list.innerHTML = "";
+      for (const artifact of data.items || []) {
+        importedPaths.add(artifact.original_path);
+        addReportLink(list, `${artifact.generated_at} | ${artifact.report_type} | ${artifact.original_path}`, () =>
+          openArtifact(artifact.id),
         );
       }
+      nextCursor = data.next_cursor;
     } catch (e) {
-      list.innerHTML = `<div class="muted">${escapeHtml(e.message)}</div>`;
+      if (append) {
+        list.append(el("div", { className: "muted", text: e.message }));
+        return;
+      }
+    }
+
+    if (!append) {
+      try {
+        const data = await api(`/api/reports/${source}`);
+        for (const name of data.files || []) {
+          if (importedPaths.has(`${source}/${name}`)) continue;
+          addReportLink(list, `${name} | legacy file`, () => openReport(name));
+        }
+      } catch (e) {
+        if (!list.childElementCount) list.append(el("div", { className: "muted", text: e.message }));
+      }
+    }
+
+    if (nextCursor) {
+      const button = document.createElement("button");
+      button.id = "load-more";
+      button.className = "btn";
+      button.textContent = "Load more";
+      button.addEventListener("click", () => loadFiles(true));
+      list.append(button);
+    }
+    if (!list.childElementCount) list.innerHTML = '<div class="muted">No reports in this source.</div>';
+  }
+
+  async function openArtifact(id) {
+    document.getElementById("content-title").textContent = "Database report";
+    const pre = document.getElementById("content");
+    pre.textContent = "Loading…";
+    try {
+      const data = await api(`/api/report-artifacts/${id}`);
+      document.getElementById("content-title").textContent = data.original_path;
+      const payload = data.payload || {};
+      pre.textContent = payload.format === "text" ? payload.content || "" : JSON.stringify(payload, null, 2);
+    } catch (e) {
+      pre.textContent = `Error: ${e.message}`;
     }
   }
 
@@ -69,8 +117,8 @@ Select a report on the left to view it.</pre
     }
   }
 
-  document.getElementById("refresh-btn").addEventListener("click", loadFiles);
-  document.getElementById("source").addEventListener("change", loadFiles);
+  document.getElementById("refresh-btn").addEventListener("click", () => loadFiles());
+  document.getElementById("source").addEventListener("change", () => loadFiles());
   loadFiles();
 </script>
 {% endblock %}

app/templates/scan.html

@@ -1,185 +1,185 @@
-{% extends "base.html" %} {% block title %}Scan — Intrusion Detection{% endblock
-%} {% block content %}
-<h1>Run a scan</h1>
-<p class="muted">
-  Scans run in the background; only one at a time. You can leave this page and
-  come back.
-</p>
-
-<div class="panel">
-  <form id="scan-form" class="form-grid">
-    <label
-      >Type
-      <select id="kind">
-        <option value="consolidated">Consolidated (copyparty + npm)</option>
-        <option value="copyparty">copyparty only</option>
-        <option value="npm">npm only</option>
-      </select>
-    </label>
-    <label>Top IPs <input id="top" type="number" value="30" min="1" /></label>
-    <label
-      >Min denied (copyparty)
-      <input id="min_denied" type="number" value="1" min="0"
-    /></label>
-    <label
-      >Min errors (npm) <input id="min_errors" type="number" value="5" min="0"
-    /></label>
-    <label
-      >Error ratio (npm)
-      <input
-        id="error_ratio"
-        type="number"
-        step="0.05"
-        value="0.5"
-        min="0"
-        max="1"
-    /></label>
-    <label class="check"
-      ><input id="update_firewall" type="checkbox" /> Update firewall after
-      scan</label
-    >
-    <label class="check"
-      ><input id="dry_run" type="checkbox" /> Dry-run (preview, no
-      changes)</label
-    >
-    <label class="check"
-      ><input id="full" type="checkbox" /> Full scan (ignore incremental
-      state)</label
-    >
-  </form>
-  <div class="panel-actions">
-    <button class="btn primary" id="run-btn">Run scan</button>
-    <span class="muted" id="busy-hint"></span>
-  </div>
-  <div id="job-status" class="job-status"></div>
-</div>
-
-<div class="panel">
-  <div class="panel-head"><h2>Recent jobs</h2></div>
-  <table class="table">
-    <thead>
-      <tr>
-        <th>Job</th>
-        <th>Kind</th>
-        <th>State</th>
-        <th>Started</th>
-        <th>Finished</th>
-        <th class="num">Blocked</th>
-      </tr>
-    </thead>
-    <tbody id="jobs"></tbody>
-  </table>
-</div>
-{% endblock %} {% block scripts %}
-<script>
-  let pollTimer = null;
-
-  const form = document.getElementById("scan-form");
-  const runBtn = document.getElementById("run-btn");
-  const hint = document.getElementById("busy-hint");
-
-  async function runScan() {
-    const kind = document.getElementById("kind").value;
-    const params = {
-      top: parseInt(document.getElementById("top").value, 10) || 30,
-      min_denied:
-        parseInt(document.getElementById("min_denied").value, 10) || 1,
-      min_errors:
-        parseInt(document.getElementById("min_errors").value, 10) || 5,
-      error_ratio:
-        parseFloat(document.getElementById("error_ratio").value) || 0.5,
-      update_firewall: document.getElementById("update_firewall").checked,
-      dry_run: document.getElementById("dry_run").checked,
-      full: document.getElementById("full").checked,
-      quiet: true,
-    };
-    runBtn.disabled = true;
-    try {
-      const job = await startScan(kind, params);
-      setStatus(`Job ${job.job_id} started (${job.state}).`, "ok");
-      poll(job.job_id);
-    } catch (e) {
-      setStatus(e.message, "error");
-      runBtn.disabled = false;
-    }
-  }
-
-  function setStatus(msg, type) {
-    const box = document.getElementById("job-status");
-    box.textContent = msg;
-    box.style.color =
-      type === "error"
-        ? "var(--danger)"
-        : type === "ok"
-          ? "var(--ok)"
-          : "inherit";
-  }
-
-  async function poll(jobId) {
-    clearTimeout(pollTimer);
-    let job;
-    try {
-      job = await api(`/api/scan/${jobId}`);
-    } catch (e) {
-      setStatus(e.message, "error");
-      runBtn.disabled = false;
-      return;
-    }
-    const r = job.result || {};
-    setStatus(
-      `[${job.state}] ${job.kind} — started ${fmtTime(job.started)}` +
-        (job.finished ? `, finished ${fmtTime(job.finished)}` : "") +
-        (r.blocked_targets != null
-          ? `, blocked targets: ${r.blocked_targets}`
-          : "") +
-        (job.error ? `, error: ${job.error}` : ""),
-      job.state === "error" ? "error" : job.state === "done" ? "ok" : "",
-    );
-    if (job.state === "running" || job.state === "queued") {
-      pollTimer = setTimeout(() => poll(jobId), 2000);
-    } else {
-      runBtn.disabled = false;
-      refreshJobs();
-      const running = (await api("/api/scan/status")).running;
-      hint.textContent = running ? "" : "";
-    }
-  }
-
-  async function refreshJobs() {
-    try {
-      const s = await api("/api/scan/status");
-      hint.textContent = s.running ? "A scan is currently running…" : "";
-      const body = document.getElementById("jobs");
-      body.innerHTML = "";
-      const jobs = s.latest_scans || [];
-      if (!jobs.length) {
-        body.innerHTML =
-          '<tr><td colspan="6" class="muted">No jobs yet.</td></tr>';
-        return;
-      }
-      for (const j of jobs) {
-        const r = j.result || {};
-        body.append(
-          el(
-            "tr",
-            el("td", { class: "mono", text: j.job_id }),
-            el("td", { text: j.kind }),
-            el("td", {}, badge(j.state)),
-            el("td", { text: fmtTime(j.started) }),
-            el("td", { text: fmtTime(j.finished) }),
-            el("td", { class: "num", text: r.blocked_targets ?? "—" }),
-          ),
-        );
-      }
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  runBtn.addEventListener("click", runScan);
-  refreshJobs();
-  // /api/scan/status is only polled every 15 s; the active job's own
-  // 2 s poll keeps live progress without hammering the status endpoint.
-  setInterval(refreshJobs, 15000);
-</script>
-{% endblock %}
+{% extends "base.html" %} {% block title %}Scan — Intrusion Detection{% endblock
+%} {% block content %}
+<h1>Run a scan</h1>
+<p class="muted">
+  Scans run in the background; only one at a time. You can leave this page and
+  come back.
+</p>
+
+<div class="panel">
+  <form id="scan-form" class="form-grid">
+    <label
+      >Type
+      <select id="kind">
+        <option value="consolidated">Consolidated (copyparty + npm)</option>
+        <option value="copyparty">copyparty only</option>
+        <option value="npm">npm only</option>
+      </select>
+    </label>
+    <label>Top IPs <input id="top" type="number" value="30" min="1" /></label>
+    <label
+      >Min denied (copyparty)
+      <input id="min_denied" type="number" value="1" min="0"
+    /></label>
+    <label
+      >Min errors (npm) <input id="min_errors" type="number" value="5" min="0"
+    /></label>
+    <label
+      >Error ratio (npm)
+      <input
+        id="error_ratio"
+        type="number"
+        step="0.05"
+        value="0.5"
+        min="0"
+        max="1"
+    /></label>
+    <label class="check"
+      ><input id="update_firewall" type="checkbox" /> Update firewall after
+      scan</label
+    >
+    <label class="check"
+      ><input id="dry_run" type="checkbox" /> Dry-run (preview, no
+      changes)</label
+    >
+    <label class="check"
+      ><input id="full" type="checkbox" /> Full scan (ignore incremental
+      state)</label
+    >
+  </form>
+  <div class="panel-actions">
+    <button class="btn primary" id="run-btn">Run scan</button>
+    <span class="muted" id="busy-hint"></span>
+  </div>
+  <div id="job-status" class="job-status"></div>
+</div>
+
+<div class="panel">
+  <div class="panel-head"><h2>Recent jobs</h2></div>
+  <table class="table">
+    <thead>
+      <tr>
+        <th>Job</th>
+        <th>Kind</th>
+        <th>State</th>
+        <th>Started</th>
+        <th>Finished</th>
+        <th class="num">Blocked</th>
+      </tr>
+    </thead>
+    <tbody id="jobs"></tbody>
+  </table>
+</div>
+{% endblock %} {% block scripts %}
+<script>
+  let pollTimer = null;
+
+  const form = document.getElementById("scan-form");
+  const runBtn = document.getElementById("run-btn");
+  const hint = document.getElementById("busy-hint");
+
+  async function runScan() {
+    const kind = document.getElementById("kind").value;
+    const params = {
+      top: parseInt(document.getElementById("top").value, 10) || 30,
+      min_denied:
+        parseInt(document.getElementById("min_denied").value, 10) || 1,
+      min_errors:
+        parseInt(document.getElementById("min_errors").value, 10) || 5,
+      error_ratio:
+        parseFloat(document.getElementById("error_ratio").value) || 0.5,
+      update_firewall: document.getElementById("update_firewall").checked,
+      dry_run: document.getElementById("dry_run").checked,
+      full: document.getElementById("full").checked,
+      quiet: true,
+    };
+    runBtn.disabled = true;
+    try {
+      const job = await startScan(kind, params);
+      setStatus(`Job ${job.job_id} started (${job.state}).`, "ok");
+      poll(job.job_id);
+    } catch (e) {
+      setStatus(e.message, "error");
+      runBtn.disabled = false;
+    }
+  }
+
+  function setStatus(msg, type) {
+    const box = document.getElementById("job-status");
+    box.textContent = msg;
+    box.style.color =
+      type === "error"
+        ? "var(--danger)"
+        : type === "ok"
+          ? "var(--ok)"
+          : "inherit";
+  }
+
+  async function poll(jobId) {
+    clearTimeout(pollTimer);
+    let job;
+    try {
+      job = await api(`/api/scan/${jobId}`);
+    } catch (e) {
+      setStatus(e.message, "error");
+      runBtn.disabled = false;
+      return;
+    }
+    const r = job.result || {};
+    setStatus(
+      `[${job.state}] ${job.kind} — started ${fmtTime(job.started)}` +
+        (job.finished ? `, finished ${fmtTime(job.finished)}` : "") +
+        (r.blocked_targets != null
+          ? `, blocked targets: ${r.blocked_targets}`
+          : "") +
+        (job.error ? `, error: ${job.error}` : ""),
+      job.state === "error" ? "error" : job.state === "done" ? "ok" : "",
+    );
+    if (job.state === "running" || job.state === "queued") {
+      pollTimer = setTimeout(() => poll(jobId), 2000);
+    } else {
+      runBtn.disabled = false;
+      refreshJobs();
+      const running = (await api("/api/scan/status")).running;
+      hint.textContent = running ? "" : "";
+    }
+  }
+
+  async function refreshJobs() {
+    try {
+      const s = await api("/api/scan/status");
+      hint.textContent = s.running ? "A scan is currently running…" : "";
+      const body = document.getElementById("jobs");
+      body.innerHTML = "";
+      const jobs = s.latest_scans || [];
+      if (!jobs.length) {
+        body.innerHTML =
+          '<tr><td colspan="6" class="muted">No jobs yet.</td></tr>';
+        return;
+      }
+      for (const j of jobs) {
+        const r = j.result || {};
+        body.append(
+          el(
+            "tr",
+            el("td", { class: "mono", text: j.job_id }),
+            el("td", { text: j.kind }),
+            el("td", {}, badge(j.state)),
+            el("td", { text: fmtTime(j.started) }),
+            el("td", { text: fmtTime(j.finished) }),
+            el("td", { class: "num", text: r.blocked_targets ?? "—" }),
+          ),
+        );
+      }
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  runBtn.addEventListener("click", runScan);
+  refreshJobs();
+  // /api/scan/status is only polled every 15 s; the active job's own
+  // 2 s poll keeps live progress without hammering the status endpoint.
+  setInterval(refreshJobs, 15000);
+</script>
+{% endblock %}

app/templates/whitelist.html

@@ -1,110 +1,110 @@
-{% extends "base.html" %} {% block title %}Whitelist — Intrusion Detection{%
-endblock %} {% block content %}
-<h1>Whitelist</h1>
-<p class="muted">
-  Provider whitelists (from the router snapshot). Never add a whole cloud
-  provider to the blocklist.
-</p>
-
-<div class="panel">
-  <div class="panel-head"><h2>Add / remove CIDR</h2></div>
-  <div class="panel-actions" style="margin-top: 0">
-    <select id="wl-name">
-      {% for name in whitelist_names %}
-      <option value="{{ name }}">{{ name }}</option>
-      {% endfor %}
-    </select>
-    <input
-      id="wl-address"
-      placeholder="CIDR, e.g. 1.2.3.0/24"
-      style="min-width: 220px"
-    />
-    <button class="btn primary" id="wl-add-btn">Add</button>
-    <button class="btn danger" id="wl-remove-btn">Remove</button>
-  </div>
-</div>
-
-<div id="whitelists"></div>
-{% endblock %} {% block scripts %}
-<script>
-  async function loadWhitelists() {
-    try {
-      const data = await api("/api/whitelist");
-      const box = document.getElementById("whitelists");
-      box.innerHTML = "";
-      for (const name of data.names || []) {
-        const w = data.whitelists[name] || {};
-        const rows = [];
-        for (const version of ["ipv4", "ipv6"]) {
-          const entries = (w[version] || []).sort();
-          rows.push(
-            el(
-              "tr",
-              el("td", { text: version }),
-              el("td", { class: "mono", text: entries.join(", ") || "—" }),
-            ),
-          );
-        }
-        box.append(
-          el(
-            "div",
-            { class: "panel" },
-            el(
-              "div",
-              { class: "panel-head" },
-              el("h2", {
-                text: `${name} (${(w.ipv4 || []).length + (w.ipv6 || []).length})`,
-              }),
-            ),
-            el(
-              "table",
-              { class: "table" },
-              el(
-                "thead",
-                {},
-                el(
-                  "tr",
-                  {},
-                  el("th", { text: "Version" }),
-                  el("th", { text: "Addresses" }),
-                ),
-              ),
-              el("tbody", {}, ...rows),
-            ),
-          ),
-        );
-      }
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  async function act(action) {
-    const name = document.getElementById("wl-name").value;
-    const address = document.getElementById("wl-address").value.trim();
-    if (!address) {
-      toast("Enter a CIDR", "warn");
-      return;
-    }
-    try {
-      await api("/api/whitelist/update", {
-        method: "POST",
-        body: { name, address, action },
-      });
-      toast(`${action} ${address} on ${name}`, "ok");
-      document.getElementById("wl-address").value = "";
-      loadWhitelists();
-    } catch (e) {
-      toast(e.message, "error");
-    }
-  }
-
-  document
-    .getElementById("wl-add-btn")
-    .addEventListener("click", () => act("add"));
-  document
-    .getElementById("wl-remove-btn")
-    .addEventListener("click", () => act("remove"));
-  loadWhitelists();
-</script>
-{% endblock %}
+{% extends "base.html" %} {% block title %}Whitelist — Intrusion Detection{%
+endblock %} {% block content %}
+<h1>Whitelist</h1>
+<p class="muted">
+  Provider whitelists (from the router snapshot). Never add a whole cloud
+  provider to the blocklist.
+</p>
+
+<div class="panel">
+  <div class="panel-head"><h2>Add / remove CIDR</h2></div>
+  <div class="panel-actions" style="margin-top: 0">
+    <select id="wl-name">
+      {% for name in whitelist_names %}
+      <option value="{{ name }}">{{ name }}</option>
+      {% endfor %}
+    </select>
+    <input
+      id="wl-address"
+      placeholder="CIDR, e.g. 1.2.3.0/24"
+      style="min-width: 220px"
+    />
+    <button class="btn primary" id="wl-add-btn">Add</button>
+    <button class="btn danger" id="wl-remove-btn">Remove</button>
+  </div>
+</div>
+
+<div id="whitelists"></div>
+{% endblock %} {% block scripts %}
+<script>
+  async function loadWhitelists() {
+    try {
+      const data = await api("/api/whitelist");
+      const box = document.getElementById("whitelists");
+      box.innerHTML = "";
+      for (const name of data.names || []) {
+        const w = data.whitelists[name] || {};
+        const rows = [];
+        for (const version of ["ipv4", "ipv6"]) {
+          const entries = (w[version] || []).sort();
+          rows.push(
+            el(
+              "tr",
+              el("td", { text: version }),
+              el("td", { class: "mono", text: entries.join(", ") || "—" }),
+            ),
+          );
+        }
+        box.append(
+          el(
+            "div",
+            { class: "panel" },
+            el(
+              "div",
+              { class: "panel-head" },
+              el("h2", {
+                text: `${name} (${(w.ipv4 || []).length + (w.ipv6 || []).length})`,
+              }),
+            ),
+            el(
+              "table",
+              { class: "table" },
+              el(
+                "thead",
+                {},
+                el(
+                  "tr",
+                  {},
+                  el("th", { text: "Version" }),
+                  el("th", { text: "Addresses" }),
+                ),
+              ),
+              el("tbody", {}, ...rows),
+            ),
+          ),
+        );
+      }
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  async function act(action) {
+    const name = document.getElementById("wl-name").value;
+    const address = document.getElementById("wl-address").value.trim();
+    if (!address) {
+      toast("Enter a CIDR", "warn");
+      return;
+    }
+    try {
+      await api("/api/whitelist/update", {
+        method: "POST",
+        body: { name, address, action },
+      });
+      toast(`${action} ${address} on ${name}`, "ok");
+      document.getElementById("wl-address").value = "";
+      loadWhitelists();
+    } catch (e) {
+      toast(e.message, "error");
+    }
+  }
+
+  document
+    .getElementById("wl-add-btn")
+    .addEventListener("click", () => act("add"));
+  document
+    .getElementById("wl-remove-btn")
+    .addEventListener("click", () => act("remove"));
+  loadWhitelists();
+</script>
+{% endblock %}

app/utils/__init__.py

@@ -0,0 +1,13 @@
+# Intrusion Detection System — shared utilities
+#
+# Modules:
+#   constants.py          central configuration, paths, env vars
+#   files.py              filesystem & I/O helpers
+#   ip_utils.py           IP address parsing, validation, whitelist lookup
+#   models.py             data models (Request, IPStats) + aggregate()
+#   reporting.py          text rendering, table formatting
+#   utils.py              shared CLI arg parser
+#   whitelists.py         whitelist CIDR download, caching, validation
+#   whois.py              WHOIS subnet queries, IP consolidation
+#   blocklist_history.py  git-backed cumulative blocklist reconstruction
+#   retention.py          report file retention & pruning

app/utils/blocklist_history.py

@@ -0,0 +1,202 @@
+#!/usr/bin/env python3
+"""Git-backed cumulative blocklist history.
+
+The consolidated blocklist lives in git-tracked files
+(``reports/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``). To make the
+MikroTik blocklist grow-only, the "complete set of all blocked IPs" is
+reconstructed from **every git revision** of those files (the union of all
+lines ever present), plus the current working files and the latest scan
+targets.
+
+This keeps IPs blocked even after they leave the NPM ~30-day log window, and
+only ever removes router entries that were never blocked by the system.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import os
+import re
+import subprocess
+from datetime import datetime
+from typing import Any, Iterable
+import hashlib
+
+from .files import write_json
+from .reporting import timestamp_name
+
+#: A ``+ 1.2.3.4/32`` added line in a git patch; ``+++ b/...`` is the file
+#: header and is excluded by the negative lookahead.
+_ADDED_LINE_RE = re.compile(r"^\+(?!\+)(.*)$")
+
+
+def _run_git(*args: str) -> subprocess.CompletedProcess[str] | None:
+    """Run a git command, returning None on any failure (never raises)."""
+    try:
+        return subprocess.run(
+            ["git", *args], capture_output=True, text=True, check=False)
+    except (OSError, subprocess.SubprocessError):
+        return None
+
+
+def _valid_line(line: str) -> str | None:
+    """Normalize a blocklist line to a canonical CIDR string, or None."""
+    line = line.strip()
+    if not line:
+        return None
+    try:
+        return str(ipaddress.ip_network(line, strict=False))
+    except ValueError:
+        return None
+
+
+def committed_lines(path: str) -> set[str]:
+    """Return the union of all lines ever present in ``path`` across git.
+
+    Uses a single ``git log --all -p -- <path>`` and collects the ``+`` added
+    lines (excluding the ``+++`` file header). A line present in any revision
+    appears as an added line in some commit's diff, so the union equals "all
+    lines ever present".
+
+    Returns an empty set when git is unavailable, the path is untracked, or the
+    repository has no history — callers then fall back to the working file.
+    """
+    proc = _run_git("log", "--all", "-p", "--", path)
+    if proc is None or proc.returncode != 0:
+        return set()
+    lines: set[str] = set()
+    for raw in proc.stdout.splitlines():
+        m = _ADDED_LINE_RE.match(raw)
+        if not m:
+            continue
+        clean = _valid_line(m.group(1))
+        if clean is not None:
+            lines.add(clean)
+    return lines
+
+
+def current_file_lines(path: str) -> set[str]:
+    """Return the validated lines currently in ``path`` (missing file -> {})."""
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return {clean for clean in (_valid_line(x) for x in f)
+                    if clean is not None}
+    except OSError:
+        return set()
+
+
+def cumulative_blocklist(v4_path: str, v6_path: str,
+                         desired_v4: Iterable[str],
+                         desired_v6: Iterable[str],
+                         ) -> tuple[set[str], set[str]]:
+    """Complete set of all blocked targets (history | working file | desired).
+
+    Returns ``(complete_v4, complete_v6)`` of canonical CIDR strings.
+    """
+    complete_v4 = (
+        committed_lines(v4_path)
+        | current_file_lines(v4_path)
+        | {c for c in (_valid_line(x) for x in desired_v4) if c is not None})
+    complete_v6 = (
+        committed_lines(v6_path)
+        | current_file_lines(v6_path)
+        | {c for c in (_valid_line(x) for x in desired_v6) if c is not None})
+    return complete_v4, complete_v6
+
+
+def compute_changes(current_v4: set[str], current_v6: set[str],
+                    complete_v4: set[str], complete_v6: set[str],
+                    whitelisted_v4: set[str], whitelisted_v6: set[str],
+                    ) -> dict[str, Any]:
+    """Compute the add/remove diff for a cumulative (grow-only) blocklist.
+
+    Entries are added when present in the complete set but not yet on the
+    router and not whitelisted. Entries are removed only when they sit on the
+    router but are absent from the complete set — i.e. they were never blocked
+    by the system (manual/rogue entries). Callers may attach extra keys (e.g.
+    ``dry_run``).
+    """
+    return {
+        "added_ipv4": sorted(complete_v4 - current_v4 - whitelisted_v4),
+        "added_ipv6": sorted(complete_v6 - current_v6 - whitelisted_v6),
+        "removed_ipv4": sorted(current_v4 - complete_v4),
+        "removed_ipv6": sorted(current_v6 - complete_v6),
+    }
+
+
+def blocklist_delta(history_v4: set[str], history_v6: set[str],
+                    prev_v4: set[str], prev_v6: set[str],
+                    complete_v4: set[str], complete_v6: set[str],
+                    ) -> dict[str, list[str]]:
+    """Complete-set delta between the previous and current blocklist state.
+
+    ``added`` are entries newly part of the blocklist (this run's targets not
+    blocked before); ``removed`` are entries no longer part of it — normally
+    empty in the cumulative model.
+    """
+    before_v4 = history_v4 | prev_v4
+    before_v6 = history_v6 | prev_v6
+    return {
+        "added_ipv4": sorted(complete_v4 - before_v4),
+        "added_ipv6": sorted(complete_v6 - before_v6),
+        "removed_ipv4": sorted(before_v4 - complete_v4),
+        "removed_ipv6": sorted(before_v6 - complete_v6),
+    }
+
+
+def write_blocklist_report(report_dir: str,
+                           *,
+                           history_v4: set[str], history_v6: set[str],
+                           prev_v4: set[str], prev_v6: set[str],
+                           complete_v4: set[str], complete_v6: set[str],
+                           firewall_changes: dict | None = None,
+                           dry_run: bool = False,
+                           errors: Iterable[str] = (),
+                           scan_id: int | None = None,
+                           ) -> str:
+    """Write the timestamped blocklist report to ``<report_dir>/app/``.
+
+    Records the complete-set delta (added/removed) plus the full cumulative
+    blocked list (git history | working files | scan targets), and — when
+    supplied — the router sync changes. Returns the written file path.
+    """
+    delta = blocklist_delta(
+        history_v4, history_v6, prev_v4, prev_v6, complete_v4, complete_v6)
+    payload = {
+        "generated_at": datetime.now().astimezone().isoformat(
+            timespec="seconds"),
+        "report_dir": report_dir,
+        "dry_run": dry_run,
+        "summary": {
+            "added": len(delta["added_ipv4"]) + len(delta["added_ipv6"]),
+            "removed": len(delta["removed_ipv4"]) + len(delta["removed_ipv6"]),
+            "total_blocked": len(complete_v4) + len(complete_v6),
+        },
+        "added_ipv4": delta["added_ipv4"],
+        "added_ipv6": delta["added_ipv6"],
+        "removed_ipv4": delta["removed_ipv4"],
+        "removed_ipv6": delta["removed_ipv6"],
+        "blocked_ipv4": sorted(complete_v4),
+        "blocked_ipv6": sorted(complete_v6),
+        "firewall_changes": firewall_changes,
+        "errors": list(errors),
+    }
+    app_dir = os.path.join(report_dir, "app")
+    os.makedirs(app_dir, exist_ok=True)
+    path = os.path.join(app_dir, f"{timestamp_name()}.json")
+    write_json(path, payload)
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        with open(path, "rb") as file:
+            content = file.read()
+        repo.insert_report_artifact(
+            get_connection(), source="app", report_type="blocklist_change",
+            generated_at=payload["generated_at"], payload=payload,
+            original_path=f"app/{os.path.basename(path)}",
+            content_sha256=hashlib.sha256(content).hexdigest(),
+            scan_id=scan_id,
+        )
+    except Exception as exc:
+        print(f"WARN: blocklist report DB write skipped ({exc})")
+    return path

app/utils/constants.py

@@ -53,6 +53,11 @@ MIKROTIK_DATA_DIR = os.path.join(DATA_DIR, "mikrotik")
 
 WHOIS_CACHE_PATH = os.path.join(DATA_DIR, "whois_cache.json")
 
+# Local SQLite database (see .github/instructions/DATABASE.md). The default
+# lives under data/ and holds log records, scan metadata, blocklist/whitelist
+# and firewall history. Override with IDS_DB_PATH (used by app/db).
+DATA_DB_PATH = os.environ.get("IDS_DB_PATH", os.path.join(DATA_DIR, "ids.db"))
+
 # Incremental-scan state: absolute log path -> last processed byte offset
 SCAN_STATE_PATH = os.path.join(DATA_DIR, "scan_state.json")
 
@@ -62,12 +67,20 @@ SCAN_STATE_PATH = os.path.join(DATA_DIR, "scan_state.json")
 SCAN_JOBS_PATH = os.path.join(DATA_DIR, "scan_jobs.json")
 SCAN_JOBS_MAX = 50
 
+# Report retention: per-directory cleanup of generated files under /reports/.
+# Each directory keeps at most REPORT_RETENTION_MAX_FILES files, all newer
+# than REPORT_RETENTION_DAYS days (see app/retention.py).
+REPORT_RETENTION_DAYS = int(os.environ.get("REPORT_RETENTION_DAYS", "30"))
+REPORT_RETENTION_MAX_FILES = int(
+    os.environ.get("REPORT_RETENTION_MAX_FILES", "10"))
+
 # ---------------------------------------------------------------------------
 # Log sources (configurable via environment)
 # ---------------------------------------------------------------------------
 
-COPYPARTY_LOG_PATH = os.environ.get("COPYPARTY_LOG_PATH", r"P:\copyparty")
-NPM_LOG_PATH = os.environ.get("NPM_LOG_PATH", r"Y:\npm\logs")
+COPYPARTY_LOG_PATH = os.environ.get(
+    "COPYPARTY_LOG_PATH", "/srv/projects/copyparty")
+NPM_LOG_PATH = os.environ.get("NPM_LOG_PATH", "/mnt/backups/npm/logs")
 
 # ---------------------------------------------------------------------------
 # MikroTik router connection (configurable via environment)
@@ -84,7 +97,7 @@ BLOCKLIST_NAME = os.environ.get("BLOCKLIST_NAME", "Blocked")
 WHITELIST_NAMES = [
     name.strip()
     for name in os.environ.get(
-        "WHITELIST_NAMES", "Cloudflare,Openfactory").split(",")
+        "WHITELIST_NAMES", "Cloudflare,Init7").split(",")
     if name.strip()
 ]
 
@@ -108,7 +121,7 @@ DEFAULT_MIN_DENIED = int(os.environ.get(
     "DEFAULT_MIN_DENIED", "1"))  # copyparty
 DEFAULT_MIN_ERRORS = int(os.environ.get("DEFAULT_MIN_ERRORS", "5"))  # npm
 DEFAULT_ERROR_RATIO = float(os.environ.get(
-    "DEFAULT_ERROR_RATIO", "0.5"))  # npm
+    "DEFAULT_ERROR_RATIO", "0.4"))  # npm
 DEFAULT_TOP_PATHS = int(os.environ.get("DEFAULT_TOP_PATHS", "20"))
 
 # ---------------------------------------------------------------------------
@@ -124,8 +137,6 @@ WEB_PORT = int(os.environ.get("WEB_PORT", "54666"))
 # Whitelist sources (configurable via environment)
 # ---------------------------------------------------------------------------
 
-OPENFACTORY_URL = os.environ.get(
-    "OPENFACTORY_URL", "https://openfactory.net/geofeed.csv")
 CLOUDFLARE_URL_V4 = os.environ.get(
     "CLOUDFLARE_URL_V4", "https://www.cloudflare.com/ips-v4")
 CLOUDFLARE_URL_V6 = os.environ.get(
@@ -136,7 +147,6 @@ PALOALTO_URL = os.environ.get(
 )
 
 # Local whitelist files (checked before remote download)
-OPENFACTORY_FILE = os.path.join(DATA_DIR, "openfactory", "geofeed.csv")
 CLOUDFLARE_V4_FILE = os.path.join(DATA_DIR, "cloudflare", "ips-v4.txt")
 CLOUDFLARE_V6_FILE = os.path.join(DATA_DIR, "cloudflare", "ips-v6.txt")
 PALOALTO_FILE = os.path.join(DATA_DIR, "paloalto", "scanner-ips.txt")

app/utils/files.py

@@ -1,104 +1,104 @@
-#!/usr/bin/env python3
-"""Filesystem and I/O helpers for the intrusion-detection system."""
-
-from __future__ import annotations
-
-import csv
-import gzip
-import json
-import os
-import re
-from typing import Any, Iterable, Iterator, Sequence
-
-ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
-
-
-# -- directory --------------------------------------------------------------
-
-def validate_dir(path: str, need_write: bool = False) -> str:
-    if not os.path.exists(path):
-        raise FileNotFoundError(f"path does not exist: {path}")
-    if not os.path.isdir(path):
-        raise NotADirectoryError(f"path is not a directory: {path}")
-    if not os.access(path, os.R_OK):
-        raise PermissionError(f"path not readable: {path}")
-    if need_write and not os.access(path, os.W_OK):
-        raise PermissionError(f"path not writable: {path}")
-    return path
-
-
-def ensure_dirs(*paths: str) -> None:
-    for p in paths:
-        os.makedirs(p, exist_ok=True)
-
-
-# -- log discovery ----------------------------------------------------------
-
-def find_logs(path: str, pattern: re.Pattern | None = None,
-              suffixes: Sequence[str] = (),
-              exclude_prefixes: Sequence[str] = ()) -> list[str]:
-    """List matching log file names (not full paths), sorted."""
-    out = []
-    for name in os.listdir(path):
-        if not os.path.isfile(os.path.join(path, name)):
-            continue
-        if any(name.startswith(p) for p in exclude_prefixes):
-            continue
-        if pattern is not None and not pattern.match(name):
-            continue
-        if suffixes and not name.endswith(tuple(suffixes)):
-            continue
-        out.append(name)
-    return sorted(out)
-
-
-def read_lines(path: str) -> Iterator[str]:
-    """Yield ANSI-stripped, newline-stripped lines. Transparently gunzips."""
-    opener = gzip.open if path.endswith(".gz") else open
-    with opener(path, "rt", encoding="utf-8", errors="replace") as f:
-        for line in f:
-            yield ANSI_RE.sub("", line.rstrip("\r\n"))
-
-
-def read_lines_from(path: str, start: int = 0) -> Iterator[tuple[str, int]]:
-    """Yield ``(line, resume_offset)`` for a plain-text log from byte ``start``.
-
-    Only seekable (non-gzip) files can be resumed. ``resume_offset`` points
-    after the last complete line; a trailing partial line (no newline yet) is
-    yielded but resets the resume offset to its start so it is re-read once
-    completed. The last yielded ``resume_offset`` is the value to store.
-    """
-    with open(path, "rb") as f:
-        f.seek(start)
-        pos = start
-        for raw in f:
-            line_start = pos
-            pos += len(raw)
-            complete = raw.endswith(b"\n") or raw.endswith(b"\r")
-            text = ANSI_RE.sub(
-                "", raw.decode("utf-8", errors="replace").rstrip("\r\n"))
-            yield text, (pos if complete else line_start)
-
-
-# -- writers ----------------------------------------------------------------
-
-def write_csv(path: str, header: Sequence[str],
-              rows: Iterable[Sequence[Any]]) -> str:
-    with open(path, "w", newline="", encoding="utf-8") as f:
-        w = csv.writer(f)
-        w.writerow(header)
-        w.writerows(rows)
-    return path
-
-
-def write_json(path: str, obj: Any) -> str:
-    with open(path, "w", encoding="utf-8") as f:
-        json.dump(obj, f, indent=2, default=str)
-    return path
-
-
-def write_lines(path: str, lines: Iterable[str]) -> str:
-    with open(path, "w", encoding="utf-8") as f:
-        for line in lines:
-            f.write(line + "\n")
-    return path
+#!/usr/bin/env python3
+"""Filesystem and I/O helpers for the intrusion-detection system."""
+
+from __future__ import annotations
+
+import csv
+import gzip
+import json
+import os
+import re
+from typing import Any, Iterable, Iterator, Sequence
+
+ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+
+# -- directory --------------------------------------------------------------
+
+def validate_dir(path: str, need_write: bool = False) -> str:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"path does not exist: {path}")
+    if not os.path.isdir(path):
+        raise NotADirectoryError(f"path is not a directory: {path}")
+    if not os.access(path, os.R_OK):
+        raise PermissionError(f"path not readable: {path}")
+    if need_write and not os.access(path, os.W_OK):
+        raise PermissionError(f"path not writable: {path}")
+    return path
+
+
+def ensure_dirs(*paths: str) -> None:
+    for p in paths:
+        os.makedirs(p, exist_ok=True)
+
+
+# -- log discovery ----------------------------------------------------------
+
+def find_logs(path: str, pattern: re.Pattern | None = None,
+              suffixes: Sequence[str] = (),
+              exclude_prefixes: Sequence[str] = ()) -> list[str]:
+    """List matching log file names (not full paths), sorted."""
+    out = []
+    for name in os.listdir(path):
+        if not os.path.isfile(os.path.join(path, name)):
+            continue
+        if any(name.startswith(p) for p in exclude_prefixes):
+            continue
+        if pattern is not None and not pattern.match(name):
+            continue
+        if suffixes and not name.endswith(tuple(suffixes)):
+            continue
+        out.append(name)
+    return sorted(out)
+
+
+def read_lines(path: str) -> Iterator[str]:
+    """Yield ANSI-stripped, newline-stripped lines. Transparently gunzips."""
+    opener = gzip.open if path.endswith(".gz") else open
+    with opener(path, "rt", encoding="utf-8", errors="replace") as f:
+        for line in f:
+            yield ANSI_RE.sub("", line.rstrip("\r\n"))
+
+
+def read_lines_from(path: str, start: int = 0) -> Iterator[tuple[str, int]]:
+    """Yield ``(line, resume_offset)`` for a plain-text log from byte ``start``.
+
+    Only seekable (non-gzip) files can be resumed. ``resume_offset`` points
+    after the last complete line; a trailing partial line (no newline yet) is
+    yielded but resets the resume offset to its start so it is re-read once
+    completed. The last yielded ``resume_offset`` is the value to store.
+    """
+    with open(path, "rb") as f:
+        f.seek(start)
+        pos = start
+        for raw in f:
+            line_start = pos
+            pos += len(raw)
+            complete = raw.endswith(b"\n") or raw.endswith(b"\r")
+            text = ANSI_RE.sub(
+                "", raw.decode("utf-8", errors="replace").rstrip("\r\n"))
+            yield text, (pos if complete else line_start)
+
+
+# -- writers ----------------------------------------------------------------
+
+def write_csv(path: str, header: Sequence[str],
+              rows: Iterable[Sequence[Any]]) -> str:
+    with open(path, "w", newline="", encoding="utf-8") as f:
+        w = csv.writer(f)
+        w.writerow(header)
+        w.writerows(rows)
+    return path
+
+
+def write_json(path: str, obj: Any) -> str:
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(obj, f, indent=2, default=str)
+    return path
+
+
+def write_lines(path: str, lines: Iterable[str]) -> str:
+    with open(path, "w", encoding="utf-8") as f:
+        for line in lines:
+            f.write(line + "\n")
+    return path

app/utils/ip_utils.py

@@ -1,63 +1,63 @@
-#!/usr/bin/env python3
-"""IP helper functions for the intrusion-detection system."""
-
-from __future__ import annotations
-
-import ipaddress
-from typing import Iterable
-
-from .whitelists import WHITELIST_NETS as _WHITELIST_NETS
-
-
-def parse_ip(value: str):
-    """Return an ip_address object, or None if `value` is not an IP."""
-    try:
-        return ipaddress.ip_address(value)
-    except (ValueError, TypeError):
-        return None
-
-
-def ip_version(value: str) -> int | None:
-    addr = parse_ip(value)
-    return addr.version if addr else None
-
-
-def is_ipv6(value: str) -> bool:
-    return ip_version(value) == 6
-
-
-def is_ipv4(value: str) -> bool:
-    return ip_version(value) == 4
-
-
-def whitelist_of(value: str) -> str | None:
-    """Return the whitelist name an IP belongs to, else None."""
-    addr = parse_ip(value)
-    if addr is None:
-        return None
-    for name, nets in _WHITELIST_NETS.items():
-        for net in nets:
-            if addr.version == net.version and addr in net:
-                return name
-    return None
-
-
-def is_whitelisted(value: str) -> bool:
-    return whitelist_of(value) is not None
-
-
-def sort_ips(ips: Iterable[str]) -> list[str]:
-    """Numeric sort, IPv4 before IPv6, unparseable last."""
-    def key(ip: str):
-        addr = parse_ip(ip)
-        if addr is None:
-            return (9, b"", ip)
-        return (addr.version, addr.packed, "")
-    return sorted(ips, key=key)
-
-
-def split_versions(ips: Iterable[str]) -> tuple[list[str], list[str]]:
-    """Split an IP iterable into (sorted IPv4, sorted IPv6)."""
-    ips = list(ips)
-    return (sort_ips(ip for ip in ips if is_ipv4(ip)),
-            sort_ips(ip for ip in ips if is_ipv6(ip)))
+#!/usr/bin/env python3
+"""IP helper functions for the intrusion-detection system."""
+
+from __future__ import annotations
+
+import ipaddress
+from typing import Iterable
+
+from .whitelists import WHITELIST_NETS as _WHITELIST_NETS
+
+
+def parse_ip(value: str):
+    """Return an ip_address object, or None if `value` is not an IP."""
+    try:
+        return ipaddress.ip_address(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def ip_version(value: str) -> int | None:
+    addr = parse_ip(value)
+    return addr.version if addr else None
+
+
+def is_ipv6(value: str) -> bool:
+    return ip_version(value) == 6
+
+
+def is_ipv4(value: str) -> bool:
+    return ip_version(value) == 4
+
+
+def whitelist_of(value: str) -> str | None:
+    """Return the whitelist name an IP belongs to, else None."""
+    addr = parse_ip(value)
+    if addr is None:
+        return None
+    for name, nets in _WHITELIST_NETS.items():
+        for net in nets:
+            if addr.version == net.version and addr in net:
+                return name
+    return None
+
+
+def is_whitelisted(value: str) -> bool:
+    return whitelist_of(value) is not None
+
+
+def sort_ips(ips: Iterable[str]) -> list[str]:
+    """Numeric sort, IPv4 before IPv6, unparseable last."""
+    def key(ip: str):
+        addr = parse_ip(ip)
+        if addr is None:
+            return (9, b"", ip)
+        return (addr.version, addr.packed, "")
+    return sorted(ips, key=key)
+
+
+def split_versions(ips: Iterable[str]) -> tuple[list[str], list[str]]:
+    """Split an IP iterable into (sorted IPv4, sorted IPv6)."""
+    ips = list(ips)
+    return (sort_ips(ip for ip in ips if is_ipv4(ip)),
+            sort_ips(ip for ip in ips if is_ipv6(ip)))

app/utils/models.py

@@ -1,150 +1,150 @@
-#!/usr/bin/env python3
-"""Data models for the intrusion-detection system.
-
-`Request` and `IPStats` plus the `aggregate()` fold function.
-"""
-
-from __future__ import annotations
-
-from collections import Counter
-from dataclasses import dataclass, field
-from typing import Iterable
-
-from . import bad_patterns
-from .ip_utils import ip_version, whitelist_of
-
-
-@dataclass(slots=True)
-class Request:
-    """One normalised request line, produced by any log parser."""
-
-    ts: str = ""
-    ip: str = ""
-    method: str | None = None
-    path: str | None = None
-    status: int | None = None
-    upstream_status: int | None = None
-    host: str | None = None
-    user: str | None = None
-    ua: str | None = None
-    referer: str | None = None
-    denied: bool = False
-    banned: bool = False
-    extra: dict = field(default_factory=dict)
-
-    @property
-    def is_client_error(self) -> bool:
-        return self.status is not None and 400 <= self.status < 500
-
-    @property
-    def is_server_error(self) -> bool:
-        return self.status is not None and 500 <= self.status < 600
-
-    @property
-    def is_error(self) -> bool:
-        return self.denied or self.is_client_error or self.is_server_error
-
-
-@dataclass
-class IPStats:
-    """Aggregated per-IP counters."""
-
-    ip: str
-    version: int = 0
-    total: int = 0
-    errors: int = 0
-    c4xx: int = 0
-    c5xx: int = 0
-    denied: int = 0
-    banned: int = 0
-    first: str = ""
-    last: str = ""
-    paths: Counter = field(default_factory=Counter)
-    methods: Counter = field(default_factory=Counter)
-    hosts: Counter = field(default_factory=Counter)
-    uas: Counter = field(default_factory=Counter)
-    users: set[str] = field(default_factory=set)
-    malicious_hits: int = 0
-    malicious_patterns: dict[str, int] = field(default_factory=dict)
-
-    @property
-    def error_ratio(self) -> float:
-        return round(self.errors / self.total, 3) if self.total else 0.0
-
-    @property
-    def authenticated(self) -> bool:
-        """True if the IP ever presented a real (non-anonymous) user."""
-        return any(u and u != "*" for u in self.users)
-
-    @property
-    def whitelist(self) -> str:
-        return whitelist_of(self.ip) or ""
-
-    @property
-    def users_str(self) -> str:
-        return ",".join(sorted(self.users)) or "-"
-
-    @property
-    def methods_str(self) -> str:
-        return ",".join(sorted(self.methods)) or "-"
-
-    def to_dict(self) -> dict:
-        return {
-            "ip": self.ip,
-            "version": self.version,
-            "total": self.total,
-            "errors": self.errors,
-            "error_ratio": self.error_ratio,
-            "status_4xx": self.c4xx,
-            "status_5xx": self.c5xx,
-            "denied": self.denied,
-            "banned": self.banned,
-            "first": self.first,
-            "last": self.last,
-            "whitelist": self.whitelist or None,
-            "users": sorted(self.users),
-            "methods": dict(self.methods),
-            "hosts": dict(self.hosts),
-            "paths": dict(self.paths.most_common()),
-            "uas": dict(self.uas.most_common(10)),
-            "malicious_hits": self.malicious_hits,
-            "malicious_patterns": dict(self.malicious_patterns),
-        }
-
-
-def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]:
-    """Fold request records into per-IP statistics."""
-    stats: dict[str, IPStats] = {}
-    for r in requests:
-        s = stats.get(r.ip)
-        if s is None:
-            s = stats[r.ip] = IPStats(
-                ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts)
-        s.total += 1
-        s.last = r.ts
-        if r.denied:
-            s.denied += 1
-        if r.banned:
-            s.banned += 1
-        if r.is_client_error:
-            s.c4xx += 1
-        if r.is_server_error:
-            s.c5xx += 1
-        if r.is_error:
-            s.errors += 1
-        if r.path:
-            s.paths[r.path] += 1
-        if r.method:
-            s.methods[r.method] += 1
-        if r.host:
-            s.hosts[r.host] += 1
-        if r.ua and r.ua != "-":
-            s.uas[r.ua] += 1
-        if r.user:
-            s.users.add(r.user)
-        if r.path and bad_patterns.is_malicious(r.path):
-            s.malicious_hits += 1
-            for p in bad_patterns.classify_path(r.path):
-                s.malicious_patterns[p.name] = s.malicious_patterns.get(
-                    p.name, 0) + 1
-    return stats
+#!/usr/bin/env python3
+"""Data models for the intrusion-detection system.
+
+`Request` and `IPStats` plus the `aggregate()` fold function.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Iterable
+
+from .. import bad_patterns
+from .ip_utils import ip_version, whitelist_of
+
+
+@dataclass(slots=True)
+class Request:
+    """One normalised request line, produced by any log parser."""
+
+    ts: str = ""
+    ip: str = ""
+    method: str | None = None
+    path: str | None = None
+    status: int | None = None
+    upstream_status: int | None = None
+    host: str | None = None
+    user: str | None = None
+    ua: str | None = None
+    referer: str | None = None
+    denied: bool = False
+    banned: bool = False
+    extra: dict = field(default_factory=dict)
+
+    @property
+    def is_client_error(self) -> bool:
+        return self.status is not None and 400 <= self.status < 500
+
+    @property
+    def is_server_error(self) -> bool:
+        return self.status is not None and 500 <= self.status < 600
+
+    @property
+    def is_error(self) -> bool:
+        return self.denied or self.is_client_error or self.is_server_error
+
+
+@dataclass
+class IPStats:
+    """Aggregated per-IP counters."""
+
+    ip: str
+    version: int = 0
+    total: int = 0
+    errors: int = 0
+    c4xx: int = 0
+    c5xx: int = 0
+    denied: int = 0
+    banned: int = 0
+    first: str = ""
+    last: str = ""
+    paths: Counter = field(default_factory=Counter)
+    methods: Counter = field(default_factory=Counter)
+    hosts: Counter = field(default_factory=Counter)
+    uas: Counter = field(default_factory=Counter)
+    users: set[str] = field(default_factory=set)
+    malicious_hits: int = 0
+    malicious_patterns: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def error_ratio(self) -> float:
+        return round(self.errors / self.total, 3) if self.total else 0.0
+
+    @property
+    def authenticated(self) -> bool:
+        """True if the IP ever presented a real (non-anonymous) user."""
+        return any(u and u != "*" for u in self.users)
+
+    @property
+    def whitelist(self) -> str:
+        return whitelist_of(self.ip) or ""
+
+    @property
+    def users_str(self) -> str:
+        return ",".join(sorted(self.users)) or "-"
+
+    @property
+    def methods_str(self) -> str:
+        return ",".join(sorted(self.methods)) or "-"
+
+    def to_dict(self) -> dict:
+        return {
+            "ip": self.ip,
+            "version": self.version,
+            "total": self.total,
+            "errors": self.errors,
+            "error_ratio": self.error_ratio,
+            "status_4xx": self.c4xx,
+            "status_5xx": self.c5xx,
+            "denied": self.denied,
+            "banned": self.banned,
+            "first": self.first,
+            "last": self.last,
+            "whitelist": self.whitelist or None,
+            "users": sorted(self.users),
+            "methods": dict(self.methods),
+            "hosts": dict(self.hosts),
+            "paths": dict(self.paths.most_common()),
+            "uas": dict(self.uas.most_common(10)),
+            "malicious_hits": self.malicious_hits,
+            "malicious_patterns": dict(self.malicious_patterns),
+        }
+
+
+def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]:
+    """Fold request records into per-IP statistics."""
+    stats: dict[str, IPStats] = {}
+    for r in requests:
+        s = stats.get(r.ip)
+        if s is None:
+            s = stats[r.ip] = IPStats(
+                ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts)
+        s.total += 1
+        s.last = r.ts
+        if r.denied:
+            s.denied += 1
+        if r.banned:
+            s.banned += 1
+        if r.is_client_error:
+            s.c4xx += 1
+        if r.is_server_error:
+            s.c5xx += 1
+        if r.is_error:
+            s.errors += 1
+        if r.path:
+            s.paths[r.path] += 1
+        if r.method:
+            s.methods[r.method] += 1
+        if r.host:
+            s.hosts[r.host] += 1
+        if r.ua and r.ua != "-":
+            s.uas[r.ua] += 1
+        if r.user:
+            s.users.add(r.user)
+        if r.path and bad_patterns.is_malicious(r.path):
+            s.malicious_hits += 1
+            for p in bad_patterns.classify_path(r.path):
+                s.malicious_patterns[p.name] = s.malicious_patterns.get(
+                    p.name, 0) + 1
+    return stats

app/utils/reporting.py

@@ -1,55 +1,55 @@
-#!/usr/bin/env python3
-"""Text rendering helpers for the intrusion-detection system."""
-
-from __future__ import annotations
-
-from datetime import datetime
-from typing import Any, Callable, Sequence
-
-#: (header, getter) pair used for CSV + console tables
-Column = tuple[str, Callable[[Any], Any]]
-
-
-def columns_header(columns: Sequence[Column]) -> list[str]:
-    return [c[0] for c in columns]
-
-
-def columns_row(columns: Sequence[Column], obj: Any) -> list[Any]:
-    return [fn(obj) for _, fn in columns]
-
-
-def render_table(headers: Sequence[str], rows: Sequence[Sequence[Any]],
-                 max_width: int = 48) -> str:
-    """Aligned fixed-width text table. Numeric columns right-aligned."""
-    def cell(v: Any) -> str:
-        s = "" if v is None else str(v)
-        return s if len(s) <= max_width else s[:max_width - 1] + "~"
-
-    body = [[cell(v) for v in row] for row in rows]
-    aligns = []
-    for i in range(len(headers)):
-        numeric = bool(rows) and all(
-            isinstance(row[i], (int, float)) and not isinstance(row[i], bool)
-            for row in rows)
-        aligns.append(">" if numeric else "<")
-    widths = [len(h) for h in headers]
-    for row in body:
-        for i, v in enumerate(row):
-            widths[i] = max(widths[i], len(v))
-
-    def line(values: Sequence[str]) -> str:
-        return "  ".join(
-            f"{v:{a}{w}}" for v, a, w in zip(values, aligns, widths)).rstrip()
-
-    out = [line(list(headers)), "-" * (sum(widths) + 2 * (len(widths) - 1))]
-    out += [line(row) for row in body]
-    return "\n".join(out)
-
-
-def render_kv(rows: Sequence[tuple[str, Any]]) -> str:
-    width = max((len(k) for k, _ in rows), default=0) + 1
-    return "\n".join(f"{k + ':':<{width + 1}} {v}" for k, v in rows)
-
-
-def timestamp_name() -> str:
-    return datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+#!/usr/bin/env python3
+"""Text rendering helpers for the intrusion-detection system."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Callable, Sequence
+
+#: (header, getter) pair used for CSV + console tables
+Column = tuple[str, Callable[[Any], Any]]
+
+
+def columns_header(columns: Sequence[Column]) -> list[str]:
+    return [c[0] for c in columns]
+
+
+def columns_row(columns: Sequence[Column], obj: Any) -> list[Any]:
+    return [fn(obj) for _, fn in columns]
+
+
+def render_table(headers: Sequence[str], rows: Sequence[Sequence[Any]],
+                 max_width: int = 48) -> str:
+    """Aligned fixed-width text table. Numeric columns right-aligned."""
+    def cell(v: Any) -> str:
+        s = "" if v is None else str(v)
+        return s if len(s) <= max_width else s[:max_width - 1] + "~"
+
+    body = [[cell(v) for v in row] for row in rows]
+    aligns = []
+    for i in range(len(headers)):
+        numeric = bool(rows) and all(
+            isinstance(row[i], (int, float)) and not isinstance(row[i], bool)
+            for row in rows)
+        aligns.append(">" if numeric else "<")
+    widths = [len(h) for h in headers]
+    for row in body:
+        for i, v in enumerate(row):
+            widths[i] = max(widths[i], len(v))
+
+    def line(values: Sequence[str]) -> str:
+        return "  ".join(
+            f"{v:{a}{w}}" for v, a, w in zip(values, aligns, widths)).rstrip()
+
+    out = [line(list(headers)), "-" * (sum(widths) + 2 * (len(widths) - 1))]
+    out += [line(row) for row in body]
+    return "\n".join(out)
+
+
+def render_kv(rows: Sequence[tuple[str, Any]]) -> str:
+    width = max((len(k) for k, _ in rows), default=0) + 1
+    return "\n".join(f"{k + ':':<{width + 1}} {v}" for k, v in rows)
+
+
+def timestamp_name() -> str:
+    return datetime.now().strftime("%Y-%m-%d-%H-%M-%S")

app/utils/retention.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Report retention for generated files under ``/reports/``.
+
+Every subdirectory keeps at most ``REPORT_RETENTION_MAX_FILES`` files, all
+newer than ``REPORT_RETENTION_DAYS`` days. Older timestamped reports (e.g.
+``reports/copyparty/*.txt``) are pruned automatically after each consolidated
+scan so the tree does not grow without bound.
+
+The git-tracked blocklist source of truth
+(``<report_dir>/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``) is
+always protected and never pruned.
+"""
+
+from __future__ import annotations
+
+import os
+import time
+from typing import Iterable
+
+from .constants import (
+    REPORT_RETENTION_DAYS,
+    REPORT_RETENTION_MAX_FILES,
+)
+
+#: default protected files (never pruned): the cumulative blocklist
+BLOCKLIST_FILES = ("blocklist_ipv4.txt", "blocklist_ipv6.txt")
+
+
+def _age_days(mtime: float) -> float:
+    return (time.time() - mtime) / 86400.0
+
+
+def prune_report_tree(root: str,
+                      *,
+                      max_age_days: int = REPORT_RETENTION_DAYS,
+                      max_files: int = REPORT_RETENTION_MAX_FILES,
+                      dry_run: bool = False,
+                      protected: Iterable[str] = ()) -> dict[str, list[str]]:
+    """Prune a report tree so each directory keeps ≤ ``max_files`` recent files.
+
+    A file is kept only if it is both among the ``max_files`` most recently
+    modified files in its directory AND newer than ``max_age_days`` days.
+    Protected paths are never touched. Returns ``{dir: [deleted paths]}``.
+    """
+    protected_abs = {os.path.abspath(p) for p in protected}
+    removed: dict[str, list[str]] = {}
+
+    for dirpath, _dirnames, filenames in os.walk(root):
+        files: list[tuple[str, float]] = []
+        for name in filenames:
+            path = os.path.join(dirpath, name)
+            if os.path.abspath(path) in protected_abs:
+                continue  # protected files are neither kept-counted nor pruned
+            try:
+                mtime = os.path.getmtime(path)
+            except OSError:
+                continue
+            files.append((path, mtime))
+
+        # Newest first; only these (within the age window) survive.
+        files.sort(key=lambda t: t[1], reverse=True)
+        newest = {path for path, _ in files[:max_files]}
+        cutoff_days = max_age_days
+
+        deleted: list[str] = []
+        for path, mtime in files:
+            if path in newest and _age_days(mtime) <= cutoff_days:
+                continue  # keep: recent and within the file cap
+            deleted.append(path)
+            if not dry_run:
+                try:
+                    os.remove(path)
+                except OSError:
+                    # e.g. file in use on Windows — skip and leave it.
+                    continue
+        if deleted:
+            removed[dirpath] = deleted
+
+    return removed
+
+
+def prune_report_dirs(*roots: str,
+                      dry_run: bool = False,
+                      max_age_days: int = REPORT_RETENTION_DAYS,
+                      max_files: int = REPORT_RETENTION_MAX_FILES,
+                      ) -> dict[str, list[str]]:
+    """Prune each of the given report roots (deduplicated by real path).
+
+    The blocklist source-of-truth files are always protected.
+    Returns ``{dir: [deleted paths]}`` across all roots.
+    """
+    protected = [
+        os.path.join(root, "app", "data", name)
+        for root in roots
+        for name in BLOCKLIST_FILES
+    ]
+    seen: set[str] = set()
+    summary: dict[str, list[str]] = {}
+    for root in roots:
+        real = os.path.realpath(root)
+        if real in seen or not os.path.isdir(root):
+            continue
+        seen.add(real)
+        summary.update(prune_report_tree(
+            root, max_age_days=max_age_days, max_files=max_files,
+            dry_run=dry_run, protected=protected))
+    return summary

app/utils/utils.py

@@ -1,35 +1,35 @@
-#!/usr/bin/env python3
-"""Shared core for the local-dev intrusion-detection scripts.
-
-`copyparty.py` and `npm.py` are thin adapters. They only provide:
-
-  * a log-file matcher (glob-ish pattern / suffixes)
-  * a line parser producing `Request` objects
-  * a flagging rule (which IPs look hostile)
-  * column definitions for their CSV output
-
-Everything else -- IP aggregation, whitelisting, console + file reports,
-MikroTik RouterOS export -- lives in this module.
-"""
-
-from __future__ import annotations
-
-import argparse
-
-from .constants import DEFAULT_TOP
-
-
-def arg_parser(description: str, default_log_path: str,
-               default_report_dir: str) -> argparse.ArgumentParser:
-    p = argparse.ArgumentParser(description=description)
-    p.add_argument("--log-path", default=default_log_path,
-                   help=f"log directory (default: {default_log_path})")
-    p.add_argument("--report-dir", default=default_report_dir,
-                   help=f"report output dir (default: {default_report_dir})")
-    p.add_argument("--top", type=int, default=DEFAULT_TOP,
-                   help=f"rows in the top-IP table (default: {DEFAULT_TOP})")
-    p.add_argument("--quiet", action="store_true",
-                   help="write files, print nothing but the paths")
-    p.add_argument("--no-rsc", action="store_true",
-                   help="skip MikroTik .rsc generation")
-    return p
+#!/usr/bin/env python3
+"""Shared core for the local-dev intrusion-detection scripts.
+
+`copyparty.py` and `npm.py` are thin adapters. They only provide:
+
+  * a log-file matcher (glob-ish pattern / suffixes)
+  * a line parser producing `Request` objects
+  * a flagging rule (which IPs look hostile)
+  * column definitions for their CSV output
+
+Everything else -- IP aggregation, whitelisting, console + file reports,
+MikroTik RouterOS export -- lives in this module.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+from .constants import DEFAULT_TOP
+
+
+def arg_parser(description: str, default_log_path: str,
+               default_report_dir: str) -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(description=description)
+    p.add_argument("--log-path", default=default_log_path,
+                   help=f"log directory (default: {default_log_path})")
+    p.add_argument("--report-dir", default=default_report_dir,
+                   help=f"report output dir (default: {default_report_dir})")
+    p.add_argument("--top", type=int, default=DEFAULT_TOP,
+                   help=f"rows in the top-IP table (default: {DEFAULT_TOP})")
+    p.add_argument("--quiet", action="store_true",
+                   help="write files, print nothing but the paths")
+    p.add_argument("--no-rsc", action="store_true",
+                   help="skip MikroTik .rsc generation")
+    return p

app/utils/whitelists.py

@@ -1,277 +1,294 @@
-#!/usr/bin/env python3
-"""Whitelist management for the intrusion-detection system.
-
-Owns provider CIDR constants, remote download/refresh logic, and
-CIDR extraction/validation.  Imported by `utils.py` which provides
-the convenience lookups `whitelist_of()` and `is_whitelisted()`.
-
-Whitelist loading order (per provider):
-  1. Local file in `data/<provider>/` (if present and readable)
-  2. Remote download (if network available)
-  3. Built-in hardcoded fallback
-"""
-
-from __future__ import annotations
-
-import ipaddress
-import os
-import re
-from typing import Any
-
-from .constants import (
-    CLOUDFLARE_URL_V4,
-    CLOUDFLARE_URL_V6,
-    CLOUDFLARE_V4_FILE,
-    CLOUDFLARE_V6_FILE,
-    OPENFACTORY_FILE,
-    OPENFACTORY_URL,
-    PALOALTO_FILE,
-    PALOALTO_URL,
-)
-
-# ---------------------------------------------------------------------------
-# CIDR extraction regex
-# ---------------------------------------------------------------------------
-
-CIDR_RE = re.compile(
-    r"(?<![0-9A-Fa-f:.])([0-9A-Fa-f:.]+/\d{1,3})(?![0-9A-Fa-f:.])")
-
-# ---------------------------------------------------------------------------
-# Local file paths (checked first, before remote download)
-# ---------------------------------------------------------------------------
-
-# (paths live in app/constants.py as OPENFACTORY_FILE, CLOUDFLARE_V4_FILE,
-#  CLOUDFLARE_V6_FILE, PALOALTO_FILE; URLs as OPENFACTORY_URL, ...)
-
-# ---------------------------------------------------------------------------
-# Built-in fallback CIDR lists
-# ---------------------------------------------------------------------------
-
-#: Openfactory AS58299 colocation IP range (https://openfactory.net/geofeed.csv)
-OPENFACTORY_IPV4 = ["5.226.144.0/21"]
-
-#: https://www.cloudflare.com/ips-v4
-CLOUDFLARE_IPV4 = [
-    "173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22",
-    "103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18",
-    "190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22",
-    "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13",
-    "104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
-]
-
-#: https://www.cloudflare.com/ips-v6
-CLOUDFLARE_IPV6 = [
-    "2400:cb00::/32", "2606:4700::/32", "2803:f800::/32",
-    "2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29",
-    "2c0f:f248::/32",
-]
-
-#: https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity
-PALOALTO_IPV4 = [
-    "35.203.210.0/23", "144.86.173.0/24", "147.185.132.0/23",
-    "162.216.149.0/24", "162.216.150.0/24", "172.105.147.0/24",
-    "198.235.24.0/24", "205.210.31.0/24", "216.25.88.0/21",
-]
-
-PALOALTO_IPV6 = [
-    "2604:a940:300:5b6::/64",
-    "2604:a940:301:225::/64",
-    "2604:a940:302:118::/64",
-]
-
-# ---------------------------------------------------------------------------
-# Active whitelist state (mutated by refresh)
-# ---------------------------------------------------------------------------
-
-#: address-list name -> CIDR ranges (v4 + v6 mixed, split at render time)
-WHITELISTS: dict[str, list[str]] = {
-    "Openfactory": OPENFACTORY_IPV4,
-    "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
-    "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
-}
-
-#: pre-parsed networks, so matching does not re-parse CIDRs per IP
-WHITELIST_NETS: dict[str, tuple[Any, ...]] = {
-    name: tuple(ipaddress.ip_network(c) for c in cidrs)
-    for name, cidrs in WHITELISTS.items()
-}
-
-#: guard so refresh runs at most once per process
-_refreshed = False
-
-
-# ---------------------------------------------------------------------------
-# CIDR extraction / validation
-# ---------------------------------------------------------------------------
-
-def extract_cidrs(text: str) -> list[str]:
-    """Extract valid CIDR strings from plain text, CSV, or HTML."""
-    seen: set[str] = set()
-    cidrs: list[str] = []
-    for match in CIDR_RE.finditer(text):
-        candidate = match.group(1).strip()
-        try:
-            network = ipaddress.ip_network(candidate, strict=False)
-        except ValueError:
-            continue
-        rendered = str(network)
-        if rendered not in seen:
-            seen.add(rendered)
-            cidrs.append(rendered)
-    return cidrs
-
-
-# ---------------------------------------------------------------------------
-# Local file loading
-# ---------------------------------------------------------------------------
-
-def _read_local_file(path: str) -> list[str]:
-    """Read a local whitelist file and extract valid CIDRs."""
-    try:
-        with open(path, "r", encoding="utf-8", errors="replace") as f:
-            return extract_cidrs(f.read())
-    except Exception:
-        return []
-
-
-def _load_local_whitelists() -> dict[str, list[str]]:
-    """Load whitelist CIDRs from local data/ files.
-
-    Returns a dict of {name: [CIDR strings]}. Only keys with
-    successfully parsed CIDRs are included.
-    """
-    out: dict[str, list[str]] = {}
-    cidrs = _read_local_file(OPENFACTORY_FILE)
-    if cidrs:
-        out["Openfactory"] = cidrs
-    cf_v4 = _read_local_file(CLOUDFLARE_V4_FILE)
-    cf_v6 = _read_local_file(CLOUDFLARE_V6_FILE)
-    if cf_v4 or cf_v6:
-        out["Cloudflare"] = cf_v4 + cf_v6
-    cidrs = _read_local_file(PALOALTO_FILE)
-    if cidrs:
-        out["PaloAlto"] = cidrs
-    return out
-
-
-# ---------------------------------------------------------------------------
-# HTTP download helpers
-# ---------------------------------------------------------------------------
-
-def _download_text(url: str) -> str:
-    """Download a text file from a URL, return as string."""
-    import urllib.request
-    request = urllib.request.Request(
-        url,
-        headers={
-            "User-Agent": "Mozilla/5.0 (compatible; intrusion-detection/1.0)",
-            "Accept": "text/plain,text/csv,text/html;q=0.9,*/*;q=0.8",
-        },
-    )
-    with urllib.request.urlopen(request) as response:
-        return response.read().decode("utf-8", errors="replace")
-
-
-def _download_cidrs(url: str) -> list[str]:
-    """Download a whitelist source and extract valid CIDRs from it."""
-    return extract_cidrs(_download_text(url))
-
-
-# ---------------------------------------------------------------------------
-# Refresh pipeline
-# ---------------------------------------------------------------------------
-
-def _download_whitelists() -> dict[str, list[str]]:
-    """Download the latest whitelists from the Internet and save to local files.
-
-    Returns a dict of {name: [CIDR strings]}.  Only keys with
-    successfully parsed CIDRs are included; failures are logged
-    and skipped so built-in fallbacks remain untouched.
-    Saves each download to the corresponding local file on success.
-    """
-    out: dict[str, list[str]] = {}
-    try:
-        text = _download_text(OPENFACTORY_URL)
-        cidrs = extract_cidrs(text)
-        if cidrs:
-            out["Openfactory"] = cidrs
-            _save_local_file(OPENFACTORY_FILE, text)
-    except Exception as e:
-        print(f"Error downloading Openfactory whitelist: {e}")
-    try:
-        text_v4 = _download_text(CLOUDFLARE_URL_V4)
-        text_v6 = _download_text(CLOUDFLARE_URL_V6)
-        cidrs = extract_cidrs(text_v4) + extract_cidrs(text_v6)
-        if cidrs:
-            out["Cloudflare"] = cidrs
-            _save_local_file(CLOUDFLARE_V4_FILE, text_v4)
-            _save_local_file(CLOUDFLARE_V6_FILE, text_v6)
-    except Exception as e:
-        print(f"Error downloading Cloudflare whitelist: {e}")
-    # Do not download PaloAlto whitelist for now, as it is not a reliable source of CIDRs and often returns HTML instead of plain text.
-    # try:
-    #     text = _download_text(PALOALTO_URL)
-    #     cidrs = extract_cidrs(text)
-    #     if cidrs and len(cidrs) > 0:
-    #         out["PaloAlto"] = cidrs
-    #         _save_local_file(PALOALTO_FILE, text)
-    # except Exception as e:
-    #     print(f"Error downloading PaloAlto whitelist: {e}")
-    return out
-
-
-def _save_local_file(path: str, text: str) -> None:
-    """Save downloaded text to a local file, creating parent dirs as needed."""
-    try:
-        os.makedirs(os.path.dirname(path), exist_ok=True)
-        with open(path, "w", encoding="utf-8") as f:
-            f.write(text)
-    except Exception as e:
-        print(f"Error saving whitelist file {path}: {e}")
-
-
-def update_whitelists() -> None:
-    """Load whitelists (local files first, then remote, then built-in fallback).
-
-    Priority per provider:
-      1. Local file in data/<provider>/
-      2. Remote download (saves to local file on success)
-      3. Built-in hardcoded constants (always available)
-    """
-    global WHITELISTS, WHITELIST_NETS
-
-    # Start from built-in fallbacks, then layer local, then remote
-    WHITELISTS = {
-        "Openfactory": OPENFACTORY_IPV4,
-        "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
-        "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
-    }
-
-    # Layer 1: local files
-    local = _load_local_whitelists()
-    if local:
-        WHITELISTS.update(local)
-
-    # Layer 2: remote download (highest priority, saves local)
-    remote = _download_whitelists()
-    if remote:
-        WHITELISTS.update(remote)
-
-    # Rebuild parsed networks
-    WHITELIST_NETS = {
-        name: tuple(ipaddress.ip_network(c) for c in cidrs)
-        for name, cidrs in WHITELISTS.items()
-    }
-
-
-def refresh_whitelists(force: bool = False) -> bool:
-    """Refresh provider whitelists once per process, keeping built-in fallbacks.
-
-    Returns True if a refresh was attempted, False if already done.
-    """
-    global _refreshed
-    if _refreshed and not force:
-        return False
-    update_whitelists()
-    _refreshed = True
-    return True
+#!/usr/bin/env python3
+"""Whitelist management for the intrusion-detection system.
+
+Owns provider CIDR constants, remote download/refresh logic, and
+CIDR extraction/validation.  Imported by `utils.py` which provides
+the convenience lookups `whitelist_of()` and `is_whitelisted()`.
+
+Whitelist loading order (per provider):
+  1. Local file in `data/<provider>/` (if present and readable)
+  2. Remote download (if network available)
+  3. Built-in hardcoded fallback
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import os
+import re
+from typing import Any
+
+from .constants import (
+    CLOUDFLARE_URL_V4,
+    CLOUDFLARE_URL_V6,
+    CLOUDFLARE_V4_FILE,
+    CLOUDFLARE_V6_FILE,
+    PALOALTO_FILE,
+    PALOALTO_URL,
+)
+
+# ---------------------------------------------------------------------------
+# CIDR extraction regex
+# ---------------------------------------------------------------------------
+
+CIDR_RE = re.compile(
+    r"(?<![0-9A-Fa-f:.])([0-9A-Fa-f:.]+/\d{1,3})(?![0-9A-Fa-f:.])")
+
+# ---------------------------------------------------------------------------
+# Local file paths (checked first, before remote download)
+# ---------------------------------------------------------------------------
+
+# (paths live in app/constants.py as CLOUDFLARE_V4_FILE,
+#  CLOUDFLARE_V6_FILE, PALOALTO_FILE; URLs as CLOUDFLARE_URL_V4, ...)
+
+# ---------------------------------------------------------------------------
+# Built-in fallback CIDR lists
+# ---------------------------------------------------------------------------
+
+#: Init7 (Switzerland) Ltd. ranges — AS13030, verified via RIPE RDAP 2026-09-08.
+#: Includes the Hybrid7 WAN /24 (router WAN 77.74.80.159), init7 DNS
+#: infrastructure, and the PPPoE gateway. init7 publishes no geofeed, so
+#: these are maintained here rather than downloaded.
+INIT7_IPV4 = [
+    "77.74.80.0/24",      # Hybrid7 WAN /24
+    "77.109.128.0/18",    # init7 DNS / infrastructure
+    "213.144.128.0/19",   # init7 DNS / infrastructure
+    "5.180.132.0/22",     # init7 PPPoE gateway
+]
+
+#: init7 IPv6 allocation (contains delegated /48 2001:1620:549f::/48)
+INIT7_IPV6 = ["2001:1620::/30"]
+
+#: https://www.cloudflare.com/ips-v4
+CLOUDFLARE_IPV4 = [
+    "173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22",
+    "103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18",
+    "190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22",
+    "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13",
+    "104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
+]
+
+#: https://www.cloudflare.com/ips-v6
+CLOUDFLARE_IPV6 = [
+    "2400:cb00::/32", "2606:4700::/32", "2803:f800::/32",
+    "2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29",
+    "2c0f:f248::/32",
+]
+
+#: https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity
+PALOALTO_IPV4 = [
+    "35.203.210.0/23", "144.86.173.0/24", "147.185.132.0/23",
+    "162.216.149.0/24", "162.216.150.0/24", "172.105.147.0/24",
+    "198.235.24.0/24", "205.210.31.0/24", "216.25.88.0/21",
+]
+
+PALOALTO_IPV6 = [
+    "2604:a940:300:5b6::/64",
+    "2604:a940:301:225::/64",
+    "2604:a940:302:118::/64",
+]
+
+# ---------------------------------------------------------------------------
+# Active whitelist state (mutated by refresh)
+# ---------------------------------------------------------------------------
+
+#: address-list name -> CIDR ranges (v4 + v6 mixed, split at render time)
+WHITELISTS: dict[str, list[str]] = {
+    "Init7": INIT7_IPV4 + INIT7_IPV6,
+    "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
+    "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
+}
+
+#: pre-parsed networks, so matching does not re-parse CIDRs per IP
+WHITELIST_NETS: dict[str, tuple[Any, ...]] = {
+    name: tuple(ipaddress.ip_network(c) for c in cidrs)
+    for name, cidrs in WHITELISTS.items()
+}
+
+#: guard so refresh runs at most once per process
+_refreshed = False
+
+
+# ---------------------------------------------------------------------------
+# CIDR extraction / validation
+# ---------------------------------------------------------------------------
+
+def extract_cidrs(text: str) -> list[str]:
+    """Extract valid CIDR strings from plain text, CSV, or HTML."""
+    seen: set[str] = set()
+    cidrs: list[str] = []
+    for match in CIDR_RE.finditer(text):
+        candidate = match.group(1).strip()
+        try:
+            network = ipaddress.ip_network(candidate, strict=False)
+        except ValueError:
+            continue
+        rendered = str(network)
+        if rendered not in seen:
+            seen.add(rendered)
+            cidrs.append(rendered)
+    return cidrs
+
+
+# ---------------------------------------------------------------------------
+# Local file loading
+# ---------------------------------------------------------------------------
+
+def _read_local_file(path: str) -> list[str]:
+    """Read a local whitelist file and extract valid CIDRs."""
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            return extract_cidrs(f.read())
+    except Exception:
+        return []
+
+
+def _load_local_whitelists() -> dict[str, list[str]]:
+    """Load whitelist CIDRs from local data/ files.
+
+    Returns a dict of {name: [CIDR strings]}. Only keys with
+    successfully parsed CIDRs are included.
+    """
+    out: dict[str, list[str]] = {}
+    # Init7 has no published geofeed; its ranges are built-in (INIT7_IPV4/6).
+    cf_v4 = _read_local_file(CLOUDFLARE_V4_FILE)
+    cf_v6 = _read_local_file(CLOUDFLARE_V6_FILE)
+    if cf_v4 or cf_v6:
+        out["Cloudflare"] = cf_v4 + cf_v6
+    cidrs = _read_local_file(PALOALTO_FILE)
+    if cidrs:
+        out["PaloAlto"] = cidrs
+    return out
+
+
+# ---------------------------------------------------------------------------
+# HTTP download helpers
+# ---------------------------------------------------------------------------
+
+def _download_text(url: str) -> str:
+    """Download a text file from a URL, return as string."""
+    import urllib.request
+    request = urllib.request.Request(
+        url,
+        headers={
+            "User-Agent": "Mozilla/5.0 (compatible; intrusion-detection/1.0)",
+            "Accept": "text/plain,text/csv,text/html;q=0.9,*/*;q=0.8",
+        },
+    )
+    with urllib.request.urlopen(request) as response:
+        return response.read().decode("utf-8", errors="replace")
+
+
+def _download_cidrs(url: str) -> list[str]:
+    """Download a whitelist source and extract valid CIDRs from it."""
+    return extract_cidrs(_download_text(url))
+
+
+# ---------------------------------------------------------------------------
+# Refresh pipeline
+# ---------------------------------------------------------------------------
+
+def _download_whitelists() -> dict[str, list[str]]:
+    """Download the latest whitelists from the Internet and save to local files.
+
+    Returns a dict of {name: [CIDR strings]}.  Only keys with
+    successfully parsed CIDRs are included; failures are logged
+    and skipped so built-in fallbacks remain untouched.
+    Saves each download to the corresponding local file on success.
+    """
+    out: dict[str, list[str]] = {}
+    # Init7 has no published geofeed; its ranges are built-in (INIT7_IPV4/6).
+    try:
+        text_v4 = _download_text(CLOUDFLARE_URL_V4)
+        text_v6 = _download_text(CLOUDFLARE_URL_V6)
+        cidrs = extract_cidrs(text_v4) + extract_cidrs(text_v6)
+        if cidrs:
+            out["Cloudflare"] = cidrs
+            _save_local_file(CLOUDFLARE_V4_FILE, text_v4)
+            _save_local_file(CLOUDFLARE_V6_FILE, text_v6)
+    except Exception as e:
+        print(f"Error downloading Cloudflare whitelist: {e}")
+    # Do not download PaloAlto whitelist for now, as it is not a reliable source of CIDRs and often returns HTML instead of plain text.
+    # try:
+    #     text = _download_text(PALOALTO_URL)
+    #     cidrs = extract_cidrs(text)
+    #     if cidrs and len(cidrs) > 0:
+    #         out["PaloAlto"] = cidrs
+    #         _save_local_file(PALOALTO_FILE, text)
+    # except Exception as e:
+    #     print(f"Error downloading PaloAlto whitelist: {e}")
+    return out
+
+
+def _save_local_file(path: str, text: str) -> None:
+    """Save downloaded text to a local file, creating parent dirs as needed."""
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(text)
+    except Exception as e:
+        print(f"Error saving whitelist file {path}: {e}")
+
+
+def update_whitelists() -> None:
+    """Load whitelists (local files first, then remote, then built-in fallback).
+
+    Priority per provider:
+      1. Local file in data/<provider>/
+      2. Remote download (saves to local file on success)
+      3. Built-in hardcoded constants (always available)
+    """
+    global WHITELISTS, WHITELIST_NETS
+
+    # Start from built-in fallbacks, then layer local, then remote
+    WHITELISTS = {
+        "Init7": INIT7_IPV4 + INIT7_IPV6,
+        "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
+        "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
+    }
+
+    # Layer 1: local files
+    local = _load_local_whitelists()
+    if local:
+        WHITELISTS.update(local)
+
+    # Layer 2: remote download (highest priority, saves local)
+    remote = _download_whitelists()
+    if remote:
+        WHITELISTS.update(remote)
+
+    # Rebuild parsed networks
+    WHITELIST_NETS = {
+        name: tuple(ipaddress.ip_network(c) for c in cidrs)
+        for name, cidrs in WHITELISTS.items()
+    }
+
+
+def refresh_whitelists(force: bool = False) -> bool:
+    """Refresh provider whitelists once per process, keeping built-in fallbacks.
+
+    Returns True if a refresh was attempted, False if already done.
+    """
+    global _refreshed
+    if _refreshed and not force:
+        return False
+    update_whitelists()
+    _refreshed = True
+    return True
+
+
+def sync_whitelists_to_db() -> None:
+    """Persist the current in-memory ``WHITELISTS`` into the SQLite store.
+
+    Best-effort (never raises): keeps the DB whitelist tables consistent with
+    the in-memory/provider state so lookups and analysis can rely on them.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        conn = get_connection()
+        for provider, cidrs in WHITELISTS.items():
+            repo.set_whitelist(conn, provider, cidrs,
+                               source="builtin/refresh")
+    except Exception as exc:  # pragma: no cover - defensive
+        print(f"WARN: could not sync whitelists to db ({exc})")

app/utils/whois.py

@@ -1,128 +1,128 @@
-#!/usr/bin/env python3
-"""WHOIS subnet querying and IP consolidation."""
-
-from __future__ import annotations
-
-import ipaddress
-import json
-import os
-import re
-from typing import Iterable
-
-from .constants import (
-    SUBNET_THRESHOLD,
-    WHOIS_CACHE_PATH,
-    WHOIS_PORT,
-    WHOIS_SERVER,
-    WHOIS_TIMEOUT,
-)
-from .ip_utils import ip_version, parse_ip
-
-_whois_cache: dict[str, str] = {}
-
-
-def load_whois_cache() -> None:
-    global _whois_cache
-    if not _whois_cache and os.path.exists(WHOIS_CACHE_PATH):
-        try:
-            with open(WHOIS_CACHE_PATH, "r", encoding="utf-8") as f:
-                _whois_cache = json.load(f)
-        except Exception:
-            _whois_cache = {}
-
-
-def save_whois_cache() -> None:
-    if _whois_cache:
-        os.makedirs(os.path.dirname(WHOIS_CACHE_PATH), exist_ok=True)
-        try:
-            with open(WHOIS_CACHE_PATH, "w", encoding="utf-8") as f:
-                json.dump(_whois_cache, f, indent=2)
-        except Exception:
-            pass
-
-
-def query_whois_subnet(ip: str) -> str | None:
-    """Query whois.radb.net for the route/subnet of an IP address."""
-    addr = parse_ip(ip)
-    if addr is None:
-        return None
-
-    load_whois_cache()
-    if ip in _whois_cache:
-        return _whois_cache[ip] or None
-
-    import socket
-    try:
-        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        s.settimeout(WHOIS_TIMEOUT)
-        s.connect((WHOIS_SERVER, WHOIS_PORT))
-        s.sendall(f"{ip}\r\n".encode())
-        res = ""
-        while True:
-            chunk = s.recv(4096).decode(errors="ignore")
-            if not chunk:
-                break
-            res += chunk
-        s.close()
-
-        routes = re.findall(r"^(?:route|route6):\s*(\S+)",
-                            res, re.IGNORECASE | re.MULTILINE)
-        if routes:
-            routes_parsed = []
-            for r in routes:
-                try:
-                    routes_parsed.append(ipaddress.ip_network(r))
-                except ValueError:
-                    continue
-            if routes_parsed:
-                best_route = max(routes_parsed, key=lambda n: n.prefixlen)
-                subnet_str = str(best_route)
-                _whois_cache[ip] = subnet_str
-                save_whois_cache()
-                return subnet_str
-
-    except Exception:
-        pass
-
-    subnet_str = f"{ip}/32" if addr.version == 4 else f"{ip}/128"
-    _whois_cache[ip] = subnet_str
-    save_whois_cache()
-    return subnet_str
-
-
-def consolidate_by_subnet(ips: Iterable[str],
-                          threshold: int = SUBNET_THRESHOLD) -> set[str]:
-    """Group IPs by WHOIS subnet; if >= threshold IPs in a subnet, block the subnet.
-
-    Otherwise, block individual IPs.
-    """
-    ip_list = list(ips)
-    subnet_to_ips: dict[str, set[str]] = {}
-    ip_to_subnet: dict[str, str] = {}
-
-    for ip in ip_list:
-        subnet = query_whois_subnet(ip)
-        if subnet:
-            try:
-                net = ipaddress.ip_network(subnet, strict=False)
-                addr = ipaddress.ip_address(ip)
-                if addr in net:
-                    subnet_to_ips.setdefault(subnet, set()).add(ip)
-                    ip_to_subnet[ip] = subnet
-                    continue
-            except ValueError:
-                pass
-        fallback_net = f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128"
-        subnet_to_ips.setdefault(fallback_net, set()).add(ip)
-        ip_to_subnet[ip] = fallback_net
-
-    blocked_targets: set[str] = set()
-    for subnet, mapped_ips in subnet_to_ips.items():
-        if len(mapped_ips) >= threshold:
-            blocked_targets.add(subnet)
-        else:
-            for ip in mapped_ips:
-                blocked_targets.add(
-                    f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128")
-
-    return blocked_targets
+#!/usr/bin/env python3
+"""WHOIS subnet querying and IP consolidation."""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import os
+import re
+from typing import Iterable
+
+from .constants import (
+    SUBNET_THRESHOLD,
+    WHOIS_CACHE_PATH,
+    WHOIS_PORT,
+    WHOIS_SERVER,
+    WHOIS_TIMEOUT,
+)
+from .ip_utils import ip_version, parse_ip
+
+_whois_cache: dict[str, str] = {}
+
+
+def load_whois_cache() -> None:
+    global _whois_cache
+    if not _whois_cache and os.path.exists(WHOIS_CACHE_PATH):
+        try:
+            with open(WHOIS_CACHE_PATH, "r", encoding="utf-8") as f:
+                _whois_cache = json.load(f)
+        except Exception:
+            _whois_cache = {}
+
+
+def save_whois_cache() -> None:
+    if _whois_cache:
+        os.makedirs(os.path.dirname(WHOIS_CACHE_PATH), exist_ok=True)
+        try:
+            with open(WHOIS_CACHE_PATH, "w", encoding="utf-8") as f:
+                json.dump(_whois_cache, f, indent=2)
+        except Exception:
+            pass
+
+
+def query_whois_subnet(ip: str) -> str | None:
+    """Query whois.radb.net for the route/subnet of an IP address."""
+    addr = parse_ip(ip)
+    if addr is None:
+        return None
+
+    load_whois_cache()
+    if ip in _whois_cache:
+        return _whois_cache[ip] or None
+
+    import socket
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(WHOIS_TIMEOUT)
+        s.connect((WHOIS_SERVER, WHOIS_PORT))
+        s.sendall(f"{ip}\r\n".encode())
+        res = ""
+        while True:
+            chunk = s.recv(4096).decode(errors="ignore")
+            if not chunk:
+                break
+            res += chunk
+        s.close()
+
+        routes = re.findall(r"^(?:route|route6):\s*(\S+)",
+                            res, re.IGNORECASE | re.MULTILINE)
+        if routes:
+            routes_parsed = []
+            for r in routes:
+                try:
+                    routes_parsed.append(ipaddress.ip_network(r))
+                except ValueError:
+                    continue
+            if routes_parsed:
+                best_route = max(routes_parsed, key=lambda n: n.prefixlen)
+                subnet_str = str(best_route)
+                _whois_cache[ip] = subnet_str
+                save_whois_cache()
+                return subnet_str
+
+    except Exception:
+        pass
+
+    subnet_str = f"{ip}/32" if addr.version == 4 else f"{ip}/128"
+    _whois_cache[ip] = subnet_str
+    save_whois_cache()
+    return subnet_str
+
+
+def consolidate_by_subnet(ips: Iterable[str],
+                          threshold: int = SUBNET_THRESHOLD) -> set[str]:
+    """Group IPs by WHOIS subnet; if >= threshold IPs in a subnet, block the subnet.
+
+    Otherwise, block individual IPs.
+    """
+    ip_list = list(ips)
+    subnet_to_ips: dict[str, set[str]] = {}
+    ip_to_subnet: dict[str, str] = {}
+
+    for ip in ip_list:
+        subnet = query_whois_subnet(ip)
+        if subnet:
+            try:
+                net = ipaddress.ip_network(subnet, strict=False)
+                addr = ipaddress.ip_address(ip)
+                if addr in net:
+                    subnet_to_ips.setdefault(subnet, set()).add(ip)
+                    ip_to_subnet[ip] = subnet
+                    continue
+            except ValueError:
+                pass
+        fallback_net = f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128"
+        subnet_to_ips.setdefault(fallback_net, set()).add(ip)
+        ip_to_subnet[ip] = fallback_net
+
+    blocked_targets: set[str] = set()
+    for subnet, mapped_ips in subnet_to_ips.items():
+        if len(mapped_ips) >= threshold:
+            blocked_targets.add(subnet)
+        else:
+            for ip in mapped_ips:
+                blocked_targets.add(
+                    f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128")
+
+    return blocked_targets

app/web/__init__.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""Flask web UI for the intrusion-detection system.
+
+A localhost-only web server (``127.0.0.1:<high port>``) for running scans,
+browsing reports, and managing the MikroTik blocklist / firewall.
+
+Run directly:  ``python -m app.web``
+Or via the entry point: ``python app.py --web`` (see ``serve()``).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+
+from flask import Flask, jsonify, request
+
+from ..utils.constants import (
+    APP_REPORT_DATA_DIR,
+    APP_REPORT_DIR,
+    COPYPARTY_LOG_PATH,
+    COPYPARTY_REPORT_DIR,
+    DEFAULT_ERROR_RATIO,
+    DEFAULT_MIN_DENIED,
+    DEFAULT_MIN_ERRORS,
+    DEFAULT_TOP,
+    MIKROTIK_REPORT_DIR,
+    NPM_LOG_PATH,
+    NPM_REPORT_DIR,
+    REPORT_DIR,
+    WEB_HOST,
+    WEB_PORT,
+)
+from ..firewall import MikrotikFirewall
+from ..scan import ScanParams
+from . import web_scan
+
+#: report sources exposed by the API -> report directory
+REPORT_SOURCES = {
+    "copyparty": COPYPARTY_REPORT_DIR,
+    "npm": NPM_REPORT_DIR,
+    "app": APP_REPORT_DIR,
+    "mikrotik": MIKROTIK_REPORT_DIR,
+}
+
+SCAN_KINDS = ("copyparty", "npm", "consolidated")
+
+
+# ---------------------------------------------------------------------------
+# helpers
+# ---------------------------------------------------------------------------
+
+def _count_lines(path: str) -> int | None:
+    """Return the line count of a file, or None if it cannot be read."""
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return sum(1 for _ in f)
+    except OSError:
+        return None
+
+
+def _read_lines(path: str) -> list[str]:
+    """Return non-empty lines of a file, or [] if it cannot be read."""
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return [line.strip() for line in f if line.strip()]
+    except OSError:
+        return []
+
+
+def _read_json(path: str, default: list | dict | None = None) -> list | dict:
+    """Read a JSON file, returning ``default`` (or []) on failure."""
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            return json.load(f)
+    except (OSError, ValueError):
+        return default if default is not None else []
+
+
+def _params_from_request() -> ScanParams:
+    """Build ScanParams from the JSON request body, falling back to defaults."""
+    data = request.get_json(silent=True) or {}
+
+    def _pick(key: str, default):
+        value = data.get(key)
+        return default if value is None or value == "" else value
+
+    return ScanParams(
+        copyparty_log_path=str(
+            _pick("copyparty_log_path", COPYPARTY_LOG_PATH)),
+        copyparty_report_dir=str(
+            _pick("copyparty_report_dir", COPYPARTY_REPORT_DIR)),
+        min_denied=int(_pick("min_denied", DEFAULT_MIN_DENIED)),
+        npm_log_path=str(_pick("npm_log_path", NPM_LOG_PATH)),
+        npm_report_dir=str(_pick("npm_report_dir", NPM_REPORT_DIR)),
+        min_errors=int(_pick("min_errors", DEFAULT_MIN_ERRORS)),
+        error_ratio=float(_pick("error_ratio", DEFAULT_ERROR_RATIO)),
+        report_dir=str(_pick("report_dir", REPORT_DIR)),
+        top=int(_pick("top", DEFAULT_TOP)),
+        quiet=bool(data.get("quiet", False)),
+        full=bool(data.get("full", False)),
+        update_firewall=bool(data.get("update_firewall", False)),
+        dry_run=bool(data.get("dry_run", False)),
+    )
+
+
+def _with_router(fn):
+    """Connect to the router, run ``fn(fw)``, always disconnect.
+
+    Returns a JSON error response (502) if the router is unreachable.
+    """
+    fw = MikrotikFirewall()
+    try:
+        fw.connect()
+        return fn(fw)
+    except Exception as e:
+        return jsonify({"error": str(e)}), 502
+    finally:
+        fw.disconnect()
+
+
+def _sync_firewall(dry_run: bool):
+    """Sync the latest consolidated blocklist to the router (preview or apply)."""
+    v4 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
+    v6 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt"))
+
+    def _sync(fw):
+        changes = fw.sync_blocklist(v4, v6, dry_run=dry_run)
+        return jsonify({**changes, "dry_run": dry_run})
+
+    return _with_router(_sync)
+
+
+def _analysis_helpers():
+    """SQLite-backed log-analysis query helpers (None when DB unavailable).
+
+    Functions take the shared DB connection each call so routes stay stateless
+    across the background scan thread.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+    except Exception:
+        return None
+
+    def top_ips(since: str | None = None, limit: int = 50) -> list:
+        return repo.top_ips(get_connection(), since=since, limit=limit)
+
+    def ip_timeline(ip: str) -> list:
+        return repo.ip_timeline(get_connection(), ip=ip)
+
+    def cross_source(limit: int = 100) -> list:
+        return repo.correlate_across_sources(get_connection(), limit=limit)
+
+    def conn():
+        return get_connection()
+
+    def request_count(conn) -> int:
+        return repo.count_requests(conn)
+
+    def scan_count(conn) -> int:
+        return repo.count_scans(conn)
+
+    return {"top_ips": top_ips, "ip_timeline": ip_timeline,
+            "cross_source": cross_source, "conn": conn,
+            "request_count": request_count, "scan_count": scan_count}
+
+
+# ---------------------------------------------------------------------------
+# app factory
+# ---------------------------------------------------------------------------
+
+def create_app() -> Flask:
+    """Build and configure the Flask application."""
+    _app_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+    app = Flask(__name__,
+                template_folder=os.path.join(_app_dir, "templates"),
+                static_folder=os.path.join(_app_dir, "static"))
+
+    # Seed the in-memory job store from disk so the dashboard keeps showing
+    # recent scans across server restarts.
+    web_scan.load_persisted()
+
+    from .routes import register_routes
+
+    helpers = {
+        "_count_lines": _count_lines,
+        "_read_lines": _read_lines,
+        "_read_json": _read_json,
+        "_params_from_request": _params_from_request,
+        "_with_router": _with_router,
+        "_sync_firewall": _sync_firewall,
+        "_analysis": _analysis_helpers(),
+    }
+    register_routes(app, REPORT_SOURCES, SCAN_KINDS, web_scan, helpers)
+    return app
+
+
+def serve(host: str = WEB_HOST, port: int = WEB_PORT) -> None:
+    """Run the web server on localhost at the configured high port."""
+    app = create_app()
+    print(f"Intrusion Detection web UI running at http://{host}:{port}")
+    app.run(host=host, port=port, use_reloader=False)
+
+
+if __name__ == "__main__":
+    serve()

app/web/routes.py

@@ -1,414 +1,418 @@
-#!/usr/bin/env python3
-"""Flask web UI for the intrusion-detection system.
-
-A localhost-only web server (``127.0.0.1:<high port>``) for running scans,
-browsing reports, and managing the MikroTik blocklist / firewall.
-
-Run directly:  ``python -m app.web``
-Or via the entry point: ``python app.py --web`` (see ``serve()``).
-"""
-
-from __future__ import annotations
-
-import json
-import os
-
-from flask import Flask, abort, jsonify, render_template, request
-
-from .constants import (
-    APP_REPORT_DATA_DIR,
-    APP_REPORT_DIR,
-    BLOCKLIST_NAME,
-    COPYPARTY_LOG_PATH,
-    COPYPARTY_REPORT_DIR,
-    DEFAULT_ERROR_RATIO,
-    DEFAULT_MIN_DENIED,
-    DEFAULT_MIN_ERRORS,
-    DEFAULT_TOP,
-    MIKROTIK_DATA_DIR,
-    MIKROTIK_REPORT_DIR,
-    NPM_LOG_PATH,
-    NPM_REPORT_DIR,
-    REPORT_DIR,
-    WEB_HOST,
-    WEB_PORT,
-    WHITELIST_NAMES,
-)
-from .firewall import MikrotikFirewall
-from .ip_utils import ip_version
-from .scan import ScanParams
-from . import web_scan
-
-#: report sources exposed by the API -> report directory
-REPORT_SOURCES = {
-    "copyparty": COPYPARTY_REPORT_DIR,
-    "npm": NPM_REPORT_DIR,
-    "app": APP_REPORT_DIR,
-    "mikrotik": MIKROTIK_REPORT_DIR,
-}
-
-SCAN_KINDS = ("copyparty", "npm", "consolidated")
-
-
-# ---------------------------------------------------------------------------
-# helpers
-# ---------------------------------------------------------------------------
-
-def _count_lines(path: str) -> int | None:
-    """Return the line count of a file, or None if it cannot be read."""
-    try:
-        with open(path, "r", encoding="utf-8") as f:
-            return sum(1 for _ in f)
-    except OSError:
-        return None
-
-
-def _read_lines(path: str) -> list[str]:
-    """Return non-empty lines of a file, or [] if it cannot be read."""
-    try:
-        with open(path, "r", encoding="utf-8") as f:
-            return [line.strip() for line in f if line.strip()]
-    except OSError:
-        return []
-
-
-def _read_json(path: str, default: list | dict | None = None) -> list | dict:
-    """Read a JSON file, returning ``default`` (or []) on failure."""
-    try:
-        with open(path, "r", encoding="utf-8") as f:
-            return json.load(f)
-    except (OSError, ValueError):
-        return default if default is not None else []
-
-
-def _params_from_request() -> ScanParams:
-    """Build ScanParams from the JSON request body, falling back to defaults."""
-    data = request.get_json(silent=True) or {}
-
-    def _pick(key: str, default):
-        value = data.get(key)
-        return default if value is None or value == "" else value
-
-    return ScanParams(
-        copyparty_log_path=str(
-            _pick("copyparty_log_path", COPYPARTY_LOG_PATH)),
-        copyparty_report_dir=str(
-            _pick("copyparty_report_dir", COPYPARTY_REPORT_DIR)),
-        min_denied=int(_pick("min_denied", DEFAULT_MIN_DENIED)),
-        npm_log_path=str(_pick("npm_log_path", NPM_LOG_PATH)),
-        npm_report_dir=str(_pick("npm_report_dir", NPM_REPORT_DIR)),
-        min_errors=int(_pick("min_errors", DEFAULT_MIN_ERRORS)),
-        error_ratio=float(_pick("error_ratio", DEFAULT_ERROR_RATIO)),
-        report_dir=str(_pick("report_dir", REPORT_DIR)),
-        top=int(_pick("top", DEFAULT_TOP)),
-        quiet=bool(data.get("quiet", False)),
-        full=bool(data.get("full", False)),
-        update_firewall=bool(data.get("update_firewall", False)),
-        dry_run=bool(data.get("dry_run", False)),
-    )
-
-
-def _with_router(fn):
-    """Connect to the router, run ``fn(fw)``, always disconnect.
-
-    Returns a JSON error response (502) if the router is unreachable.
-    """
-    fw = MikrotikFirewall()
-    try:
-        fw.connect()
-        return fn(fw)
-    except Exception as e:
-        return jsonify({"error": str(e)}), 502
-    finally:
-        fw.disconnect()
-
-
-def _sync_firewall(dry_run: bool):
-    """Sync the latest consolidated blocklist to the router (preview or apply)."""
-    v4 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
-    v6 = _read_lines(os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt"))
-
-    def _sync(fw):
-        changes = fw.sync_blocklist(v4, v6, dry_run=dry_run)
-        return jsonify({**changes, "dry_run": dry_run})
-
-    return _with_router(_sync)
-
-
-# ---------------------------------------------------------------------------
-# app factory
-# ---------------------------------------------------------------------------
-
-def create_app() -> Flask:
-    """Build and configure the Flask application."""
-    app = Flask(__name__)
-
-    # Seed the in-memory job store from disk so the dashboard keeps showing
-    # recent scans across server restarts.
-    web_scan.load_persisted()
-
-    @app.get("/")
-    def index():
-        return render_template("dashboard.html")
-
-    @app.get("/scan")
-    def scan_page():
-        return render_template("scan.html")
-
-    @app.get("/reports")
-    def reports_page():
-        return render_template("reports.html")
-
-    @app.get("/blocklist")
-    def blocklist_page():
-        return render_template("blocklist.html")
-
-    @app.get("/firewall")
-    def firewall_page():
-        return render_template("firewall.html")
-
-    @app.get("/whitelist")
-    def whitelist_page():
-        return render_template("whitelist.html", whitelist_names=WHITELIST_NAMES)
-
-    @app.get("/graph")
-    def graph_page():
-        return render_template("graph.html")
-
-    # --- status ------------------------------------------------------------
-
-    @app.get("/api/status")
-    def status():
-        v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
-        v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
-        return jsonify({
-            "host": WEB_HOST,
-            "port": WEB_PORT,
-            "scan_running": web_scan.is_scan_running(),
-            "latest_scans": web_scan.latest_scans(5),
-            "blocklist_ipv4_count": _count_lines(v4_path),
-            "blocklist_ipv6_count": _count_lines(v6_path),
-            "firewall_snapshot": {
-                "exists": os.path.isdir(MIKROTIK_DATA_DIR),
-                "path": MIKROTIK_DATA_DIR,
-            },
-        })
-
-    # --- scans -------------------------------------------------------------
-
-    @app.post("/api/scan/<kind>")
-    def start_scan(kind):
-        if kind not in SCAN_KINDS:
-            abort(404, f"unknown scan kind: {kind}")
-        job = web_scan.start_scan(kind, _params_from_request())
-        # A busy-lock rejection is a bare {"error": ...}; a started job has
-        # "error" == None, so test for a truthy value, not key presence.
-        if job.get("error"):
-            return jsonify(job), 409
-        return jsonify(job), 202
-
-    @app.get("/api/scan/status")
-    def scan_status():
-        return jsonify({
-            "running": web_scan.is_scan_running(),
-            "latest_scans": web_scan.latest_scans(10),
-        })
-
-    @app.get("/api/scan/<job_id>")
-    def scan_job(job_id):
-        job = web_scan.get_scan(job_id)
-        if job is None:
-            abort(404, f"unknown job: {job_id}")
-        return jsonify(job)
-
-    # --- reports -----------------------------------------------------------
-
-    @app.get("/api/reports/<source>")
-    def list_reports(source):
-        base = REPORT_SOURCES.get(source)
-        if base is None:
-            abort(404, f"unknown report source: {source}")
-        names = sorted(
-            n for n in os.listdir(base)
-            if os.path.isfile(os.path.join(base, n)))
-        return jsonify({"source": source, "files": names})
-
-    @app.get("/api/reports/<source>/<path:name>")
-    def get_report(source, name):
-        base = REPORT_SOURCES.get(source)
-        if base is None:
-            abort(404, f"unknown report source: {source}")
-        full = os.path.realpath(os.path.join(base, name))
-        if not full.startswith(os.path.realpath(base)):
-            abort(403, "invalid report path")
-        if not os.path.isfile(full):
-            abort(404, "report not found")
-        with open(full, "r", encoding="utf-8", errors="replace") as f:
-            content = f.read()
-        return jsonify({"source": source, "name": name, "content": content})
-
-    # --- blocklist ---------------------------------------------------------
-
-    @app.get("/api/blocklist")
-    def blocklist():
-        return jsonify({
-            "ipv4": _read_lines(
-                os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")),
-            "ipv6": _read_lines(
-                os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")),
-        })
-
-    @app.post("/api/blocklist/add")
-    def blocklist_add():
-        data = request.get_json(silent=True) or {}
-        ip = (data.get("ip") or "").strip()
-        version = ip_version(ip)
-        if not ip or version not in (4, 6):
-            return jsonify({"error": "missing or invalid 'ip' field"}), 400
-
-        def _add(fw):
-            if version == 4:
-                fw.write_address_list_ipv4(BLOCKLIST_NAME, [ip])
-            else:
-                fw.write_address_list_ipv6(BLOCKLIST_NAME, [ip])
-            return jsonify({"ok": True, "action": "add", "ip": ip})
-
-        return _with_router(_add)
-
-    @app.post("/api/blocklist/remove")
-    def blocklist_remove():
-        data = request.get_json(silent=True) or {}
-        ip = (data.get("ip") or "").strip()
-        version = ip_version(ip)
-        if not ip or version not in (4, 6):
-            return jsonify({"error": "missing or invalid 'ip' field"}), 400
-
-        def _remove(fw):
-            path = "/ip" if version == 4 else "/ipv6"
-            fw.execute_command(
-                f"{path} firewall address-list remove "
-                f"[find list={BLOCKLIST_NAME} address={ip}]")
-            return jsonify({"ok": True, "action": "remove", "ip": ip})
-
-        return _with_router(_remove)
-
-    # --- graph --------------------------------------------------------------
-
-    @app.get("/api/graph/ipv4")
-    def graph_ipv4():
-        """Data for the IPv4 /8 tile map.
-
-        Blocked targets come from the consolidated blocklist
-        (``reports/app/data/blocklist_ipv4.txt``); whitelisted ranges from the
-        router snapshot (``data/mikrotik/address_list_ipv4.json``) filtered to
-        the configured whitelist address-list names.
-        """
-        blocked = _read_lines(
-            os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
-        addr_v4 = _read_json(
-            os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
-        whitelisted = sorted({e.get("address") for e in addr_v4
-                              if e.get("list") in WHITELIST_NAMES
-                              and e.get("address")})
-        return jsonify({
-            "blocked": blocked,
-            "whitelisted": whitelisted,
-            "counts": {
-                "blocked": len(blocked),
-                "whitelisted": len(whitelisted),
-            },
-            "blocklist_name": BLOCKLIST_NAME,
-            "whitelist_names": WHITELIST_NAMES,
-        })
-
-    # --- firewall ----------------------------------------------------------
-
-    @app.get("/api/firewall/state")
-    def firewall_state():
-        return jsonify({
-            "address_lists": {
-                "ipv4": _read_json(
-                    os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json")),
-                "ipv6": _read_json(
-                    os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json")),
-            },
-            "firewall_rules": {
-                "ipv4": _read_json(
-                    os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv4.json")),
-                "ipv6": _read_json(
-                    os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv6.json")),
-            },
-            "snapshot_dir": MIKROTIK_DATA_DIR,
-        })
-
-    @app.post("/api/firewall/sync")
-    def firewall_sync():
-        """Preview the blocklist sync (dry-run, no changes applied)."""
-        return _sync_firewall(dry_run=True)
-
-    @app.post("/api/firewall/apply")
-    def firewall_apply():
-        """Apply the blocklist sync (add new / remove stale entries)."""
-        return _sync_firewall(dry_run=False)
-
-    # --- whitelist ---------------------------------------------------------
-
-    @app.get("/api/whitelist")
-    def whitelist():
-        addr_v4 = _read_json(
-            os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
-        addr_v6 = _read_json(
-            os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json"))
-        result: dict[str, dict[str, list[str]]] = {}
-        for name in WHITELIST_NAMES:
-            result[name] = {
-                "ipv4": sorted(e.get("address") for e in addr_v4
-                               if e.get("list") == name),
-                "ipv6": sorted(e.get("address") for e in addr_v6
-                               if e.get("list") == name),
-            }
-        return jsonify({"names": WHITELIST_NAMES, "whitelists": result})
-
-    @app.post("/api/whitelist/update")
-    def whitelist_update():
-        data = request.get_json(silent=True) or {}
-        name = (data.get("name") or "").strip()
-        address = (data.get("address") or "").strip()
-        action = (data.get("action") or "add").strip().lower()
-        version = ip_version(address)
-
-        if name not in WHITELIST_NAMES:
-            return jsonify({"error": f"unknown whitelist: {name}"}), 400
-        if not address or version not in (4, 6):
-            return jsonify({"error": "missing or invalid 'address' field"}), 400
-        if action not in ("add", "remove"):
-            return jsonify({"error": "action must be 'add' or 'remove'"}), 400
-
-        def _update(fw):
-            path = "/ip" if version == 4 else "/ipv6"
-            if action == "add":
-                if version == 4:
-                    fw.write_address_list_ipv4(name, [address])
-                else:
-                    fw.write_address_list_ipv6(name, [address])
-            else:
-                fw.execute_command(
-                    f"{path} firewall address-list remove "
-                    f"[find list={name} address={address}]")
-            return jsonify({"ok": True, "action": action,
-                            "name": name, "address": address})
-
-        return _with_router(_update)
-
-    return app
-
-
-def serve(host: str = WEB_HOST, port: int = WEB_PORT) -> None:
-    """Run the web server on localhost at the configured high port."""
-    app = create_app()
-    print(f"Intrusion Detection web UI running at http://{host}:{port}")
-    app.run(host=host, port=port, use_reloader=False)
-
-
-if __name__ == "__main__":
-    serve()
+#!/usr/bin/env python3
+"""Route definitions for the intrusion-detection web UI."""
+
+from __future__ import annotations
+
+import csv
+import io
+import json
+import os
+
+from flask import Flask, Response, abort, jsonify, render_template, request
+from werkzeug.utils import secure_filename
+
+from ..utils.constants import (
+    APP_REPORT_DATA_DIR,
+    BLOCKLIST_NAME,
+    MIKROTIK_DATA_DIR,
+    WEB_HOST,
+    WEB_PORT,
+    WHITELIST_NAMES,
+)
+from ..utils.ip_utils import ip_version
+
+
+def register_routes(app: Flask,
+                    report_sources: dict,
+                    scan_kinds: tuple,
+                    web_scan,
+                    helpers) -> None:
+    """Register all web UI routes on the Flask application."""
+
+    _count_lines = helpers["_count_lines"]
+    _read_lines = helpers["_read_lines"]
+    _read_json = helpers["_read_json"]
+    _params_from_request = helpers["_params_from_request"]
+    _with_router = helpers["_with_router"]
+    _sync_firewall = helpers["_sync_firewall"]
+
+    # --- log analysis (SQLite-backed) --------------------------------------
+
+    _analysis = helpers.get("_analysis")
+
+    @app.get("/analysis")
+    def analysis_page():
+        return render_template("analysis.html")
+
+    @app.get("/api/analysis/top-ips")
+    def analysis_top_ips():
+        if _analysis is None:
+            return jsonify({"error": "analysis unavailable"}), 503
+        since = (request.args.get("since") or "").strip() or None
+        limit = request.args.get("limit", 50, type=int)
+        return jsonify({"top_ips": _analysis["top_ips"](since=since, limit=limit)})
+
+    @app.get("/api/analysis/ip/<ip>")
+    def analysis_ip(ip):
+        if _analysis is None:
+            return jsonify({"error": "analysis unavailable"}), 503
+        return jsonify({"ip": ip,
+                        "timeline": _analysis["ip_timeline"](ip)})
+
+    @app.get("/api/analysis/cross-source")
+    def analysis_cross_source():
+        if _analysis is None:
+            return jsonify({"error": "analysis unavailable"}), 503
+        limit = request.args.get("limit", 100, type=int)
+        return jsonify({"ips": _analysis["cross_source"](limit=limit)})
+
+    @app.get("/api/analysis/manifest")
+    def analysis_manifest():
+        return jsonify({"ok": _analysis is not None})
+
+    # --- pages -------------------------------------------------------------
+
+    @app.get("/")
+    def index():
+        return render_template("dashboard.html")
+
+    @app.get("/scan")
+    def scan_page():
+        return render_template("scan.html")
+
+    @app.get("/reports")
+    def reports_page():
+        return render_template("reports.html")
+
+    @app.get("/blocklist")
+    def blocklist_page():
+        return render_template("blocklist.html")
+
+    @app.get("/firewall")
+    def firewall_page():
+        return render_template("firewall.html")
+
+    @app.get("/whitelist")
+    def whitelist_page():
+        return render_template("whitelist.html", whitelist_names=WHITELIST_NAMES)
+
+    @app.get("/graph")
+    def graph_page():
+        return render_template("graph.html")
+
+    # --- status ------------------------------------------------------------
+
+    @app.get("/api/status")
+    def status():
+        v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
+        v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
+        analysis = {}
+        if _analysis is not None:
+            try:
+                conn = _analysis["conn"]()
+                analysis = {
+                    "available": True,
+                    "request_count": _analysis["request_count"](conn),
+                    "scan_count": _analysis["scan_count"](conn),
+                }
+            except Exception:
+                analysis = {"available": False}
+        return jsonify({
+            "host": WEB_HOST,
+            "port": WEB_PORT,
+            "scan_running": web_scan.is_scan_running(),
+            "latest_scans": web_scan.latest_scans(5),
+            "blocklist_ipv4_count": _count_lines(v4_path),
+            "blocklist_ipv6_count": _count_lines(v6_path),
+            "firewall_snapshot": {
+                "exists": os.path.isdir(MIKROTIK_DATA_DIR),
+                "path": MIKROTIK_DATA_DIR,
+            },
+            "analysis": analysis,
+        })
+
+    # --- scans -------------------------------------------------------------
+
+    @app.post("/api/scan/<kind>")
+    def start_scan(kind):
+        if kind not in scan_kinds:
+            abort(404, f"unknown scan kind: {kind}")
+        job = web_scan.start_scan(kind, _params_from_request())
+        if job.get("error"):
+            return jsonify(job), 409
+        return jsonify(job), 202
+
+    @app.get("/api/scan/status")
+    def scan_status():
+        return jsonify({
+            "running": web_scan.is_scan_running(),
+            "latest_scans": web_scan.latest_scans(10),
+        })
+
+    @app.get("/api/scan/<job_id>")
+    def scan_job(job_id):
+        job = web_scan.get_scan(job_id)
+        if job is None:
+            abort(404, f"unknown job: {job_id}")
+        return jsonify(job)
+
+    # --- reports -----------------------------------------------------------
+
+    @app.get("/api/reports")
+    def list_report_artifacts():
+        source = (request.args.get("source") or "").strip() or None
+        if source is not None and source not in report_sources:
+            return jsonify({"error": "unknown report source"}), 400
+        limit = request.args.get("limit", default=50, type=int)
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            result = repo.list_report_artifacts(
+                get_connection(), source=source,
+                report_type=(request.args.get("type") or "").strip() or None,
+                since=(request.args.get("since") or "").strip() or None,
+                until=(request.args.get("until") or "").strip() or None,
+                limit=limit,
+                cursor=(request.args.get("cursor") or "").strip() or None,
+            )
+            return jsonify(result)
+        except ValueError as exc:
+            return jsonify({"error": str(exc)}), 400
+        except Exception as exc:
+            return jsonify({"error": f"report database unavailable: {exc}"}), 503
+
+    @app.get("/api/report-artifacts/<int:artifact_id>")
+    def get_report_artifact(artifact_id: int):
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            artifact = repo.get_report_artifact(get_connection(), artifact_id)
+        except Exception as exc:
+            return jsonify({"error": f"report database unavailable: {exc}"}), 503
+        if artifact is None:
+            abort(404, "report artifact not found")
+        return jsonify(artifact)
+
+    @app.get("/api/report-artifacts/<int:artifact_id>/export")
+    def export_report_artifact(artifact_id: int):
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            artifact = repo.get_report_artifact(get_connection(), artifact_id)
+        except Exception as exc:
+            return jsonify({"error": f"report database unavailable: {exc}"}), 503
+        if artifact is None:
+            abort(404, "report artifact not found")
+
+        payload = artifact["payload"]
+        export_format = (request.args.get("format") or "json").lower()
+        filename = secure_filename(os.path.basename(
+            artifact["original_path"])) or f"report-{artifact_id}"
+        if export_format == "json":
+            content = json.dumps(payload, ensure_ascii=False, indent=2)
+            mimetype = "application/json"
+            extension = ".json"
+        elif export_format == "txt" and payload.get("format") == "text":
+            content = payload.get("content", "")
+            mimetype = "text/plain; charset=utf-8"
+            extension = ".txt"
+        elif export_format == "csv" and payload.get("format") == "csv":
+            output = io.StringIO(newline="")
+            writer = csv.DictWriter(
+                output, fieldnames=payload.get("columns", []),
+                extrasaction="ignore")
+            writer.writeheader()
+            writer.writerows(payload.get("rows", []))
+            content = output.getvalue()
+            mimetype = "text/csv; charset=utf-8"
+            extension = ".csv"
+        else:
+            return jsonify({"error": "unsupported export format"}), 406
+
+        download_name = (filename if filename.lower().endswith(extension)
+                         else f"{filename}{extension}")
+        response = Response(content, mimetype=mimetype)
+        response.headers["Content-Disposition"] = (
+            f'attachment; filename="{download_name}"')
+        return response
+
+    @app.get("/api/reports/<source>")
+    def list_reports(source):
+        base = report_sources.get(source)
+        if base is None:
+            abort(404, f"unknown report source: {source}")
+        names = sorted(
+            n for n in os.listdir(base)
+            if os.path.isfile(os.path.join(base, n)))
+        return jsonify({"source": source, "files": names})
+
+    @app.get("/api/reports/<source>/<path:name>")
+    def get_report(source, name):
+        base = report_sources.get(source)
+        if base is None:
+            abort(404, f"unknown report source: {source}")
+        full = os.path.realpath(os.path.join(base, name))
+        if not full.startswith(os.path.realpath(base)):
+            abort(403, "invalid report path")
+        if not os.path.isfile(full):
+            abort(404, "report not found")
+        with open(full, "r", encoding="utf-8", errors="replace") as f:
+            content = f.read()
+        return jsonify({"source": source, "name": name, "content": content})
+
+    # --- blocklist ---------------------------------------------------------
+
+    @app.get("/api/blocklist")
+    def blocklist():
+        return jsonify({
+            "ipv4": _read_lines(
+                os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")),
+            "ipv6": _read_lines(
+                os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")),
+        })
+
+    @app.post("/api/blocklist/add")
+    def blocklist_add():
+        data = request.get_json(silent=True) or {}
+        ip = (data.get("ip") or "").strip()
+        version = ip_version(ip)
+        if not ip or version not in (4, 6):
+            return jsonify({"error": "missing or invalid 'ip' field"}), 400
+
+        def _add(fw):
+            if version == 4:
+                fw.write_address_list_ipv4(BLOCKLIST_NAME, [ip])
+            else:
+                fw.write_address_list_ipv6(BLOCKLIST_NAME, [ip])
+            return jsonify({"ok": True, "action": "add", "ip": ip})
+
+        return _with_router(_add)
+
+    @app.post("/api/blocklist/remove")
+    def blocklist_remove():
+        data = request.get_json(silent=True) or {}
+        ip = (data.get("ip") or "").strip()
+        version = ip_version(ip)
+        if not ip or version not in (4, 6):
+            return jsonify({"error": "missing or invalid 'ip' field"}), 400
+
+        def _remove(fw):
+            path = "/ip" if version == 4 else "/ipv6"
+            fw.execute_command(
+                f"{path} firewall address-list remove "
+                f"[find list={BLOCKLIST_NAME} address={ip}]")
+            return jsonify({"ok": True, "action": "remove", "ip": ip})
+
+        return _with_router(_remove)
+
+    # --- graph -------------------------------------------------------------
+
+    @app.get("/api/graph/ipv4")
+    def graph_ipv4():
+        blocked = _read_lines(
+            os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt"))
+        addr_v4 = _read_json(
+            os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
+        whitelisted = sorted({e.get("address") for e in addr_v4
+                              if e.get("list") in WHITELIST_NAMES
+                              and e.get("address")})
+        return jsonify({
+            "blocked": blocked,
+            "whitelisted": whitelisted,
+            "counts": {
+                "blocked": len(blocked),
+                "whitelisted": len(whitelisted),
+            },
+            "blocklist_name": BLOCKLIST_NAME,
+            "whitelist_names": WHITELIST_NAMES,
+        })
+
+    # --- firewall ----------------------------------------------------------
+
+    @app.get("/api/firewall/state")
+    def firewall_state():
+        return jsonify({
+            "address_lists": {
+                "ipv4": _read_json(
+                    os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json")),
+                "ipv6": _read_json(
+                    os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json")),
+            },
+            "firewall_rules": {
+                "ipv4": _read_json(
+                    os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv4.json")),
+                "ipv6": _read_json(
+                    os.path.join(MIKROTIK_DATA_DIR, "firewall_ipv6.json")),
+            },
+            "snapshot_dir": MIKROTIK_DATA_DIR,
+        })
+
+    @app.get("/api/firewall/snapshots")
+    def firewall_snapshots():
+        """Recent firewall snapshots persisted in the SQLite store (F001)."""
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            conn = get_connection()
+            limit = request.args.get("limit", 10, type=int)
+            return jsonify({"snapshots": repo.recent_firewall_snapshots(
+                conn, limit=limit)})
+        except Exception as exc:
+            return jsonify({"error": str(exc), "snapshots": []}), 503
+
+    @app.post("/api/firewall/sync")
+    def firewall_sync():
+        return _sync_firewall(dry_run=True)
+
+    @app.post("/api/firewall/apply")
+    def firewall_apply():
+        return _sync_firewall(dry_run=False)
+
+    # --- whitelist ---------------------------------------------------------
+
+    @app.get("/api/whitelist")
+    def whitelist():
+        addr_v4 = _read_json(
+            os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv4.json"))
+        addr_v6 = _read_json(
+            os.path.join(MIKROTIK_DATA_DIR, "address_list_ipv6.json"))
+        result: dict[str, dict[str, list[str]]] = {}
+        for name in WHITELIST_NAMES:
+            result[name] = {
+                "ipv4": sorted(e.get("address") for e in addr_v4
+                               if e.get("list") == name),
+                "ipv6": sorted(e.get("address") for e in addr_v6
+                               if e.get("list") == name),
+            }
+        return jsonify({"names": WHITELIST_NAMES, "whitelists": result})
+
+    @app.post("/api/whitelist/update")
+    def whitelist_update():
+        data = request.get_json(silent=True) or {}
+        name = (data.get("name") or "").strip()
+        address = (data.get("address") or "").strip()
+        action = (data.get("action") or "add").strip().lower()
+        version = ip_version(address)
+
+        if name not in WHITELIST_NAMES:
+            return jsonify({"error": f"unknown whitelist: {name}"}), 400
+        if not address or version not in (4, 6):
+            return jsonify({"error": "missing or invalid 'address' field"}), 400
+        if action not in ("add", "remove"):
+            return jsonify({"error": "action must be 'add' or 'remove'"}), 400
+
+        def _update(fw):
+            path = "/ip" if version == 4 else "/ipv6"
+            if action == "add":
+                if version == 4:
+                    fw.write_address_list_ipv4(name, [address])
+                else:
+                    fw.write_address_list_ipv6(name, [address])
+            else:
+                fw.execute_command(
+                    f"{path} firewall address-list remove "
+                    f"[find list={name} address={address}]")
+            return jsonify({"ok": True, "action": action,
+                            "name": name, "address": address})
+
+        return _with_router(_update)

app/web/web_scan.py

@@ -1,360 +1,408 @@
-#!/usr/bin/env python3
-"""Background scan runner for the web UI.
-
-Runs scans in a daemon thread so HTTP requests never block, keeps an
-in-memory job store, and serializes scans with a busy lock (WHOIS and
-whitelist refresh must not run concurrently).
-"""
-
-from __future__ import annotations
-
-import json
-import os
-import threading
-import uuid
-from dataclasses import asdict, dataclass, field
-from datetime import datetime
-from typing import Any, Literal
-
-from .constants import (
-    APP_REPORT_DATA_DIR,
-    COPYPARTY_REPORT_DIR,
-    MIKROTIK_REPORT_DIR,
-    NPM_REPORT_DIR,
-    SCAN_JOBS_MAX,
-    SCAN_JOBS_PATH,
-)
-from .copyparty import CopypartyAnalyzer
-from .npm import NpmAnalyzer
-from .scan import ScanParams, ScanResult, run_scan
-
-ScanKind = Literal["copyparty", "npm", "consolidated"]
-
-#: serializes scans — only one runs at a time
-_scan_lock = threading.Lock()
-
-#: in-memory job store (job_id -> ScanJob)
-_jobs: dict[str, "ScanJob"] = {}
-_jobs_lock = threading.Lock()
-
-#: True once persisted jobs have been loaded from disk
-_loaded = False
-
-
-def _now() -> str:
-    return datetime.now().astimezone().isoformat(timespec="seconds")
-
-
-@dataclass
-class ScanJob:
-    """One background scan job."""
-
-    job_id: str
-    kind: ScanKind
-    params: ScanParams
-    state: str = "queued"  # queued | running | done | error
-    started: str | None = None
-    finished: str | None = None
-    result: ScanResult | None = None
-    error: str | None = None
-    busy: bool = False
-    details: dict = field(default_factory=dict)
-
-    def to_dict(self) -> dict[str, Any]:
-        result = self.result
-        return {
-            "job_id": self.job_id,
-            "kind": self.kind,
-            "params": asdict(self.params),
-            "state": self.state,
-            "started": self.started,
-            "finished": self.finished,
-            "error": self.error,
-            "busy": self.busy,
-            "details": self.details,
-            "result": None if result is None else {
-                "ok": result.ok,
-                "cp_exists": result.cp_exists,
-                "npm_exists": result.npm_exists,
-                "cp_candidates": result.cp_candidates,
-                "npm_candidates": result.npm_candidates,
-                "blocked_ips": result.blocked_ips,
-                "blocked_targets": result.blocked_targets,
-                "blocked_ipv4": result.blocked_ipv4,
-                "blocked_ipv6": result.blocked_ipv6,
-                "blocklist_v4_path": result.blocklist_v4_path,
-                "blocklist_v6_path": result.blocklist_v6_path,
-                "firewall_connected": result.firewall_connected,
-                "firewall_changes": result.firewall_changes,
-                "errors": result.errors,
-            },
-        }
-
-
-def start_scan(kind: ScanKind, params: ScanParams) -> dict[str, Any]:
-    """Start a scan in the background and return its job dict.
-
-    If another scan is already running, returns a dict with an ``error`` key.
-    """
-    if not _scan_lock.acquire(blocking=False):
-        return {"error": "a scan is already running; try again when it finishes"}
-
-    job = ScanJob(job_id=uuid.uuid4().hex[:12], kind=kind,
-                  params=params, busy=True)
-    with _jobs_lock:
-        _jobs[job.job_id] = job
-    _save_jobs()
-
-    def _run() -> None:
-        job.started = _now()
-        job.state = "running"
-        try:
-            if kind == "consolidated":
-                job.result = run_scan(params)
-            elif kind == "copyparty":
-                info = CopypartyAnalyzer(
-                    log_path=params.copyparty_log_path,
-                    report_dir=params.copyparty_report_dir,
-                    top=params.top,
-                    quiet=params.quiet,
-                    full=params.full,
-                    min_denied=params.min_denied,
-                ).run()
-                job.result = ScanResult(ok=True)
-                job.details = {"report": info.get("report"),
-                               "data_dir": info.get("data_dir")}
-            elif kind == "npm":
-                info = NpmAnalyzer(
-                    log_path=params.npm_log_path,
-                    report_dir=params.npm_report_dir,
-                    top=params.top,
-                    quiet=params.quiet,
-                    full=params.full,
-                    min_errors=params.min_errors,
-                    error_ratio=params.error_ratio,
-                ).run()
-                job.result = ScanResult(ok=True)
-                job.details = {"report": info.get("report"),
-                               "data_dir": info.get("data_dir")}
-            else:  # pragma: no cover - guarded by type hint
-                raise ValueError(f"unknown scan kind: {kind}")
-            job.state = "done"
-        except Exception as e:
-            job.error = str(e)
-            job.state = "error"
-        finally:
-            job.finished = _now()
-            job.busy = False
-            _scan_lock.release()
-            _save_jobs()
-
-    threading.Thread(target=_run, daemon=True,
-                     name=f"scan-{job.job_id}").start()
-    return job.to_dict()
-
-
-def get_scan(job_id: str) -> dict[str, Any] | None:
-    """Return a single job as a dict, or None if unknown."""
-    load_persisted()
-    with _jobs_lock:
-        job = _jobs.get(job_id)
-    return job.to_dict() if job else None
-
-
-def _iso_key(value: str | None) -> float:
-    """ISO-8601 string -> epoch seconds (0.0 when unparseable/empty)."""
-    if not value:
-        return 0.0
-    try:
-        return datetime.fromisoformat(value).timestamp()
-    except ValueError:
-        return 0.0
-
-
-def latest_file_scan() -> dict[str, Any] | None:
-    """Build a ``done`` scan entry from the newest report files on disk.
-
-    Lets the dashboard show the latest completed scan even when it was not
-    run through the web UI (server restart, CLI-run scans). Returns None when
-    no scan outputs exist yet.
-    """
-
-    def newest(dirpath: str, suffix: str = "") -> str | None:
-        """Newest file in ``dirpath`` (optionally ending in ``suffix``)."""
-        try:
-            names = [n for n in os.listdir(dirpath)
-                     if os.path.isfile(os.path.join(dirpath, n))
-                     and (not suffix or n.endswith(suffix))]
-        except OSError:
-            return None
-        return os.path.join(dirpath, max(names)) if names else None
-
-    mikrotik = newest(MIKROTIK_REPORT_DIR, ".json")
-    cp_report = newest(COPYPARTY_REPORT_DIR, ".txt")
-    npm_report = newest(NPM_REPORT_DIR, ".txt")
-    v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
-    v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
-    have_outputs = (mikrotik is not None or cp_report is not None
-                    or npm_report is not None
-                    or os.path.exists(v4_path) or os.path.exists(v6_path))
-    if not have_outputs:
-        return None
-
-    # Timestamp: prefer the newest snapshot's collected_at, else the newest
-    # report file's mtime, else now.
-    timestamp: str | None = None
-    if mikrotik:
-        try:
-            with open(mikrotik, "r", encoding="utf-8") as f:
-                timestamp = json.load(f).get("collected_at")
-        except (OSError, ValueError):
-            timestamp = None
-    if not timestamp:
-        candidates = [p for p in (mikrotik, cp_report, npm_report,
-                                  v4_path, v6_path)
-                      if p and os.path.exists(p)]
-        if candidates:
-            tz = datetime.now().astimezone().tzinfo
-            timestamp = datetime.fromtimestamp(
-                max(os.path.getmtime(p) for p in candidates),
-                tz=tz).isoformat(timespec="seconds")
-    if not timestamp:
-        timestamp = _now()
-
-    blocked_targets = 0
-    for path in (v4_path, v6_path):
-        if os.path.exists(path):
-            try:
-                with open(path, "r", encoding="utf-8") as f:
-                    blocked_targets += sum(1 for _ in f)
-            except OSError:
-                pass
-
-    return {
-        "job_id": "files-latest",
-        "kind": "consolidated",
-        "state": "done",
-        "started": timestamp,
-        "finished": timestamp,
-        "error": None,
-        "busy": False,
-        "details": {},
-        "result": {
-            "ok": True,
-            "cp_exists": cp_report is not None,
-            "npm_exists": npm_report is not None,
-            "cp_candidates": 0,
-            "npm_candidates": 0,
-            "blocked_ips": blocked_targets,
-            "blocked_targets": blocked_targets,
-            "blocked_ipv4": [],
-            "blocked_ipv6": [],
-            "blocklist_v4_path": v4_path,
-            "blocklist_v6_path": v6_path,
-            "firewall_connected": mikrotik is not None,
-            "firewall_changes": None,
-            "errors": [],
-        },
-    }
-
-
-def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
-    """Return the most recently started jobs, newest first.
-
-    Appends a file-derived "latest scan" entry when the newest report files on
-    disk reflect a scan newer than any known job (e.g. after a restart or a
-    CLI run), so the dashboard always shows the latest completed scan.
-    """
-    load_persisted()
-    with _jobs_lock:
-        jobs = sorted(_jobs.values(),
-                      key=lambda j: j.started or "", reverse=True)
-    entries = [j.to_dict() for j in jobs]
-    file_entry = latest_file_scan()
-    if file_entry is not None:
-        newest = jobs[0] if jobs else None
-        newest_ts = _iso_key(
-            (newest.finished or newest.started) if newest else None)
-        # Only surface the file-derived entry when it is genuinely newer than
-        # every known job (tolerance avoids duplicating the run that produced
-        # it, where collected_at lands a moment after the job finished).
-        if newest_ts == 0 or _iso_key(file_entry["started"]) - newest_ts > 2.0:
-            entries.append(file_entry)
-    entries.sort(key=lambda e: _iso_key(e.get("started")), reverse=True)
-    return entries[:limit]
-
-
-# ---------------------------------------------------------------------------
-# persistence
-# ---------------------------------------------------------------------------
-
-def _save_jobs() -> None:
-    """Persist the newest jobs to ``SCAN_JOBS_PATH`` (atomic tmp+rename)."""
-    with _jobs_lock:
-        jobs = sorted(_jobs.values(),
-                      key=lambda j: j.started or "", reverse=True)
-        payload = [j.to_dict() for j in jobs[:SCAN_JOBS_MAX]]
-    os.makedirs(os.path.dirname(SCAN_JOBS_PATH), exist_ok=True)
-    tmp = SCAN_JOBS_PATH + ".tmp"
-    with open(tmp, "w", encoding="utf-8") as f:
-        json.dump(payload, f, indent=2, sort_keys=True)
-    os.replace(tmp, SCAN_JOBS_PATH)
-
-
-def _load_jobs() -> None:
-    """Rebuild the in-memory job store from ``SCAN_JOBS_PATH``."""
-    try:
-        with open(SCAN_JOBS_PATH, "r", encoding="utf-8") as f:
-            payload = json.load(f)
-    except (OSError, ValueError, TypeError):
-        payload = []
-    if not isinstance(payload, list):
-        return
-
-    jobs: dict[str, ScanJob] = {}
-    for item in payload:
-        if not isinstance(item, dict):
-            continue
-        try:
-            params = ScanParams(**item.get("params", {}))
-        except (TypeError, ValueError):
-            continue
-        try:
-            job = ScanJob(
-                job_id=str(item["job_id"]),
-                kind=item.get("kind", "consolidated"),
-                params=params,
-                state=item.get("state", "done"),
-                started=item.get("started"),
-                finished=item.get("finished"),
-                error=item.get("error"),
-                busy=False,
-                details=item.get("details", {}) or {},
-            )
-        except (KeyError, TypeError, ValueError):
-            continue
-        result = item.get("result")
-        if isinstance(result, dict):
-            try:
-                job.result = ScanResult(**result)
-            except TypeError:
-                job.result = None
-        jobs[job.job_id] = job
-
-    with _jobs_lock:
-        _jobs.clear()
-        _jobs.update(jobs)
-
-
-def load_persisted() -> None:
-    """Load persisted jobs from disk once (idempotent)."""
-    global _loaded
-    if _loaded:
-        return
-    _load_jobs()
-    _loaded = True
-
-
-def is_scan_running() -> bool:
-    """True if any job currently holds the scan lock."""
-    return _scan_lock.locked()
+#!/usr/bin/env python3
+"""Background scan runner for the web UI.
+
+Runs scans in a daemon thread so HTTP requests never block, keeps an
+in-memory job store, and serializes scans with a busy lock (WHOIS and
+whitelist refresh must not run concurrently).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import threading
+import uuid
+from dataclasses import asdict, dataclass, field
+from datetime import datetime
+from typing import Any, Literal
+
+from ..utils.constants import (
+    APP_REPORT_DATA_DIR,
+    COPYPARTY_REPORT_DIR,
+    MIKROTIK_REPORT_DIR,
+    NPM_REPORT_DIR,
+    SCAN_JOBS_MAX,
+    SCAN_JOBS_PATH,
+)
+from ..copyparty import CopypartyAnalyzer
+from ..npm import NpmAnalyzer
+from ..scan import ScanParams, ScanResult, run_scan
+
+ScanKind = Literal["copyparty", "npm", "consolidated"]
+
+#: serializes scans — only one runs at a time
+_scan_lock = threading.Lock()
+
+#: in-memory job store (job_id -> ScanJob)
+_jobs: dict[str, "ScanJob"] = {}
+_jobs_lock = threading.Lock()
+
+#: True once persisted jobs have been loaded from disk
+_loaded = False
+
+
+def _now() -> str:
+    return datetime.now().astimezone().isoformat(timespec="seconds")
+
+
+@dataclass
+class ScanJob:
+    """One background scan job."""
+
+    job_id: str
+    kind: ScanKind
+    params: ScanParams
+    state: str = "queued"  # queued | running | done | error
+    started: str | None = None
+    finished: str | None = None
+    result: ScanResult | None = None
+    error: str | None = None
+    busy: bool = False
+    details: dict = field(default_factory=dict)
+
+    def to_dict(self) -> dict[str, Any]:
+        result = self.result
+        return {
+            "job_id": self.job_id,
+            "kind": self.kind,
+            "params": asdict(self.params),
+            "state": self.state,
+            "started": self.started,
+            "finished": self.finished,
+            "error": self.error,
+            "busy": self.busy,
+            "details": self.details,
+            "result": None if result is None else {
+                "ok": result.ok,
+                "cp_exists": result.cp_exists,
+                "npm_exists": result.npm_exists,
+                "cp_candidates": result.cp_candidates,
+                "npm_candidates": result.npm_candidates,
+                "blocked_ips": result.blocked_ips,
+                "blocked_targets": result.blocked_targets,
+                "blocked_ipv4": result.blocked_ipv4,
+                "blocked_ipv6": result.blocked_ipv6,
+                "blocklist_v4_path": result.blocklist_v4_path,
+                "blocklist_v6_path": result.blocklist_v6_path,
+                "firewall_connected": result.firewall_connected,
+                "firewall_changes": result.firewall_changes,
+                "errors": result.errors,
+            },
+        }
+
+
+def start_scan(kind: ScanKind, params: ScanParams) -> dict[str, Any]:
+    """Start a scan in the background and return its job dict.
+
+    If another scan is already running, returns a dict with an ``error`` key.
+    """
+    if not _scan_lock.acquire(blocking=False):
+        return {"error": "a scan is already running; try again when it finishes"}
+
+    job = ScanJob(job_id=uuid.uuid4().hex[:12], kind=kind,
+                  params=params, busy=True)
+    with _jobs_lock:
+        _jobs[job.job_id] = job
+    _save_jobs()
+
+    def _run() -> None:
+        job.started = _now()
+        job.state = "running"
+        try:
+            if kind == "consolidated":
+                job.result = run_scan(params)
+            elif kind == "copyparty":
+                info = CopypartyAnalyzer(
+                    log_path=params.copyparty_log_path,
+                    report_dir=params.copyparty_report_dir,
+                    top=params.top,
+                    quiet=params.quiet,
+                    full=params.full,
+                    min_denied=params.min_denied,
+                ).run()
+                job.result = ScanResult(ok=True)
+                job.details = {"report": info.get("report"),
+                               "data_dir": info.get("data_dir")}
+            elif kind == "npm":
+                info = NpmAnalyzer(
+                    log_path=params.npm_log_path,
+                    report_dir=params.npm_report_dir,
+                    top=params.top,
+                    quiet=params.quiet,
+                    full=params.full,
+                    min_errors=params.min_errors,
+                    error_ratio=params.error_ratio,
+                ).run()
+                job.result = ScanResult(ok=True)
+                job.details = {"report": info.get("report"),
+                               "data_dir": info.get("data_dir")}
+            else:  # pragma: no cover - guarded by type hint
+                raise ValueError(f"unknown scan kind: {kind}")
+            job.state = "done"
+        except Exception as e:
+            job.error = str(e)
+            job.state = "error"
+        finally:
+            job.finished = _now()
+            job.busy = False
+            _scan_lock.release()
+            _save_jobs()
+
+    threading.Thread(target=_run, daemon=True,
+                     name=f"scan-{job.job_id}").start()
+    return job.to_dict()
+
+
+def get_scan(job_id: str) -> dict[str, Any] | None:
+    """Return a single job as a dict, or None if unknown."""
+    load_persisted()
+    with _jobs_lock:
+        job = _jobs.get(job_id)
+    return job.to_dict() if job else None
+
+
+def _iso_key(value: str | None) -> float:
+    """ISO-8601 string -> epoch seconds (0.0 when unparseable/empty)."""
+    if not value:
+        return 0.0
+    try:
+        return datetime.fromisoformat(value).timestamp()
+    except ValueError:
+        return 0.0
+
+
+def latest_file_scan() -> dict[str, Any] | None:
+    """Build a ``done`` scan entry from the newest report files on disk.
+
+    Lets the dashboard show the latest completed scan even when it was not
+    run through the web UI (server restart, CLI-run scans). Returns None when
+    no scan outputs exist yet.
+    """
+
+    def newest(dirpath: str, suffix: str = "") -> str | None:
+        """Newest file in ``dirpath`` (optionally ending in ``suffix``)."""
+        try:
+            names = [n for n in os.listdir(dirpath)
+                     if os.path.isfile(os.path.join(dirpath, n))
+                     and (not suffix or n.endswith(suffix))]
+        except OSError:
+            return None
+        return os.path.join(dirpath, max(names)) if names else None
+
+    mikrotik = newest(MIKROTIK_REPORT_DIR, ".json")
+    cp_report = newest(COPYPARTY_REPORT_DIR, ".txt")
+    npm_report = newest(NPM_REPORT_DIR, ".txt")
+    v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
+    v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
+    have_outputs = (mikrotik is not None or cp_report is not None
+                    or npm_report is not None
+                    or os.path.exists(v4_path) or os.path.exists(v6_path))
+    if not have_outputs:
+        return None
+
+    # Timestamp: prefer the newest snapshot's collected_at, else the newest
+    # report file's mtime, else now.
+    timestamp: str | None = None
+    if mikrotik:
+        try:
+            with open(mikrotik, "r", encoding="utf-8") as f:
+                timestamp = json.load(f).get("collected_at")
+        except (OSError, ValueError):
+            timestamp = None
+    if not timestamp:
+        candidates = [p for p in (mikrotik, cp_report, npm_report,
+                                  v4_path, v6_path)
+                      if p and os.path.exists(p)]
+        if candidates:
+            tz = datetime.now().astimezone().tzinfo
+            timestamp = datetime.fromtimestamp(
+                max(os.path.getmtime(p) for p in candidates),
+                tz=tz).isoformat(timespec="seconds")
+    if not timestamp:
+        timestamp = _now()
+
+    blocked_targets = 0
+    for path in (v4_path, v6_path):
+        if os.path.exists(path):
+            try:
+                with open(path, "r", encoding="utf-8") as f:
+                    blocked_targets += sum(1 for _ in f)
+            except OSError:
+                pass
+
+    return {
+        "job_id": "files-latest",
+        "kind": "consolidated",
+        "state": "done",
+        "started": timestamp,
+        "finished": timestamp,
+        "error": None,
+        "busy": False,
+        "details": {},
+        "result": {
+            "ok": True,
+            "cp_exists": cp_report is not None,
+            "npm_exists": npm_report is not None,
+            "cp_candidates": 0,
+            "npm_candidates": 0,
+            "blocked_ips": blocked_targets,
+            "blocked_targets": blocked_targets,
+            "blocked_ipv4": [],
+            "blocked_ipv6": [],
+            "blocklist_v4_path": v4_path,
+            "blocklist_v6_path": v6_path,
+            "firewall_connected": mikrotik is not None,
+            "firewall_changes": None,
+            "errors": [],
+        },
+    }
+
+
+def _db_scans(limit: int = 10) -> list[dict[str, Any]]:
+    """Recent scan records from the SQLite ``scans`` table (S001).
+
+    Converts DB rows into the web job-dict shape so the dashboard can surface
+    scans that were persisted by ``run_scan`` / analyzers even if they were
+    never run through the in-memory job store (e.g. CLI scans or restarts).
+    Returns [] when the DB is unavailable.
+    """
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        conn = get_connection()
+        rows = repo.latest_scans(conn, limit=limit)
+    except Exception:
+        return []
+    out = []
+    for r in rows:
+        try:
+            result_json = r.get("result_json")
+            result = json.loads(result_json) if result_json else {}
+        except (TypeError, ValueError):
+            result = {}
+        out.append({
+            "job_id": f"db-scan-{r['id']}",
+            "kind": r.get("kind") or "consolidated",
+            "state": r.get("status") or "done",
+            "started": r.get("started_at"),
+            "finished": r.get("finished_at"),
+            "error": r.get("error"),
+            "busy": False,
+            "details": {},
+            "result": result or None,
+        })
+    return out
+
+
+def latest_scans(limit: int = 10) -> list[dict[str, Any]]:
+    """Return the most recently started jobs, newest first.
+
+    Merges three sources, newest-first:
+      1. the in-memory job store (currently running / last-run jobs),
+      2. the SQLite ``scans`` table (persisted by run_scan/analyzers — S001),
+      3. a file-derived "latest scan" fallback for pre-DB runs.
+    Any DB-backed scan not already represented in memory is appended so the
+    dashboard shows scans from CLI runs and across restarts.
+    """
+    load_persisted()
+    with _jobs_lock:
+        jobs = sorted(_jobs.values(),
+                      key=lambda j: j.started or "", reverse=True)
+    entries = [j.to_dict() for j in jobs]
+
+    # Add DB scan records that aren't already in the in-memory store. Key on
+    # (kind, started_at) to avoid duplicating a job that run_scan also logged.
+    known = {(e.get("kind"), e.get("started")) for e in entries}
+    for db_row in _db_scans(limit=limit * 2):
+        if (db_row.get("kind"), db_row.get("started")) not in known:
+            entries.append(db_row)
+
+    file_entry = latest_file_scan()
+    if file_entry is not None:
+        newest = (entries or [None])[0]
+        newest_key = (
+            (newest.get("finished") or newest.get("started")) if newest else None)
+        newest_ts = _iso_key(newest_key)
+        # Only surface the file-derived entry when it is genuinely newer than
+        # every known job (tolerance avoids duplicating the run that produced
+        # it, where collected_at lands a moment after the job finished).
+        if newest_ts == 0 or _iso_key(file_entry["started"]) - newest_ts > 2.0:
+            entries.append(file_entry)
+    entries.sort(key=lambda e: _iso_key(e.get("started")), reverse=True)
+    return entries[:limit]
+
+
+# ---------------------------------------------------------------------------
+# persistence
+# ---------------------------------------------------------------------------
+
+def _save_jobs() -> None:
+    """Persist the newest jobs to ``SCAN_JOBS_PATH`` (atomic tmp+rename)."""
+    with _jobs_lock:
+        jobs = sorted(_jobs.values(),
+                      key=lambda j: j.started or "", reverse=True)
+        payload = [j.to_dict() for j in jobs[:SCAN_JOBS_MAX]]
+    os.makedirs(os.path.dirname(SCAN_JOBS_PATH), exist_ok=True)
+    tmp = SCAN_JOBS_PATH + ".tmp"
+    with open(tmp, "w", encoding="utf-8") as f:
+        json.dump(payload, f, indent=2, sort_keys=True)
+    os.replace(tmp, SCAN_JOBS_PATH)
+
+
+def _load_jobs() -> None:
+    """Rebuild the in-memory job store from ``SCAN_JOBS_PATH``."""
+    try:
+        with open(SCAN_JOBS_PATH, "r", encoding="utf-8") as f:
+            payload = json.load(f)
+    except (OSError, ValueError, TypeError):
+        payload = []
+    if not isinstance(payload, list):
+        return
+
+    jobs: dict[str, ScanJob] = {}
+    for item in payload:
+        if not isinstance(item, dict):
+            continue
+        try:
+            params = ScanParams(**item.get("params", {}))
+        except (TypeError, ValueError):
+            continue
+        try:
+            job = ScanJob(
+                job_id=str(item["job_id"]),
+                kind=item.get("kind", "consolidated"),
+                params=params,
+                state=item.get("state", "done"),
+                started=item.get("started"),
+                finished=item.get("finished"),
+                error=item.get("error"),
+                busy=False,
+                details=item.get("details", {}) or {},
+            )
+        except (KeyError, TypeError, ValueError):
+            continue
+        result = item.get("result")
+        if isinstance(result, dict):
+            try:
+                job.result = ScanResult(**result)
+            except TypeError:
+                job.result = None
+        jobs[job.job_id] = job
+
+    with _jobs_lock:
+        _jobs.clear()
+        _jobs.update(jobs)
+
+
+def load_persisted() -> None:
+    """Load persisted jobs from disk once (idempotent)."""
+    global _loaded
+    if _loaded:
+        return
+    _load_jobs()
+    _loaded = True
+
+
+def is_scan_running() -> bool:
+    """True if any job currently holds the scan lock."""
+    return _scan_lock.locked()

copy_npm_logs.sh

@@ -0,0 +1,14 @@
+#!/bin/sh
+# Shell script to be run as cron job to copy npm logs to backup location.
+
+# Stop npm service to ensure logs are not being written to while copying.
+systemctl stop npm
+
+# Only copy new/changed log files to avoid re-copying existing backups.
+# --ignore-existing: never overwrite files already present in the destination.
+# -c: compare files by checksum so matching files are not re-written.
+# -P: preserve permissions + show progress.
+rsync -qcP --ignore-existing /data/logs/*access* /data/logs/*error* /mnt/backups/npm/
+
+# Start npm service again after copying logs.
+systemctl start npm

data/README.md

@@ -1,15 +0,0 @@
-# Data directory for allucanget-ids
-
-This directory contains data files used by the `allucanget-ids` Intrusion Detection System application, including whitelists, blocklists, and other configuration files. The data files are organized into subdirectories for easier management and maintenance.
-
-## Data files
-
-- [whois cache](./whois_cache.json): Local cache of WHOIS queries to reduce network requests and improve performance. The cache is automatically updated when new IP addresses are queried.
-- [Cloudflare IPv4 list](./cloudflare/ips-v4.txt): Contains a list of Cloudflare IPv4 addresses from `https://www.cloudflare.com/ips-v4`, used for whitelisting traffic from Cloudflare services.
-- [Cloudflare IPv6 list](./cloudflare/ips-v6.txt): Contains a list of Cloudflare IPv6 addresses from `https://www.cloudflare.com/ips-v6`, used for whitelisting traffic from Cloudflare services.
-- [MikroTik RouterOS IPv4 Firewall rules](./mikrotik/firewall_ipv4.json): Current MikroTik RouterOS IPv4 firewall rules.
-- [MikroTik RouterOS IPv6 Firewall rules](./mikrotik/firewall_ipv6.json): Current MikroTik RouterOS IPv6 firewall rules.
-- [MikroTik RouterOS IPv4 address list](./mikrotik/address_list_ipv4.json): Current MikroTik RouterOS IPv4 address list.
-- [MikroTik RouterOS IPv6 address list](./mikrotik/address_list_ipv6.json): Current MikroTik RouterOS IPv6 address list.
-- [Openfactory Geofeed](./openfactory/geofeed.csv): Contains Openfactory IP addresses and associated geolocation data from `https://openfactory.net/geofeed.csv`. Used for identifying and whitelisting traffic from known Openfactory sources.
-- [PaloAlto Scanner IP list](./paloalto/scanner-ips.txt): List of known PaloAlto scanner IP addresses from `https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity`, used for whitelisting traffic from PaloAlto scanners. !NOTE: This list can not be scraped easily (Javascript-based), so it is maintained manually and may be outdated. Please verify the list against the official source if needed.

data/cloudflare/.gitkeep

@@ -1 +0,0 @@
-$ = cloudflare

data/cloudflare/ips-v4.txt

@@ -1,15 +0,0 @@
-173.245.48.0/20
-103.21.244.0/22
-103.22.200.0/22
-103.31.4.0/22
-141.101.64.0/18
-108.162.192.0/18
-190.93.240.0/20
-188.114.96.0/20
-197.234.240.0/22
-198.41.128.0/17
-162.158.0.0/15
-104.16.0.0/13
-104.24.0.0/14
-172.64.0.0/13
-131.0.72.0/22

data/cloudflare/ips-v6.txt

@@ -1,7 +0,0 @@
-2400:cb00::/32
-2606:4700::/32
-2803:f800::/32
-2405:b500::/32
-2405:8100::/32
-2a06:98c0::/29
-2c0f:f248::/32

data/mikrotik/address_list_ipv6.json

@@ -1,353 +0,0 @@
-[
-  {
-    "address": "::/128",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "::1/128",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "fec0::/10",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "::ffff:0.0.0.0/96",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "::/96",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "100::/64",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "2001:db8::/32",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "2001:10::/28",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "3ffe::/16",
-    "comment": "defconf:",
-    "creation-time": "2025-11-21",
-    "dynamic": "no",
-    "list": "bad_ipv6"
-  },
-  {
-    "address": "2a0e:8f04::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a0e:8f05::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a0e:8f06::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a0e:8f07::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a00:6340::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a00:6340:2080::/48",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2001:67c:2648::/48",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a00:6340:2044:200::/56",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a00:6340:2044:600::/56",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2a00:6340:2044:800::/56",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Openfactory"
-  },
-  {
-    "address": "2400:cb00::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Cloudflare"
-  },
-  {
-    "address": "2606:4700::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Cloudflare"
-  },
-  {
-    "address": "2803:f800::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Cloudflare"
-  },
-  {
-    "address": "2405:b500::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Cloudflare"
-  },
-  {
-    "address": "2405:8100::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Cloudflare"
-  },
-  {
-    "address": "2a06:98c0::/29",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Cloudflare"
-  },
-  {
-    "address": "2c0f:f248::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Cloudflare"
-  },
-  {
-    "address": "2604:a940:300:5b6::/64",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "PaloAlto"
-  },
-  {
-    "address": "2604:a940:301:225::/64",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "PaloAlto"
-  },
-  {
-    "address": "2604:a940:302:118::/64",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "PaloAlto"
-  },
-  {
-    "address": "2001:4ca0:108:42::24/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2400:8901::2000:62ff:fe9a:e1c2/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2600:3000::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2600:3c09::2000:5cff:fe99:c802/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2600:3c15::2000:53ff:fe31:8c0b/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2602:fa59:10:a3a::1/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a01:2a8:a0dd:1::1012/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a04:4e40:e000:0:d09:97ab:43b1:a59a/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a04:c300::/29",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a09:bac5:9443:3af::5e:4d/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a09:bac5:9443:3af::5e:88/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a0a:4cc0:2000:8528:c4e8:3ff:febb:3ee9/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2001:ba0:24c:a500::1/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2406:da18:525:0:a23e:e8f6:3ea1:4c2c/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2600:3c15::2000:9cff:fef0:8ad6/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2602:fa59::/40",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a00:6340:2008:ec80:64cc:9116:beb2:1931/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a00:6340:2008:ec80:be24:11ff:fe3b:4058/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a00:6340:2008:ec80:be24:11ff:fe82:2775/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a01:4f8:c014:7f9e::1/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a04:4e40:4400:0:537:cdd2:7250:95db/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a04:4e40:e000::/48",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a05:4140:3::37cd/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a06:a880:5:7fb3::1/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a09:bac1:76c0:460::5e:6a/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a09:bac5::/32",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2a13:adc0::2ed:89ff:fe58:19c5/128",
-    "creation-time": "2026-08-03",
-    "dynamic": "no",
-    "list": "Blocked"
-  },
-  {
-    "address": "2400:8901::2000:beff:fea3:5a44/128",
-    "creation-time": "2026-08-05",
-    "dynamic": "no",
-    "list": "Blocked"
-  }
-]

data/mikrotik/firewall_ipv4.json

@@ -1,135 +0,0 @@
-[
-  {
-    "action": "passthrough",
-    "chain": "forward",
-    "comment": "special"
-  },
-  {
-    "action": "fasttrack-connection",
-    "chain": "forward",
-    "comment": "defconf:",
-    "connection-state": "established,related",
-    "hw-offload": "yes",
-    "log": "no",
-    "log-prefix": ""
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "connection-state": "established,related",
-    "log": "no",
-    "log-prefix": ""
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "defconf:",
-    "connection-state": "invalid",
-    "log": "no",
-    "log-prefix": ""
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "Allow",
-    "connection-nat-state": "dstnat"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "allow",
-    "connection-state": "new",
-    "dst-address": "192.168.88.25",
-    "dst-port": "6667",
-    "log": "no",
-    "log-prefix": "",
-    "protocol": "tcp"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "defconf:",
-    "connection-nat-state": "!dstnat",
-    "connection-state": "new",
-    "in-interface-list": "WAN",
-    "log": "no",
-    "log-prefix": ""
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "log": "no",
-    "log-prefix": "",
-    "protocol": "icmp"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "connection-state": "established",
-    "log": "no",
-    "log-prefix": ""
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "allow",
-    "dst-port": "500",
-    "protocol": "udp"
-  },
-  {
-    "action": "drop",
-    "chain": "input",
-    "comment": "Block",
-    "dst-port": "80,443",
-    "in-interface-list": "WAN",
-    "protocol": "tcp"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "Allow",
-    "dst-port": "80,443",
-    "in-interface-list": "LAN",
-    "protocol": "tcp",
-    "src-address": "192.168.88.0/24"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "Openfactory"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "Cloudflare"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "PaloAlto"
-  },
-  {
-    "action": "drop",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "Blocked"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "ids",
-    "src-address-list": "Blocked"
-  },
-  {
-    "action": "drop",
-    "chain": "input",
-    "comment": "drop",
-    "in-interface": "pppoe-out1",
-    "log": "no",
-    "log-prefix": ""
-  }
-]

data/mikrotik/firewall_ipv6.json

@@ -1,182 +0,0 @@
-[
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "connection-state": "established,related,untracked"
-  },
-  {
-    "action": "drop",
-    "chain": "input",
-    "comment": "defconf:",
-    "connection-state": "invalid"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "protocol": "icmpv6"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "port": "33434-33534",
-    "protocol": "udp"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "dst-port": "546",
-    "protocol": "udp",
-    "src-address": "fe80::/10"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "dst-port": "500,4500",
-    "protocol": "udp"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "protocol": "ipsec-ah"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "protocol": "ipsec-esp"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "defconf:",
-    "ipsec-policy": "in,ipsec"
-  },
-  {
-    "action": "drop",
-    "chain": "input",
-    "comment": "defconf:",
-    "in-interface-list": "!LAN"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "connection-state": "established,related,untracked"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "defconf:",
-    "connection-state": "invalid"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "defconf:",
-    "src-address-list": "bad_ipv6"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "defconf:",
-    "dst-address-list": "bad_ipv6"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "defconf:",
-    "hop-limit": "equal:1",
-    "protocol": "icmpv6"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "protocol": "icmpv6"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "protocol": "139"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "dst-port": "500,4500",
-    "protocol": "udp"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "protocol": "ipsec-ah"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "protocol": "ipsec-esp"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "defconf:",
-    "ipsec-policy": "in,ipsec"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "defconf:",
-    "in-interface-list": "!LAN"
-  },
-  {
-    "action": "accept",
-    "chain": "forward",
-    "comment": "Allow",
-    "protocol": "icmpv6"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "Allow",
-    "protocol": "icmpv6"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "Openfactory"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "Cloudflare"
-  },
-  {
-    "action": "accept",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "PaloAlto"
-  },
-  {
-    "action": "drop",
-    "chain": "input",
-    "comment": "ids",
-    "src-address-list": "Blocked"
-  },
-  {
-    "action": "drop",
-    "chain": "forward",
-    "comment": "ids",
-    "src-address-list": "Blocked"
-  }
-]

data/openfactory/.gitkeep

@@ -1 +0,0 @@
-$ = openfactory

data/openfactory/geofeed.csv

@@ -1,46 +0,0 @@
-45.11.248.0/23,DE,DE-HE,Frankfurt Am Main,
-45.11.250.0/24,NL,NL-NH,Amsterdam,
-45.11.251.0/24,AT,AT-9,Vienna,
-2a0e:8f04::/32,DE,DE-HE,Frankfurt Am Main,
-2a0e:8f05::/32,GB,GB-LND,London,
-2a0e:8f06::/32,NL,NL-NH,Amsterdam,
-2a0e:8f07::/32,AT,AT-9,Vienna,
-5.226.144.0/21,CH,CH-TG,Frauenfeld,
-5.226.144.192/26,CH,CH-ZH,Winterthur,
-5.226.145.0/26,CH,CH-ZH,Wangen-Bruettisellen,
-5.226.145.64/26,LI,LI-07,Schaan,
-5.226.145.128/25,CH,CH-ZH,Kloten,
-5.226.148.0/29,CH,CH-ZH,Zurich,
-5.226.148.8/29,CH,CH-ZH,Kloten,
-5.226.148.16/28,CH,CH-ZH,Zurich,
-5.226.148.32/27,CH,CH-ZH,Kloten,
-5.226.148.64/26,CH,CH-ZH,Zurich,
-5.226.148.96/27,CH,CH-ZH,Zurich,
-5.226.148.128/25,CH,CH-ZH,Zurich,
-5.226.149.0/26,CH,CH-ZH,Zurich,
-5.226.149.64/28,CH,CH-LU,Lutzern,
-5.226.149.128/26,CH,CH-ZH,Glattfelden,
-5.226.149.240/30,CH,CH-SZ,Wollerau,
-5.226.150.0/27,CH,CH-ZH,Glattfelden,
-5.226.150.32/27,CH,CH-BE,Biel,
-5.226.150.152/29,LI,LI-07,Schaan,
-5.226.150.160/28,CH,CH-ZH,Winterthur,
-5.226.150.224/27,CH,CH-BE,Bern,
-5.226.151.0/26,NL,NL-NH,Amsterdam,
-5.226.151.64/27,CH,CH-ZH,Ruemlang,
-185.86.228.0/23,CH,CH-ZH,Zurich,
-195.138.203.0/24,CH,CH-ZH,Zurich,
-2a00:6340::/32,CH,CH-TG,Frauenfeld,
-2a00:6340:2080::/48,NL,NL-NH,Amsterdam,
-2001:67c:2648::/48,CH,CH-ZH,Winterthur,
-
-# Customer geofeed details on request
-# Having issues with your geolocation? send us a ticket!
-
-45.11.248.94/32,DE,DE-NW,Dortmund,
-2a00:6340:2044:200::/56,DE,DE-NW,Dortmund,
-2a00:6340:2044:600::/56,DE,DE-BW,Stuttgart,
-45.11.248.96/32,DE,DE-BW,Stuttgart,
-45.11.248.37/32,DE,DE-BE,Berlin,
-2a00:6340:2044:800::/56,DE,DE-BE,Berlin,
-5.226.146.49/32,CH,CH-ZH,Winterthur,8405,47.472502,8.779930

data/paloalto/.gitkeep

@@ -1 +0,0 @@
-$ = paloalto

data/whois_cache.json

@@ -1,974 +0,0 @@
-{
-  "185.177.72.31": "185.177.72.0/24",
-  "195.178.110.155": "195.178.110.0/24",
-  "66.249.74.165": "66.249.64.0/20",
-  "64.225.75.246": "64.225.64.0/20",
-  "193.19.109.151": "193.19.109.0/24",
-  "92.118.39.126": "92.118.39.0/24",
-  "54.179.129.22": "54.179.128.0/18",
-  "20.220.227.237": "20.192.0.0/10",
-  "199.16.157.180": "199.16.157.0/24",
-  "74.249.245.134": "74.248.0.0/15",
-  "136.107.208.33": "136.107.128.0/17",
-  "104.28.222.131": "104.28.222.0/24",
-  "45.148.10.62": "45.148.10.0/24",
-  "136.70.149.147": "136.70.128.0/17",
-  "158.158.32.229": "158.158.0.0/16",
-  "216.73.216.244": "216.73.216.0/22",
-  "211.175.22.56": "211.175.0.0/16",
-  "8.229.130.50": "8.229.0.0/16",
-  "107.170.9.103": "107.170.0.0/17",
-  "192.34.61.236": "192.34.56.0/21",
-  "194.5.82.61": "194.5.82.0/24",
-  "194.5.82.104": "194.5.82.0/24",
-  "104.28.247.220": "104.28.247.0/24",
-  "64.23.218.208": "64.23.192.0/19",
-  "157.245.204.205": "157.245.192.0/20",
-  "54.169.103.174": "54.169.0.0/17",
-  "20.199.183.73": "20.192.0.0/10",
-  "52.238.192.78": "52.224.0.0/11",
-  "199.16.157.182": "199.16.157.0/24",
-  "20.226.90.242": "20.192.0.0/10",
-  "168.144.106.212": "168.144.96.0/20",
-  "34.175.82.8": "34.175.0.0/17",
-  "20.170.16.216": "20.160.0.0/12",
-  "65.0.122.218": "65.0.0.0/14",
-  "51.116.233.22": "51.116.0.0/16",
-  "31.58.51.184": "31.58.51.0/24",
-  "191.235.102.147": "191.232.0.0/13",
-  "20.9.44.210": "20.0.0.0/11",
-  "123.160.175.254": "123.160.0.0/14",
-  "195.178.110.135": "195.178.110.0/24",
-  "45.61.187.30": "45.61.184.0/22",
-  "8.231.78.19": "8.231.64.0/18",
-  "20.226.114.112": "20.192.0.0/10",
-  "146.190.63.48": "146.190.32.0/19",
-  "216.73.217.54": "216.73.216.0/22",
-  "3.107.107.18": "3.104.0.0/14",
-  "69.164.220.85": "69.164.208.0/20",
-  "191.235.105.17": "191.232.0.0/13",
-  "139.28.219.70": "139.28.219.0/24",
-  "185.177.72.49": "185.177.72.0/24",
-  "20.206.67.15": "20.192.0.0/10",
-  "35.237.178.50": "35.237.0.0/16",
-  "158.158.36.150": "158.158.0.0/16",
-  "136.64.51.224": "136.64.0.0/15",
-  "94.154.43.186": "94.154.43.0/24",
-  "3.101.54.4": "3.101.0.0/16",
-  "23.234.98.171": "23.234.64.0/18",
-  "45.67.85.190": "45.67.85.0/24",
-  "199.16.157.181": "199.16.157.0/24",
-  "93.123.109.165": "93.123.109.0/24",
-  "20.79.29.209": "20.64.0.0/10",
-  "162.141.167.14": "162.141.167.0/24",
-  "163.7.11.71": "163.7.11.0/24",
-  "66.187.6.102": "66.187.6.0/24",
-  "45.153.34.16": "45.153.34.0/24",
-  "74.7.227.43": "74.7.0.0/16",
-  "188.166.245.173": "188.166.240.0/20",
-  "72.146.20.230": "72.144.0.0/14",
-  "216.73.217.145": "216.73.216.0/22",
-  "158.158.76.106": "158.158.0.0/16",
-  "44.248.34.34": "44.224.0.0/11",
-  "4.194.24.143": "4.192.0.0/12",
-  "124.88.113.10": "124.88.0.0/16",
-  "172.213.244.85": "172.208.0.0/13",
-  "170.64.144.116": "170.64.128.0/18",
-  "143.244.57.120": "143.244.56.0/23",
-  "135.119.63.61": "135.119.0.0/16",
-  "45.154.98.52": "45.154.98.0/24",
-  "112.94.252.15": "112.94.128.0/17",
-  "2600:3c09::2000:5cff:fe99:c802": "2600:3c09::/32",
-  "35.243.173.211": "35.243.160.0/20",
-  "206.189.233.36": "206.189.224.0/20",
-  "45.148.10.95": "45.148.10.0/24",
-  "168.62.180.116": "168.62.0.0/15",
-  "23.180.120.134": "23.180.120.0/24",
-  "31.59.129.193": "31.59.129.0/24",
-  "203.159.90.217": "203.159.90.0/24",
-  "194.5.82.153": "194.5.82.0/24",
-  "143.244.57.121": "143.244.56.0/23",
-  "216.73.217.121": "216.73.216.0/22",
-  "91.134.35.95": "91.134.0.0/16",
-  "135.225.93.215": "135.224.0.0/15",
-  "103.221.220.62": "103.221.220.0/22",
-  "20.116.62.111": "20.64.0.0/10",
-  "2.57.122.173": "2.57.122.0/24",
-  "20.203.221.142": "20.192.0.0/10",
-  "2a04:c300:400::99": "2a04:c300::/29",
-  "45.148.10.67": "45.148.10.0/24",
-  "34.187.200.119": "34.187.128.0/17",
-  "4.223.65.51": "4.208.0.0/12",
-  "5.226.149.150": "5.226.144.0/21",
-  "20.220.150.193": "20.192.0.0/10",
-  "201.78.128.89": "201.78.128.0/24",
-  "34.91.243.226": "34.91.240.0/20",
-  "85.204.70.90": "85.204.70.0/24",
-  "20.100.187.246": "20.64.0.0/10",
-  "185.177.72.67": "185.177.72.0/24",
-  "180.95.231.78": "180.95.128.0/17",
-  "34.186.83.134": "34.186.64.0/18",
-  "4.223.77.241": "4.208.0.0/12",
-  "146.190.83.166": "146.190.80.0/20",
-  "3.77.92.222": "3.64.0.0/12",
-  "185.177.72.205": "185.177.72.0/24",
-  "193.47.62.168": "193.47.62.0/24",
-  "20.208.40.114": "20.192.0.0/10",
-  "20.100.175.1": "20.64.0.0/10",
-  "192.133.77.14": "192.133.76.0/23",
-  "38.248.95.248": "38.248.95.0/24",
-  "136.108.11.195": "136.108.0.0/16",
-  "123.178.210.228": "123.178.0.0/15",
-  "194.5.82.59": "194.5.82.0/24",
-  "20.197.192.193": "20.192.0.0/10",
-  "64.227.32.66": "64.227.32.0/20",
-  "132.196.11.13": "132.196.0.0/16",
-  "136.110.80.8": "136.110.64.0/18",
-  "85.204.70.116": "85.204.70.0/24",
-  "81.171.72.93": "81.171.72.0/24",
-  "118.81.207.202": "118.80.0.0/15",
-  "20.226.26.5": "20.192.0.0/10",
-  "20.203.219.190": "20.192.0.0/10",
-  "158.158.32.226": "158.158.0.0/16",
-  "20.215.211.95": "20.192.0.0/10",
-  "2600:3c15::2000:53ff:fe31:8c0b": "2600:3c15::/32",
-  "23.178.112.107": "23.178.112.0/24",
-  "95.85.238.18": "95.85.238.0/24",
-  "35.231.100.82": "35.231.0.0/17",
-  "108.130.204.104": "108.128.0.0/13",
-  "206.81.12.187": "206.81.0.0/20",
-  "206.189.19.19": "206.189.16.0/20",
-  "20.230.87.222": "20.192.0.0/10",
-  "58.212.237.231": "58.208.0.0/12",
-  "185.177.72.9": "185.177.72.0/24",
-  "85.204.70.114": "85.204.70.0/24",
-  "2a09:bac5:9443:3af::5e:88": "2a09:bac5::/32",
-  "94.154.43.188": "94.154.43.0/24",
-  "34.95.239.96": "34.95.224.0/20",
-  "4.204.201.85": "4.192.0.0/12",
-  "2a0a:4cc0:2000:8528:c4e8:3ff:febb:3ee9": "2a0a:4cc0:2000::/48",
-  "220.202.112.100": "220.202.96.0/19",
-  "20.104.96.117": "20.64.0.0/10",
-  "35.192.232.168": "35.192.224.0/20",
-  "34.59.187.134": "34.59.0.0/16",
-  "216.73.216.24": "216.73.216.0/22",
-  "139.59.118.17": "139.59.112.0/20",
-  "171.104.81.195": "171.104.0.0/14",
-  "52.184.96.217": "52.160.0.0/11",
-  "216.73.216.133": "216.73.216.0/22",
-  "199.16.157.183": "199.16.157.0/24",
-  "110.179.80.63": "110.176.0.0/13",
-  "194.5.82.141": "194.5.82.0/24",
-  "216.73.216.25": "216.73.216.0/22",
-  "104.28.247.219": "104.28.247.0/24",
-  "146.70.194.230": "146.70.194.0/24",
-  "136.109.87.37": "136.109.0.0/17",
-  "93.123.109.152": "93.123.109.0/24",
-  "23.161.169.62": "23.161.169.0/24",
-  "2600:3000:2710:200::86": "2600:3000::/32",
-  "64.89.161.85": "64.89.161.0/24",
-  "185.93.89.147": "185.93.89.0/24",
-  "45.154.138.232": "45.154.138.0/24",
-  "13.57.15.41": "13.57.0.0/16",
-  "94.154.43.185": "94.154.43.0/24",
-  "20.113.56.22": "20.64.0.0/10",
-  "209.99.189.160": "209.99.184.0/21",
-  "34.86.126.198": "34.86.112.0/20",
-  "74.7.241.170": "74.7.0.0/16",
-  "136.115.17.125": "136.115.0.0/16",
-  "103.153.183.126": "103.153.182.0/23",
-  "20.104.16.169": "20.64.0.0/10",
-  "23.178.112.213": "23.178.112.0/24",
-  "194.5.82.100": "194.5.82.0/24",
-  "20.206.89.130": "20.192.0.0/10",
-  "69.30.85.194": "69.30.85.0/24",
-  "35.95.56.230": "35.80.0.0/12",
-  "68.183.180.73": "68.183.176.0/20",
-  "213.111.158.220": "213.111.158.0/23",
-  "185.177.72.54": "185.177.72.0/24",
-  "18.227.73.7": "18.224.0.0/14",
-  "73.116.194.16": "73.116.0.0/16",
-  "146.70.194.252": "146.70.194.0/24",
-  "23.178.112.104": "23.178.112.0/24",
-  "35.237.138.104": "35.237.0.0/16",
-  "124.117.194.69": "124.117.0.0/16",
-  "20.9.85.55": "20.0.0.0/11",
-  "51.79.208.246": "51.79.128.0/17",
-  "20.196.209.159": "20.192.0.0/10",
-  "185.93.89.167": "185.93.89.0/24",
-  "45.148.10.21": "45.148.10.0/24",
-  "20.220.15.170": "20.192.0.0/10",
-  "74.248.24.145": "74.248.0.0/15",
-  "209.99.189.152": "209.99.184.0/21",
-  "45.154.98.101": "45.154.98.0/24",
-  "216.73.217.176": "216.73.217.176/32",
-  "45.134.20.116": "45.134.20.116/32",
-  "2a00:6340:2008:ec80:be24:11ff:fe3b:4058": "2a00:6340:2008:ec80:be24:11ff:fe3b:4058/128",
-  "35.199.183.190": "35.199.183.190/32",
-  "91.192.10.86": "91.192.10.0/24",
-  "45.148.10.120": "45.148.10.0/24",
-  "159.89.174.87": "159.89.160.0/20",
-  "23.178.112.219": "23.178.112.0/24",
-  "52.253.111.48": "52.224.0.0/11",
-  "51.103.152.96": "51.103.152.96/32",
-  "193.19.109.122": "193.19.109.0/24",
-  "35.243.223.147": "35.243.208.0/20",
-  "158.101.144.198": "158.101.128.0/19",
-  "149.56.150.98": "149.56.0.0/16",
-  "20.119.205.224": "20.64.0.0/10",
-  "2600:3000:2710:200::83": "2600:3000::/32",
-  "194.233.91.21": "194.233.88.0/22",
-  "185.177.72.17": "185.177.72.0/24",
-  "185.151.21.124": "185.151.21.0/24",
-  "35.196.22.146": "35.196.16.0/20",
-  "4.185.41.66": "4.176.0.0/12",
-  "192.253.248.142": "192.253.248.142/32",
-  "5.253.247.16": "5.253.247.0/24",
-  "2602:fa59:10:a3a::1": "2602:fa59::/40",
-  "216.73.217.13": "216.73.216.0/22",
-  "23.180.120.147": "23.180.120.0/24",
-  "161.118.251.181": "161.118.192.0/18",
-  "51.103.133.211": "51.103.0.0/16",
-  "74.248.33.172": "74.248.33.172/32",
-  "157.143.3.35": "157.143.0.0/17",
-  "20.226.60.151": "20.192.0.0/10",
-  "159.65.144.72": "159.65.144.0/20",
-  "158.158.105.63": "158.158.0.0/16",
-  "51.120.83.160": "51.120.0.0/16",
-  "185.177.72.29": "185.177.72.29/32",
-  "144.76.19.27": "144.76.0.0/16",
-  "45.45.237.214": "45.45.237.0/24",
-  "172.213.8.106": "172.208.0.0/13",
-  "45.139.104.172": "45.139.104.0/24",
-  "23.161.169.127": "23.161.169.0/24",
-  "103.215.75.19": "103.215.75.19/32",
-  "82.102.18.188": "82.102.18.0/24",
-  "36.106.166.223": "36.106.166.0/23",
-  "130.12.180.126": "130.12.180.0/24",
-  "185.177.72.38": "185.177.72.0/24",
-  "82.102.18.190": "82.102.18.190/32",
-  "172.171.198.214": "172.160.0.0/11",
-  "43.228.157.102": "43.228.157.0/24",
-  "173.249.34.197": "173.249.34.0/23",
-  "193.148.57.33": "193.148.57.0/24",
-  "192.133.77.15": "192.133.77.15/32",
-  "195.178.110.48": "195.178.110.0/24",
-  "217.217.252.197": "217.217.252.0/22",
-  "3.237.185.144": "3.224.0.0/12",
-  "216.73.217.173": "216.73.216.0/22",
-  "3.250.52.216": "3.250.0.0/15",
-  "91.239.79.146": "91.239.78.0/23",
-  "146.70.194.220": "146.70.194.0/24",
-  "123.158.61.179": "123.158.0.0/16",
-  "172.213.225.181": "172.208.0.0/13",
-  "20.215.189.213": "20.215.189.213/32",
-  "20.200.216.112": "20.192.0.0/10",
-  "74.7.230.46": "74.7.0.0/16",
-  "185.177.72.52": "185.177.72.0/24",
-  "104.28.215.219": "104.28.215.0/24",
-  "147.93.171.188": "147.93.171.188/32",
-  "91.124.88.36": "91.124.88.0/24",
-  "34.118.184.2": "34.118.128.0/18",
-  "168.144.100.227": "168.144.96.0/20",
-  "45.148.10.247": "45.148.10.0/24",
-  "94.154.43.187": "94.154.43.0/24",
-  "13.221.106.198": "13.221.106.198/32",
-  "104.28.215.220": "104.28.215.0/24",
-  "2a04:4e40:e000:0:d09:97ab:43b1:a59a": "2a04:4e40:e000::/48",
-  "194.5.82.98": "194.5.82.0/24",
-  "54.163.186.39": "54.163.128.0/18",
-  "223.199.187.171": "223.199.0.0/16",
-  "143.244.57.123": "143.244.57.123/32",
-  "167.99.181.249": "167.99.176.0/20",
-  "216.73.216.229": "216.73.216.0/22",
-  "34.47.183.174": "34.47.128.0/17",
-  "2600:3000:2710:200::81": "2600:3000::/32",
-  "43.228.157.191": "43.228.157.0/24",
-  "104.208.94.19": "104.208.94.19/32",
-  "2600:3000:2710:200::84": "2600:3000::/32",
-  "146.70.40.68": "146.70.40.0/24",
-  "2600:3000:1305:4212::84": "2600:3000::/32",
-  "157.245.105.107": "157.245.96.0/20",
-  "68.221.69.72": "68.220.0.0/15",
-  "195.178.110.211": "195.178.110.211/32",
-  "103.168.66.237": "103.168.66.0/23",
-  "162.243.79.174": "162.243.0.0/17",
-  "172.213.144.209": "172.208.0.0/13",
-  "138.197.191.87": "138.197.176.0/20",
-  "54.216.227.171": "54.216.192.0/18",
-  "172.213.216.126": "172.213.216.126/32",
-  "209.99.190.126": "209.99.184.0/21",
-  "134.199.148.147": "134.199.144.0/20",
-  "110.179.81.90": "110.176.0.0/13",
-  "134.209.162.128": "134.209.160.0/20",
-  "195.178.110.223": "195.178.110.223/32",
-  "93.123.109.166": "93.123.109.0/24",
-  "185.177.72.12": "185.177.72.0/24",
-  "165.245.185.158": "165.245.176.0/20",
-  "172.213.27.135": "172.208.0.0/13",
-  "77.90.185.230": "77.90.185.0/24",
-  "20.9.4.9": "20.0.0.0/11",
-  "74.248.139.103": "74.248.0.0/15",
-  "117.14.153.168": "117.8.0.0/13",
-  "45.153.34.43": "45.153.34.0/24",
-  "191.237.255.150": "191.232.0.0/13",
-  "194.5.82.92": "194.5.82.0/24",
-  "118.212.120.78": "118.212.0.0/16",
-  "20.151.205.204": "20.150.0.0/15",
-  "185.177.72.10": "185.177.72.0/24",
-  "35.230.52.27": "35.230.0.0/17",
-  "31.56.58.134": "31.56.58.0/24",
-  "20.230.47.94": "20.192.0.0/10",
-  "44.245.216.107": "44.224.0.0/11",
-  "52.167.144.160": "52.160.0.0/11",
-  "34.11.127.22": "34.11.0.0/17",
-  "85.204.70.96": "85.204.70.0/24",
-  "35.157.251.190": "35.156.0.0/14",
-  "20.100.178.92": "20.64.0.0/10",
-  "4.184.121.217": "4.176.0.0/12",
-  "35.198.180.224": "35.198.176.0/20",
-  "44.251.113.120": "44.224.0.0/11",
-  "91.230.225.189": "91.230.225.0/24",
-  "34.47.194.183": "34.47.128.0/17",
-  "82.102.18.182": "82.102.18.0/24",
-  "172.213.155.173": "172.208.0.0/13",
-  "82.102.18.180": "82.102.18.0/24",
-  "23.178.112.106": "23.178.112.0/24",
-  "5.255.119.161": "5.255.96.0/19",
-  "158.158.96.17": "158.158.0.0/16",
-  "2600:3000:1305:4212::81": "2600:3000::/32",
-  "192.133.77.16": "192.133.76.0/23",
-  "172.232.175.132": "172.232.160.0/19",
-  "193.37.33.117": "193.37.33.0/24",
-  "2a04:c300:400::5d": "2a04:c300::/29",
-  "20.198.120.225": "20.192.0.0/10",
-  "192.253.248.7": "192.253.248.0/24",
-  "2400:8901::2000:62ff:fe9a:e1c2": "2400:8901::/32",
-  "20.215.218.234": "20.192.0.0/10",
-  "3.79.41.12": "3.64.0.0/12",
-  "94.154.43.183": "94.154.43.0/24",
-  "43.98.201.177": "43.98.201.0/24",
-  "20.104.50.116": "20.64.0.0/10",
-  "52.165.196.84": "52.160.0.0/11",
-  "185.177.72.53": "185.177.72.0/24",
-  "91.92.47.101": "91.92.47.0/24",
-  "216.73.216.140": "216.73.216.0/22",
-  "35.187.9.30": "35.187.0.0/20",
-  "191.237.250.106": "191.232.0.0/13",
-  "104.28.240.187": "104.28.240.0/24",
-  "66.187.5.228": "66.187.5.0/24",
-  "2a00:6340:2008:ec80:be24:11ff:fe82:2775": "2a00:6340::/32",
-  "113.57.186.101": "113.56.0.0/15",
-  "34.244.177.56": "34.240.0.0/13",
-  "2600:3000:1305:4212::85": "2600:3000::/32",
-  "20.9.15.100": "20.0.0.0/11",
-  "80.91.86.169": "80.91.86.0/24",
-  "20.91.140.156": "20.64.0.0/10",
-  "139.59.231.238": "139.59.224.0/20",
-  "23.102.123.231": "23.102.0.0/16",
-  "20.104.22.47": "20.64.0.0/10",
-  "216.73.216.209": "216.73.216.0/22",
-  "4.223.71.149": "4.208.0.0/12",
-  "192.133.77.18": "192.133.76.0/23",
-  "93.123.109.163": "93.123.109.0/24",
-  "4.225.166.222": "4.224.0.0/12",
-  "193.148.56.148": "193.148.56.0/24",
-  "23.178.112.212": "23.178.112.0/24",
-  "194.5.82.123": "194.5.82.0/24",
-  "34.171.75.208": "34.171.0.0/17",
-  "185.177.72.22": "185.177.72.0/24",
-  "20.197.178.120": "20.192.0.0/10",
-  "158.158.45.224": "158.158.0.0/16",
-  "85.204.70.100": "85.204.70.0/24",
-  "2600:3000:2710:200::82": "2600:3000::/32",
-  "34.65.14.246": "34.65.0.0/20",
-  "185.177.72.5": "185.177.72.0/24",
-  "20.195.181.10": "20.192.0.0/10",
-  "110.179.80.213": "110.176.0.0/13",
-  "34.69.107.154": "34.69.96.0/20",
-  "195.178.110.199": "195.178.110.0/24",
-  "85.204.70.118": "85.204.70.0/24",
-  "20.226.43.166": "20.192.0.0/10",
-  "130.12.182.254": "130.12.182.0/24",
-  "20.226.5.174": "20.192.0.0/10",
-  "20.91.208.34": "20.64.0.0/10",
-  "68.221.130.241": "68.220.0.0/15",
-  "85.204.70.112": "85.204.70.0/24",
-  "165.245.190.205": "165.245.176.0/20",
-  "51.120.79.193": "51.120.0.0/16",
-  "216.73.216.41": "216.73.216.0/22",
-  "216.73.216.66": "216.73.216.0/22",
-  "34.105.30.251": "34.105.16.0/20",
-  "45.148.10.244": "45.148.10.0/24",
-  "2600:3000:2710:200::85": "2600:3000::/32",
-  "20.215.216.94": "20.192.0.0/10",
-  "18.142.90.109": "18.142.0.0/15",
-  "34.242.226.103": "34.240.0.0/13",
-  "158.158.41.79": "158.158.0.0/16",
-  "195.178.110.31": "195.178.110.0/24",
-  "20.203.209.227": "20.192.0.0/10",
-  "209.99.189.190": "209.99.184.0/21",
-  "148.251.182.79": "148.251.0.0/16",
-  "16.146.99.254": "16.146.0.0/17",
-  "167.99.210.137": "167.99.208.0/20",
-  "45.134.20.128": "45.134.20.0/24",
-  "69.164.220.232": "69.164.208.0/20",
-  "34.11.108.163": "34.11.0.0/17",
-  "104.248.151.168": "104.248.144.0/20",
-  "128.199.182.55": "128.199.128.0/18",
-  "23.178.112.103": "23.178.112.0/24",
-  "91.92.41.115": "91.92.41.0/24",
-  "20.151.225.23": "20.150.0.0/15",
-  "91.92.241.196": "91.92.241.0/24",
-  "185.177.72.81": "185.177.72.0/24",
-  "20.100.176.141": "20.64.0.0/10",
-  "129.212.237.196": "129.212.224.0/20",
-  "2a09:bac5:9443:3af::5e:4d": "2a09:bac5::/32",
-  "20.100.169.152": "20.64.0.0/10",
-  "20.151.10.161": "20.150.0.0/15",
-  "34.73.128.17": "34.73.128.0/20",
-  "143.244.57.84": "143.244.56.0/23",
-  "20.198.114.210": "20.192.0.0/10",
-  "45.88.138.44": "45.88.138.0/24",
-  "167.99.182.39": "167.99.176.0/20",
-  "185.177.72.58": "185.177.72.0/24",
-  "54.82.69.43": "54.82.0.0/17",
-  "172.202.29.124": "172.200.0.0/13",
-  "64.89.161.82": "64.89.161.0/24",
-  "20.197.195.24": "20.192.0.0/10",
-  "20.251.48.208": "20.192.0.0/10",
-  "150.255.94.250": "150.255.0.0/16",
-  "82.165.82.154": "82.165.0.0/16",
-  "5.34.213.142": "5.34.208.0/21",
-  "45.154.138.181": "45.154.138.0/24",
-  "200.162.152.170": "200.162.144.0/20",
-  "4.193.178.28": "4.192.0.0/12",
-  "2a01:2a8:a0dd:1::1012": "2a01:2a8::/29",
-  "216.73.216.98": "216.73.216.0/22",
-  "20.48.234.177": "20.48.0.0/12",
-  "134.122.94.138": "134.122.80.0/20",
-  "35.231.41.225": "35.231.0.0/17",
-  "2.58.56.101": "2.58.56.0/24",
-  "146.190.63.248": "146.190.32.0/19",
-  "34.31.88.250": "34.31.0.0/16",
-  "20.220.210.52": "20.192.0.0/10",
-  "52.139.36.144": "52.136.0.0/13",
-  "194.5.82.87": "194.5.82.0/24",
-  "35.199.111.12": "35.199.96.0/20",
-  "85.204.70.106": "85.204.70.0/24",
-  "20.203.133.142": "20.192.0.0/10",
-  "91.124.88.29": "91.124.88.0/24",
-  "216.73.216.10": "216.73.216.0/22",
-  "198.98.52.229": "198.98.48.0/20",
-  "93.123.109.101": "93.123.109.0/24",
-  "34.139.141.56": "34.139.128.0/20",
-  "143.244.168.161": "143.244.160.0/20",
-  "216.73.216.230": "216.73.216.0/22",
-  "194.5.82.89": "194.5.82.0/24",
-  "195.178.110.104": "195.178.110.0/24",
-  "34.86.217.208": "34.86.208.0/20",
-  "20.214.106.237": "20.192.0.0/10",
-  "54.206.147.115": "54.206.128.0/18",
-  "91.224.92.190": "91.224.92.0/24",
-  "20.48.236.161": "20.48.0.0/12",
-  "34.181.134.189": "34.181.128.0/17",
-  "157.245.197.36": "157.245.192.0/20",
-  "45.61.150.67": "45.61.148.0/22",
-  "143.244.57.92": "143.244.56.0/23",
-  "94.154.43.184": "94.154.43.0/24",
-  "20.203.148.31": "20.192.0.0/10",
-  "34.65.18.71": "34.65.16.0/20",
-  "192.133.77.17": "192.133.76.0/23",
-  "84.17.60.251": "84.17.60.0/24",
-  "138.197.104.211": "138.197.96.0/20",
-  "206.81.24.227": "206.81.16.0/20",
-  "20.46.121.148": "20.40.0.0/13",
-  "36.106.166.92": "36.106.166.0/23",
-  "20.63.63.128": "20.48.0.0/12",
-  "20.151.200.100": "20.150.0.0/15",
-  "216.73.217.11": "216.73.216.0/22",
-  "130.12.180.39": "130.12.180.0/24",
-  "146.70.194.254": "146.70.194.0/24",
-  "91.148.244.131": "91.148.244.0/22",
-  "194.5.82.63": "194.5.82.0/24",
-  "20.205.229.184": "20.192.0.0/10",
-  "2600:3000:1305:4212::83": "2600:3000::/32",
-  "206.189.95.232": "206.189.80.0/20",
-  "130.131.231.171": "130.131.0.0/16",
-  "185.177.72.24": "185.177.72.0/24",
-  "20.220.212.5": "20.192.0.0/10",
-  "146.70.194.228": "146.70.194.0/24",
-  "20.203.135.57": "20.192.0.0/10",
-  "20.206.105.145": "20.192.0.0/10",
-  "2a04:c300:400::58": "2a04:c300::/29",
-  "20.100.203.84": "20.64.0.0/10",
-  "91.224.92.159": "91.224.92.0/24",
-  "104.196.62.35": "104.196.62.35/32",
-  "153.75.89.249": "153.75.89.0/24",
-  "104.168.93.75": "104.168.0.0/17",
-  "185.177.72.13": "185.177.72.0/24",
-  "91.92.42.86": "91.92.42.0/24",
-  "45.153.34.182": "45.153.34.182/32",
-  "185.177.72.100": "185.177.72.0/24",
-  "178.16.54.137": "178.16.54.0/24",
-  "103.215.75.66": "103.215.74.0/23",
-  "81.171.72.135": "81.171.72.135/32",
-  "20.251.58.190": "20.192.0.0/10",
-  "195.178.110.159": "195.178.110.0/24",
-  "23.178.112.211": "23.178.112.0/24",
-  "185.177.72.56": "185.177.72.0/24",
-  "143.110.178.165": "143.110.178.165/32",
-  "20.127.18.0": "20.64.0.0/10",
-  "185.177.72.69": "185.177.72.0/24",
-  "185.177.72.30": "185.177.72.0/24",
-  "147.182.149.75": "147.182.149.75/32",
-  "94.154.43.179": "94.154.43.0/24",
-  "194.5.82.95": "194.5.82.0/24",
-  "172.213.232.128": "172.208.0.0/13",
-  "118.81.196.142": "118.80.0.0/15",
-  "93.123.109.10": "93.123.109.0/24",
-  "66.228.38.221": "66.228.32.0/20",
-  "91.148.245.81": "91.148.244.0/22",
-  "74.248.35.25": "74.248.0.0/15",
-  "45.45.237.69": "45.45.237.0/24",
-  "34.17.159.73": "34.17.0.0/16",
-  "45.148.10.194": "45.148.10.0/24",
-  "8.229.1.130": "8.229.0.0/16",
-  "104.199.182.104": "104.199.176.0/20",
-  "2001:4ca0:108:42::24": "2001:4ca0::/32",
-  "111.113.89.107": "111.112.0.0/15",
-  "4.184.185.91": "4.176.0.0/12",
-  "34.146.12.12": "34.146.0.0/20",
-  "34.131.251.163": "34.131.240.0/20",
-  "45.138.16.114": "45.138.16.0/24",
-  "182.119.230.145": "182.112.0.0/12",
-  "51.120.76.82": "51.120.0.0/16",
-  "34.40.240.124": "34.40.128.0/17",
-  "111.113.88.210": "111.112.0.0/15",
-  "34.42.48.100": "34.42.0.0/16",
-  "20.220.203.117": "20.192.0.0/10",
-  "45.148.10.60": "45.148.10.0/24",
-  "8.234.157.73": "8.234.128.0/17",
-  "74.208.40.151": "74.208.0.0/16",
-  "165.22.235.3": "165.22.224.0/20",
-  "20.240.137.17": "20.192.0.0/10",
-  "144.172.97.164": "144.172.96.0/23",
-  "142.111.152.16": "142.111.152.0/24",
-  "216.73.163.148": "216.73.163.0/24",
-  "4.223.226.192": "4.208.0.0/12",
-  "93.152.221.13": "93.152.221.0/24",
-  "20.63.10.215": "20.48.0.0/12",
-  "136.116.215.248": "136.116.0.0/16",
-  "103.76.88.36": "103.76.88.0/24",
-  "213.35.113.47": "213.35.96.0/19",
-  "160.179.166.222": "160.176.0.0/14",
-  "216.81.248.20": "216.81.248.0/22",
-  "104.28.254.131": "104.28.254.0/24",
-  "3.27.206.50": "3.24.0.0/14",
-  "91.193.232.61": "91.193.232.0/24",
-  "136.66.128.85": "136.66.0.0/15",
-  "5.161.225.116": "5.161.0.0/16",
-  "34.187.20.70": "34.187.0.0/17",
-  "185.223.152.40": "185.223.152.0/24",
-  "20.172.218.20": "20.160.0.0/12",
-  "31.56.58.124": "31.56.58.0/24",
-  "173.239.240.222": "173.239.240.0/24",
-  "2001:ba0:24c:a500::1": "2001:ba0::/32",
-  "185.223.152.126": "185.223.152.0/24",
-  "62.93.179.146": "62.93.179.0/24",
-  "104.248.5.228": "104.248.0.0/20",
-  "35.182.17.8": "35.182.0.0/15",
-  "20.46.252.52": "20.40.0.0/13",
-  "34.116.255.65": "34.116.240.0/20",
-  "172.213.208.20": "172.208.0.0/13",
-  "163.7.13.205": "163.7.13.0/24",
-  "46.232.235.6": "46.232.235.0/24",
-  "4.231.36.50": "4.224.0.0/12",
-  "45.130.203.154": "45.130.202.0/23",
-  "20.100.173.28": "20.64.0.0/10",
-  "176.227.240.61": "176.227.240.0/24",
-  "2a04:c300::3ce": "2a04:c300::/29",
-  "185.251.19.30": "185.251.19.0/24",
-  "92.118.39.29": "92.118.39.0/24",
-  "185.192.71.16": "185.192.71.0/24",
-  "138.197.34.12": "138.197.32.0/20",
-  "34.90.88.37": "34.90.80.0/20",
-  "165.22.24.78": "165.22.16.0/20",
-  "144.172.101.168": "144.172.101.0/24",
-  "216.24.212.243": "216.24.212.0/24",
-  "4.236.158.50": "4.224.0.0/12",
-  "168.144.174.42": "168.144.160.0/20",
-  "2406:da18:525:0:a23e:e8f6:3ea1:4c2c": "2406:da18:400::/38",
-  "52.159.244.176": "52.152.0.0/13",
-  "34.24.155.203": "34.24.0.0/16",
-  "37.140.223.58": "37.140.223.0/24",
-  "194.156.79.68": "194.156.79.0/24",
-  "46.23.108.234": "46.23.108.0/24",
-  "74.248.96.101": "74.248.0.0/15",
-  "31.58.51.130": "31.58.51.0/24",
-  "34.94.172.72": "34.94.160.0/20",
-  "200.162.145.220": "200.162.144.0/20",
-  "13.126.86.18": "13.126.0.0/15",
-  "143.198.25.99": "143.198.16.0/20",
-  "104.28.254.134": "104.28.254.0/24",
-  "167.86.82.167": "167.86.82.0/23",
-  "91.92.243.245": "91.92.243.0/24",
-  "106.63.26.8": "106.63.0.0/17",
-  "2a09:bac1:76c0:460::5e:6a": "2a09:bac1::/32",
-  "136.66.153.34": "136.66.0.0/15",
-  "20.52.125.110": "20.48.0.0/12",
-  "15.206.146.240": "15.206.0.0/15",
-  "194.32.120.151": "194.32.120.0/24",
-  "135.232.177.160": "135.232.0.0/14",
-  "20.171.55.167": "20.160.0.0/12",
-  "64.225.63.129": "64.225.48.0/20",
-  "172.202.95.21": "172.200.0.0/13",
-  "136.144.43.127": "136.144.43.0/24",
-  "24.144.85.55": "24.144.80.0/20",
-  "128.199.182.152": "128.199.128.0/18",
-  "152.42.137.175": "152.42.128.0/20",
-  "87.121.79.42": "87.121.79.0/24",
-  "82.39.206.154": "82.39.206.0/24",
-  "167.71.39.61": "167.71.32.0/20",
-  "193.19.109.68": "193.19.109.0/24",
-  "195.178.110.18": "195.178.110.0/24",
-  "193.36.225.180": "193.36.225.0/24",
-  "136.144.33.102": "136.144.33.0/24",
-  "167.71.0.47": "167.71.0.0/20",
-  "104.28.240.188": "104.28.240.0/24",
-  "45.146.54.80": "45.146.54.0/24",
-  "193.36.225.133": "193.36.225.0/24",
-  "103.177.150.66": "103.177.150.0/24",
-  "172.86.122.106": "172.86.120.0/22",
-  "37.140.223.6": "37.140.223.0/24",
-  "216.73.163.119": "216.73.163.0/24",
-  "179.43.134.114": "179.43.134.0/24",
-  "130.12.182.12": "130.12.182.0/24",
-  "159.89.46.165": "159.89.32.0/20",
-  "62.60.130.227": "62.60.130.0/24",
-  "207.241.173.229": "207.241.172.0/23",
-  "142.93.62.168": "142.93.48.0/20",
-  "91.230.225.192": "91.230.225.0/24",
-  "54.210.219.163": "54.210.0.0/16",
-  "45.131.195.127": "45.131.195.0/24",
-  "20.199.181.219": "20.192.0.0/10",
-  "193.37.33.55": "193.37.33.0/24",
-  "216.73.163.138": "216.73.163.0/24",
-  "188.213.202.116": "188.213.202.0/24",
-  "85.204.70.98": "85.204.70.0/24",
-  "193.36.225.117": "193.36.225.0/24",
-  "188.213.202.121": "188.213.202.0/24",
-  "45.131.195.175": "45.131.195.0/24",
-  "34.75.4.161": "34.75.0.0/20",
-  "20.125.176.178": "20.64.0.0/10",
-  "34.158.83.192": "34.158.64.0/18",
-  "45.130.203.177": "45.130.202.0/23",
-  "216.24.212.233": "216.24.212.0/24",
-  "168.144.167.238": "168.144.160.0/20",
-  "216.73.163.132": "216.73.163.0/24",
-  "191.232.199.39": "191.232.0.0/13",
-  "20.203.200.218": "20.192.0.0/10",
-  "20.12.214.177": "20.0.0.0/11",
-  "98.159.37.147": "98.159.37.0/24",
-  "34.21.144.169": "34.21.128.0/17",
-  "45.146.54.137": "45.146.54.0/24",
-  "193.37.32.159": "193.37.32.0/24",
-  "172.98.33.251": "172.98.33.0/24",
-  "207.154.197.113": "207.154.192.0/20",
-  "104.251.181.106": "104.251.180.0/22",
-  "34.26.191.118": "34.26.0.0/16",
-  "193.37.33.251": "193.37.33.0/24",
-  "135.119.74.228": "135.119.0.0/16",
-  "20.226.83.28": "20.192.0.0/10",
-  "2a04:4e40:e000:0:566:f8d7:adbe:f536": "2a04:4e40:e000::/48",
-  "168.144.83.150": "168.144.80.0/20",
-  "68.221.137.211": "68.220.0.0/15",
-  "167.172.192.26": "167.172.192.0/20",
-  "143.110.217.244": "143.110.208.0/20",
-  "88.184.210.4": "88.184.192.0/19",
-  "167.172.53.28": "167.172.48.0/20",
-  "20.55.86.49": "20.48.0.0/12",
-  "35.199.115.51": "35.199.112.0/20",
-  "34.154.122.183": "34.154.0.0/17",
-  "85.204.70.92": "85.204.70.0/24",
-  "2a13:adc0::2ed:89ff:fe58:19c5": "2a13:adc0::/48",
-  "157.245.113.227": "157.245.112.0/20",
-  "45.131.195.153": "45.131.195.0/24",
-  "34.82.10.34": "34.82.0.0/20",
-  "47.85.160.47": "47.85.160.0/24",
-  "85.11.167.225": "85.11.167.0/24",
-  "4.246.70.112": "4.240.0.0/12",
-  "143.20.64.236": "143.20.64.0/24",
-  "185.192.71.76": "185.192.71.0/24",
-  "134.209.185.72": "134.209.176.0/20",
-  "144.126.207.57": "144.126.192.0/20",
-  "172.202.37.160": "172.200.0.0/13",
-  "118.194.253.208": "118.194.253.0/24",
-  "144.172.93.117": "144.172.92.0/22",
-  "4.225.203.146": "4.224.0.0/12",
-  "209.126.2.173": "209.126.0.0/20",
-  "35.204.88.252": "35.204.80.0/20",
-  "2600:3c15::2000:9cff:fef0:8ad6": "2600:3c15::/32",
-  "2a06:a880:5:7fb3::1": "2a06:a880:5::/48",
-  "94.154.43.177": "94.154.43.0/24",
-  "176.227.240.17": "176.227.240.0/24",
-  "91.230.225.186": "91.230.225.0/24",
-  "8.235.114.162": "8.235.0.0/17",
-  "159.223.132.86": "159.223.128.0/20",
-  "91.92.47.153": "91.92.47.0/24",
-  "2602:fa59:10:e70::1": "2602:fa59::/40",
-  "188.213.202.84": "188.213.202.0/24",
-  "45.146.55.48": "45.146.55.0/24",
-  "45.131.193.139": "45.131.193.0/24",
-  "34.55.200.45": "34.55.0.0/16",
-  "20.215.191.139": "20.192.0.0/10",
-  "8.234.207.247": "8.234.128.0/17",
-  "9.205.16.190": "9.205.0.0/16",
-  "35.204.141.252": "35.204.128.0/20",
-  "136.144.35.201": "136.144.35.0/24",
-  "2a04:4e40:e000:0:bfc:4b0:174b:d52d": "2a04:4e40:e000::/48",
-  "208.84.102.195": "208.84.102.0/24",
-  "160.179.166.124": "160.176.0.0/14",
-  "74.248.20.32": "74.248.0.0/15",
-  "93.123.109.167": "93.123.109.0/24",
-  "52.165.250.117": "52.160.0.0/11",
-  "5.161.62.209": "5.161.0.0/18",
-  "2602:fa59:10:4c9::1": "2602:fa59::/40",
-  "45.154.98.134": "45.154.98.0/24",
-  "172.245.102.64": "172.245.102.0/24",
-  "2a04:4e40:4400:0:537:cdd2:7250:95db": "2a04:4e40:4400::/48",
-  "216.24.210.46": "216.24.210.0/24",
-  "143.198.6.225": "143.198.0.0/20",
-  "91.230.225.2": "91.230.225.0/24",
-  "142.93.155.36": "142.93.144.0/20",
-  "20.220.225.223": "20.192.0.0/10",
-  "68.221.186.136": "68.220.0.0/15",
-  "172.170.254.20": "172.160.0.0/11",
-  "13.83.161.80": "13.64.0.0/11",
-  "103.76.88.37": "103.76.88.0/24",
-  "20.100.186.69": "20.64.0.0/10",
-  "185.192.71.247": "185.192.71.0/24",
-  "52.242.193.144": "52.224.0.0/11",
-  "92.119.121.196": "92.119.120.0/22",
-  "185.223.152.60": "185.223.152.0/24",
-  "20.184.150.240": "20.184.0.0/13",
-  "52.225.25.48": "52.224.0.0/11",
-  "94.154.43.178": "94.154.43.0/24",
-  "216.24.212.14": "216.24.212.0/24",
-  "2a04:4e40:e000:0:33c:a689:be95:5a4": "2a04:4e40:e000::/48",
-  "35.184.157.121": "35.184.144.0/20",
-  "185.192.71.5": "185.192.71.0/24",
-  "104.28.240.189": "104.28.240.0/24",
-  "147.93.169.103": "147.93.168.0/22",
-  "164.90.169.163": "164.90.160.0/20",
-  "46.232.235.5": "46.232.235.0/24",
-  "155.2.215.98": "155.2.215.0/24",
-  "134.209.25.199": "134.209.16.0/20",
-  "185.238.231.167": "185.238.231.0/24",
-  "20.199.136.34": "20.192.0.0/10",
-  "216.24.212.45": "216.24.212.0/24",
-  "147.93.168.91": "147.93.168.0/22",
-  "185.223.152.103": "185.223.152.0/24",
-  "159.89.117.109": "159.89.112.0/20",
-  "140.248.75.106": "140.248.75.0/24",
-  "193.37.32.157": "193.37.32.0/24",
-  "128.90.128.13": "128.90.128.0/24",
-  "185.192.71.3": "185.192.71.0/24",
-  "106.63.26.7": "106.63.0.0/17",
-  "158.158.41.78": "158.158.0.0/16",
-  "185.192.71.13": "185.192.71.0/24",
-  "85.204.70.102": "85.204.70.0/24",
-  "152.53.242.93": "152.53.240.0/22",
-  "144.172.100.174": "144.172.100.0/24",
-  "178.62.196.238": "178.62.192.0/18",
-  "159.203.13.62": "159.203.0.0/20",
-  "31.184.236.8": "31.184.236.0/24",
-  "128.90.141.10": "128.90.141.0/24",
-  "45.146.55.11": "45.146.55.0/24",
-  "4.232.188.49": "4.224.0.0/12",
-  "45.148.10.119": "45.148.10.0/24",
-  "172.204.17.66": "172.200.0.0/13",
-  "34.7.160.235": "34.7.128.0/17",
-  "35.201.159.247": "35.201.144.0/20",
-  "34.148.8.63": "34.148.0.0/20",
-  "4.223.140.16": "4.208.0.0/12",
-  "165.22.147.152": "165.22.144.0/20",
-  "34.90.87.32": "34.90.80.0/20",
-  "106.63.26.22": "106.63.0.0/17",
-  "138.124.103.234": "138.124.103.0/24",
-  "34.64.173.205": "34.64.160.0/20",
-  "93.123.109.178": "93.123.109.0/24",
-  "91.193.232.155": "91.193.232.0/24",
-  "209.87.169.142": "209.87.169.0/24",
-  "2a04:4e40:e000:0:954:9789:20b1:d5fc": "2a04:4e40:e000::/48",
-  "34.187.82.183": "34.187.0.0/17",
-  "193.37.32.149": "193.37.32.0/24",
-  "20.52.54.143": "20.48.0.0/12",
-  "106.63.26.21": "106.63.0.0/17",
-  "2a05:4140:3::37cd": "2a05:4140:3::/48",
-  "193.36.224.57": "193.36.224.0/24",
-  "2a09:bac5:55fb:18c8::278:a7": "2a09:bac5::/32",
-  "195.178.110.72": "195.178.110.0/24",
-  "200.162.145.50": "200.162.144.0/20",
-  "31.56.58.122": "31.56.58.0/24",
-  "2a01:4f8:c014:7f9e::1": "2a01:4f8::/32",
-  "216.24.212.43": "216.24.212.0/24",
-  "195.178.110.100": "195.178.110.0/24",
-  "102.37.110.74": "102.37.0.0/17",
-  "20.106.182.98": "20.64.0.0/10",
-  "172.202.44.182": "172.200.0.0/13",
-  "216.26.233.39": "216.26.233.0/24",
-  "52.242.217.80": "52.224.0.0/11",
-  "167.172.31.131": "167.172.16.0/20",
-  "45.8.196.200": "45.8.196.0/24",
-  "195.178.110.241": "195.178.110.0/24",
-  "41.143.154.1": "41.140.0.0/14",
-  "23.94.155.57": "23.94.155.0/24",
-  "159.89.123.252": "159.89.112.0/20",
-  "104.238.222.26": "104.238.222.0/24",
-  "23.95.226.104": "23.94.0.0/15",
-  "159.89.97.19": "159.89.96.0/20",
-  "35.255.40.65": "35.255.0.0/16",
-  "34.145.62.28": "34.145.48.0/20",
-  "185.192.71.2": "185.192.71.0/24",
-  "193.149.176.55": "193.149.176.0/24",
-  "34.7.193.250": "34.7.128.0/17",
-  "152.39.198.115": "152.39.128.0/17",
-  "18.236.136.230": "18.236.0.0/15",
-  "188.213.202.113": "188.213.202.0/24",
-  "136.144.19.51": "136.144.19.0/24",
-  "68.183.9.16": "68.183.0.0/20",
-  "62.60.130.128": "62.60.130.0/24",
-  "185.192.71.6": "185.192.71.0/24",
-  "18.191.56.164": "18.191.0.0/16",
-  "104.234.19.67": "104.234.19.0/24",
-  "172.196.32.64": "172.192.0.0/13",
-  "185.213.175.168": "185.213.175.0/24",
-  "46.232.235.3": "46.232.235.0/24",
-  "94.154.43.229": "94.154.43.0/24",
-  "34.18.38.137": "34.18.0.0/16",
-  "132.196.83.33": "132.196.0.0/16",
-  "91.193.232.168": "91.193.232.0/24",
-  "31.56.58.129": "31.56.58.0/24",
-  "141.98.11.171": "141.98.11.0/24",
-  "216.73.163.122": "216.73.163.0/24",
-  "142.111.152.57": "142.111.152.0/24",
-  "98.159.36.147": "98.159.36.0/24",
-  "2a00:6340:2008:ec80:64cc:9116:beb2:1931": "2a00:6340::/32",
-  "172.212.171.144": "172.208.0.0/13",
-  "46.232.235.4": "46.232.235.0/24",
-  "34.65.138.195": "34.65.128.0/20",
-  "159.89.12.166": "159.89.0.0/20",
-  "193.143.1.112": "193.143.1.0/24",
-  "145.132.103.19": "145.132.0.0/15",
-  "209.99.190.99": "209.99.184.0/21",
-  "20.104.18.253": "20.64.0.0/10",
-  "178.128.176.132": "178.128.176.0/20",
-  "81.171.74.60": "81.171.74.0/24",
-  "72.146.33.199": "72.144.0.0/14",
-  "147.93.169.104": "147.93.168.0/22",
-  "34.39.210.94": "34.39.128.0/17",
-  "20.215.185.25": "20.192.0.0/10",
-  "23.180.120.146": "23.180.120.0/24",
-  "167.99.204.6": "167.99.192.0/20",
-  "172.213.153.79": "172.208.0.0/13",
-  "20.203.243.122": "20.192.0.0/10",
-  "194.26.192.156": "194.26.192.0/24",
-  "74.248.16.78": "74.248.0.0/15",
-  "34.141.160.140": "34.141.160.0/20",
-  "34.83.69.198": "34.83.64.0/20",
-  "2600:3000:1305:4212::82": "2600:3000::/32",
-  "64.226.123.37": "64.226.112.0/20",
-  "172.213.218.74": "172.208.0.0/13",
-  "159.65.10.116": "159.65.0.0/20",
-  "162.141.167.71": "162.141.167.0/24",
-  "20.91.203.45": "20.64.0.0/10",
-  "80.94.92.167": "80.94.92.0/24",
-  "194.102.104.29": "194.102.104.0/24",
-  "143.244.162.58": "143.244.160.0/20",
-  "4.223.73.90": "4.208.0.0/12",
-  "136.107.217.8": "136.107.128.0/17",
-  "20.203.179.12": "20.192.0.0/10",
-  "4.184.238.149": "4.176.0.0/12",
-  "20.52.217.208": "20.48.0.0/12",
-  "20.218.243.163": "20.192.0.0/10",
-  "34.125.219.56": "34.125.208.0/20",
-  "163.7.13.115": "163.7.13.0/24",
-  "34.26.219.248": "34.26.0.0/16",
-  "2.57.122.13": "2.57.122.0/24",
-  "34.95.238.121": "34.95.224.0/20",
-  "20.208.41.112": "20.192.0.0/10",
-  "182.88.226.154": "182.88.0.0/14",
-  "2602:fa59:10:712::1": "2602:fa59::/40",
-  "34.84.59.119": "34.84.48.0/20",
-  "2a04:c300:400::6f": "2a04:c300::/29",
-  "45.146.55.165": "45.146.55.0/24",
-  "85.204.70.94": "85.204.70.0/24",
-  "157.245.63.62": "157.245.48.0/20",
-  "23.178.112.210": "23.178.112.0/24",
-  "221.13.93.71": "221.13.64.0/19",
-  "163.7.13.111": "163.7.13.0/24",
-  "222.176.200.251": "222.176.0.0/13",
-  "123.163.114.150": "123.160.0.0/14",
-  "136.64.132.60": "136.64.0.0/15",
-  "20.215.218.105": "20.192.0.0/10",
-  "20.203.208.191": "20.192.0.0/10",
-  "2400:8901::2000:beff:fea3:5a44": "2400:8901::/32",
-  "163.7.12.95": "163.7.12.0/24",
-  "209.87.169.76": "209.87.169.0/24",
-  "54.251.19.144": "54.251.16.0/21",
-  "34.125.42.15": "34.125.32.0/20",
-  "92.165.170.72": "92.165.128.0/18",
-  "35.187.221.219": "35.187.208.0/20",
-  "151.240.45.69": "151.240.45.0/24",
-  "163.7.12.6": "163.7.12.0/24",
-  "158.51.126.144": "158.51.126.0/24",
-  "163.7.12.80": "163.7.12.0/24",
-  "20.218.73.95": "20.192.0.0/10",
-  "159.223.79.25": "159.223.64.0/20",
-  "172.86.119.249": "172.86.118.0/23",
-  "35.231.118.233": "35.231.0.0/17",
-  "35.233.191.179": "35.233.128.0/17",
-  "34.74.203.130": "34.74.192.0/20",
-  "170.64.156.114": "170.64.128.0/18",
-  "172.213.2.180": "172.208.0.0/13",
-  "143.244.186.167": "143.244.176.0/20",
-  "35.185.21.88": "35.185.16.0/20",
-  "20.91.239.158": "20.64.0.0/10",
-  "52.64.227.143": "52.64.128.0/17",
-  "34.57.157.110": "34.57.0.0/16",
-  "20.170.17.213": "20.160.0.0/12",
-  "209.97.166.251": "209.97.160.0/20",
-  "173.239.196.33": "173.239.196.0/24",
-  "173.239.196.35": "173.239.196.0/24",
-  "91.92.47.81": "91.92.47.0/24",
-  "34.148.98.62": "34.148.96.0/20",
-  "20.215.186.32": "20.192.0.0/10",
-  "136.108.15.119": "136.108.0.0/16",
-  "34.62.38.199": "34.62.0.0/17",
-  "74.248.33.65": "74.248.0.0/15",
-  "209.87.169.125": "209.87.169.0/24",
-  "34.23.104.90": "34.23.0.0/16",
-  "172.213.208.13": "172.208.0.0/13",
-  "91.192.10.243": "91.192.10.0/24",
-  "91.193.232.249": "91.193.232.0/24",
-  "195.178.110.242": "195.178.110.0/24",
-  "20.251.48.84": "20.192.0.0/10",
-  "23.178.112.108": "23.178.112.0/24",
-  "23.178.112.105": "23.178.112.0/24",
-  "66.249.79.68": "66.249.64.0/20",
-  "66.249.74.164": "66.249.64.0/20",
-  "141.148.153.213": "141.148.128.0/18",
-  "216.73.217.87": "216.73.216.0/22",
-  "185.177.72.70": "185.177.72.0/24",
-  "74.7.241.46": "74.7.0.0/16",
-  "216.73.216.67": "216.73.216.0/22",
-  "185.198.240.238": "185.198.240.0/24",
-  "34.125.213.146": "34.125.208.0/20",
-  "222.176.200.21": "222.176.0.0/13",
-  "106.117.109.102": "106.117.0.0/16",
-  "121.29.51.109": "121.28.0.0/15",
-  "74.248.97.169": "74.248.0.0/15",
-  "34.176.154.83": "34.176.144.0/20",
-  "35.196.132.85": "35.196.128.0/20"
-}

deploy/README.md

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+#
+# Deployment support for the intrusion-detection IDS.
+#
+# This directory is intentionally small and uses no external framework:
+#   deploy.sh                  conflict-free `git pull` update wrapper (D004)
+#   ids.service               systemd unit template (D005)
+#   restart-service.sh         supervisor-agnostic restart shim (D005)
+#   check_runtime_git.sh       proves a scan never dirties tracked runtime files (D001)
+#
+# See .github/instructions/DEPLOYMENT.md for the full plan.
+
+set -euo pipefail

deploy/check_runtime_git.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+#
+# D001 — runtime write-path guard.
+#
+# Proves the deployment invariant: after running a scan, `git status` shows
+# NO tracked file modified by the scan — every runtime write lands in a
+# gitignored location (data/, reports/, *.db). This is what makes `git pull`
+# conflict-free on a server where scans keep running.
+#
+# If this fails, a scan wrote into a git-tracked path and `git pull` on the
+# server can conflict. Investigate the printed files before deploying.
+#
+# Usage:
+#   deploy/check_runtime_git.sh [--live]
+#     --live   additionally run a real scan first (requires log sources) and
+#              then assert the tree is still clean. Without --live it only
+#              verifies that whatever runtime files already exist are untracked.
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+if [[ "${1:-}" == "--live" ]]; then
+  echo "== Running a real scan to exercise every write path =="
+  python app.py scan --quiet --dry-run || true
+fi
+
+echo "== Checking that runtime data is gitignored / untracked =="
+# Paths a scan writes to. Any of these being tracked OR showing as (non-ignored)
+# untracked means the invariant is broken and a git pull could conflict.
+RUNTIME_PATHS=(
+  "app/data"
+  "app/reports"
+  "app/data/ids.db"
+  "app/data/ids.db-wal"
+  "app/data/ids.db-shm"
+)
+
+fail=0
+# 1) No runtime path may be TRACKED, except the single allowed read-only
+#    PaloAlto seed. `git ls-files` reflects the index (including staged
+#    changes), so unborn commits don't cause false failures.
+allowed="app/data/paloalto/scanner-ips.txt"
+tracked=$(git ls-files app/data app/reports)
+problem=$(printf '%s\n' "$tracked" | grep -xvF "$allowed" || true)
+if [[ -n "$problem" ]]; then
+  echo "FAIL: unexpected tracked files under runtime dirs:" >&2
+  echo "$problem" >&2
+  fail=1
+fi
+
+# 2) Nothing may show as untracked-but-not-ignored under the runtime dirs, i.e.
+#    a scan wrote a file git would happily add (and later conflict on).
+for p in "${RUNTIME_PATHS[@]}"; do
+  dirty=$(git status --porcelain --untracked-files=all -- "$p" 2>/dev/null || true)
+  if [[ -n "$dirty" ]]; then
+    echo "FAIL: runtime path has changes git cares about:" >&2
+    echo "$dirty" >&2
+    fail=1
+  fi
+done
+
+if [[ "$fail" -eq 0 ]]; then
+  echo "OK: no tracked runtime data; git pull will not conflict with scan writes."
+else
+  echo "BROKEN: a scan wrote into git-tracked paths — fix before deploying." >&2
+  exit 1
+fi

deploy/deploy.sh

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+#
+# D004 — conflict-free deployment update wrapper.
+#
+# Updates a deployed instance with `git pull` WITHOUT ever hitting a merge
+# conflict from files a scan wrote since the last pull. Guarantees:
+#
+#   1. Refuses to update over uncommitted tracked changes (clean-tree guard).
+#   2. Fast-forwards only (--ff-only): never creates merge commits and never
+#      silently conflicts.
+#   3. Runtime data (data/, reports/, *.db) is gitignored, so a running scan
+#      leaves no dirty tracked paths and never blocks the pull.
+#   4. Reinstalls dependencies only if requirements.txt changed.
+#   5. Applies DB migrations idempotently.
+#   6. Restarts the app under its supervisor.
+#
+# Usage:
+#   sudo deploy/deploy.sh            # update + restart (default env: production)
+#   ENV=production deploy/deploy.sh  # use deploy/ids@production.service
+#   deploy/deploy.sh --no-restart    # update deps+migrate but don't restart
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+ENV="${ENV:-production}"
+RESTART="${1:-restart}"
+ROOT="$(pwd)"
+PY="${PY:-python}"
+
+echo "== Intrusion Detection deploy (env=$ENV) =="
+
+# 1) Clean-tree guard. Runtime data is gitignored, so this only fails if a
+#    human edited tracked source without committing.
+if ! git diff --quiet HEAD || ! git diff --cached --quiet; then
+  echo "REFUSING: working tree is not clean." >&2
+  echo "Commit or stash tracked changes, then re-run." >&2
+  echo "  git status --short" >&2
+  exit 1
+fi
+
+# 2) Pre-flight: prove runtime isolation holds (fail fast before touching repo).
+if "$ROOT/deploy/check_runtime_git.sh"; then
+  echo "== runtime isolation ok =="
+else
+  echo "ABORT: runtime isolation check failed." >&2
+  exit 1
+fi
+
+# 3) Fast-forward only.
+echo "== fetching upstream =="
+git fetch origin
+echo "== fast-forward pull =="
+git pull --ff-only
+
+# 4) Dependencies (only if the requirement set changed since the last pull).
+if git diff --quiet HEAD@{1} HEAD -- requirements.txt; then
+  echo "== requirements.txt unchanged; skipping pip =="
+else
+  echo "== requirements.txt changed; installing deps =="
+  command -v "$PY" >/dev/null && "$PY" -m pip install -r requirements.txt \
+    || pip install -r requirements.txt
+fi
+
+# 5) Apply DB migrations (idempotent).
+echo "== applying DB migrations =="
+"$PY" -m app.db --migrate
+
+# 6) Restart under the supervisor.
+if [[ "$RESTART" == "--no-restart" ]]; then
+  echo "== --no-restart: update complete; restart manually =="
+else
+  echo "== restarting service =="
+  "$ROOT/deploy/restart-service.sh" "$ENV"
+fi
+
+echo "== deploy complete =="

deploy/ids.service

@@ -0,0 +1,41 @@
+# systemd unit template (D005) — intrusion-detection web UI.
+#
+# Install:
+#   - Replace /path/to/intrusion-detection and the User/Group.
+#   - Copy to /etc/systemd/system/ids.service
+#   - sudo systemctl daemon-reload && sudo systemctl enable --now ids
+#
+# The app binds to 127.0.0.1:<WEB_PORT>; it must NOT be exposed to the LAN.
+#
+# Restart policy: automatically restart on crash. `deploy/deploy.sh` restarts
+# this unit after a successful `git pull`. Scans run in a background thread, so
+# one (serialized) scan at a time; deploy waits for the worker to be idle
+# (see restart-service.sh) before restarting.
+
+[Unit]
+Description=Intrusion Detection web UI + IDS scanner
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+# Run as an unprivileged user; the app only needs read access to the log dirs
+# it consumes (COPYPARTY_LOG_PATH / NPM_LOG_PATH) and write access to its own
+# data/ + reports/ + ids.db.
+User=ids
+Group=ids
+WorkingDirectory=/path/to/intrusion-detection
+EnvironmentFile=-/etc/ids.env
+ExecStart=/path/to/intrusion-detection/.venv/bin/python /path/to/intrusion-detection/app.py
+Restart=on-failure
+RestartSec=5
+
+# Hardening
+NoNewPrivileges=yes
+PrivateTmp=yes
+ProtectSystem=full
+ProtectHome=no
+ReadWritePaths=/path/to/intrusion-detection/data /path/to/intrusion-detection/reports /var/lib/ids
+
+[Install]
+WantedBy=multi-user.target

deploy/restart-service.sh

@@ -0,0 +1,53 @@
+#!/usr/bin/env bash
+#
+# D005 — supervisor-agnostic restart shim.
+#
+# Restarts the intrusion-detection service after a deploy, preferring systemd
+# (Linux), with a fallback message for other environments (Windows dev).
+#
+# It first waits for any in-flight background scan to finish so a checkout
+# change never interleaves with a live scan (belt-and-suspenders; the runtime
+# isolation guarantee already means scans only write gitignored files).
+#
+# Usage:
+#   deploy/restart-service.sh [production]
+#   ENV=production deploy/restart-service.sh
+set -euo pipefail
+cd "$(dirname "$0")/.."
+
+ENV="${1:-${ENV:-production}}"
+UNIT="ids@${ENV}.service"
+
+# Wait for an in-flight scan to finish (~0..120s). We probe the web UI status
+# endpoint if it is reachable; a failure to reach it just means no web UI is up.
+web_host="${WEB_HOST:-127.0.0.1}"
+web_port="${WEB_PORT:-54666}"
+status_url="http://${web_host}:${web_port}/api/scan/status"
+for _ in $(seq 1 60); do
+  running=$(curl -fsS --max-time 2 "$status_url" 2>/dev/null \
+    | python3 -c 'import sys,json;print(json.load(sys.stdin).get("running",False))' 2>/dev/null \
+    || echo false)
+  [[ "$running" == "false" ]] && break
+  echo "  waiting for running scan to finish..."
+  sleep 2
+done
+
+if command -v systemctl >/dev/null 2>&1; then
+  # Installed unit is <unit>.service.template; the enabled name includes env.
+  if systemctl list-unit-files "${UNIT}" >/dev/null 2>&1; then
+    echo "== systemctl restart ${UNIT} =="
+    systemctl restart "${UNIT}"
+    systemctl --no-pager status "${UNIT}" --lines=0 || true
+  elif systemctl list-unit-files "ids.service" >/dev/null 2>&1; then
+    echo "== systemctl restart ids.service =="
+    systemctl restart ids.service
+    systemctl --no-pager status ids.service --lines=0 || true
+  else
+    echo "No ids systemd unit found; restart manually." >&2
+    exit 2
+  fi
+else
+  echo "No systemd available (Windows dev?). Restart the app manually." >&2
+  echo "  python app.py   # in a terminal, or via your dev runner" >&2
+  exit 2
+fi

reports/app/data/blocklist_ipv4.txt

@@ -1,559 +0,0 @@
-102.37.110.74/32
-103.153.183.126/32
-103.168.66.237/32
-103.177.150.66/32
-103.215.75.19/32
-103.215.75.66/32
-103.221.220.62/32
-103.76.88.36/32
-103.76.88.37/32
-104.168.93.75/32
-104.196.62.35/32
-104.199.182.104/32
-104.208.94.19/32
-104.234.19.67/32
-104.238.222.26/32
-104.248.151.168/32
-104.248.5.228/32
-104.251.181.106/32
-104.28.215.219/32
-104.28.215.220/32
-104.28.222.131/32
-104.28.240.0/24
-104.28.247.219/32
-104.28.247.220/32
-104.28.254.131/32
-104.28.254.134/32
-106.117.109.102/32
-106.63.0.0/17
-107.170.9.103/32
-108.130.204.104/32
-110.176.0.0/13
-111.113.88.210/32
-111.113.89.107/32
-112.94.252.15/32
-113.57.186.101/32
-117.14.153.168/32
-118.194.253.208/32
-118.212.120.78/32
-118.81.196.142/32
-118.81.207.202/32
-121.29.51.109/32
-123.158.61.179/32
-123.160.175.254/32
-123.163.114.150/32
-123.178.210.228/32
-124.117.194.69/32
-124.88.113.10/32
-128.199.182.152/32
-128.199.182.55/32
-128.90.128.13/32
-128.90.141.10/32
-129.212.237.196/32
-13.126.86.18/32
-13.221.106.198/32
-13.57.15.41/32
-13.83.161.80/32
-130.12.180.126/32
-130.12.180.39/32
-130.12.182.12/32
-130.12.182.254/32
-130.131.231.171/32
-132.196.11.13/32
-132.196.83.33/32
-134.122.94.138/32
-134.199.148.147/32
-134.209.162.128/32
-134.209.185.72/32
-134.209.25.199/32
-135.119.63.61/32
-135.119.74.228/32
-135.225.93.215/32
-135.232.177.160/32
-136.107.208.33/32
-136.107.217.8/32
-136.108.11.195/32
-136.108.15.119/32
-136.109.87.37/32
-136.110.80.8/32
-136.115.17.125/32
-136.116.215.248/32
-136.144.19.51/32
-136.144.33.102/32
-136.144.35.201/32
-136.144.43.127/32
-136.64.132.60/32
-136.64.51.224/32
-136.66.128.85/32
-136.66.153.34/32
-136.70.149.147/32
-138.124.103.234/32
-138.197.104.211/32
-138.197.191.87/32
-138.197.34.12/32
-139.28.219.70/32
-139.59.118.17/32
-139.59.231.238/32
-140.248.75.106/32
-141.148.153.213/32
-141.98.11.171/32
-142.111.152.16/32
-142.111.152.57/32
-142.93.155.36/32
-142.93.62.168/32
-143.110.178.165/32
-143.110.217.244/32
-143.198.25.99/32
-143.198.6.225/32
-143.20.64.236/32
-143.244.162.58/32
-143.244.168.161/32
-143.244.186.167/32
-143.244.56.0/23
-143.244.57.123/32
-144.126.207.57/32
-144.172.100.174/32
-144.172.101.168/32
-144.172.93.117/32
-144.172.97.164/32
-144.76.19.27/32
-145.132.103.19/32
-146.190.63.248/32
-146.190.63.48/32
-146.190.83.166/32
-146.70.194.0/24
-146.70.40.68/32
-147.182.149.75/32
-147.93.168.0/22
-147.93.171.188/32
-148.251.182.79/32
-149.56.150.98/32
-15.206.146.240/32
-150.255.94.250/32
-151.240.45.69/32
-152.39.198.115/32
-152.42.137.175/32
-152.53.242.93/32
-153.75.89.249/32
-155.2.215.98/32
-157.143.3.35/32
-157.245.105.107/32
-157.245.113.227/32
-157.245.197.36/32
-157.245.204.205/32
-157.245.63.62/32
-158.101.144.198/32
-158.158.0.0/16
-158.51.126.144/32
-159.203.13.62/32
-159.223.132.86/32
-159.223.79.25/32
-159.65.10.116/32
-159.65.144.72/32
-159.89.117.109/32
-159.89.12.166/32
-159.89.123.252/32
-159.89.174.87/32
-159.89.46.165/32
-159.89.97.19/32
-16.146.99.254/32
-160.179.166.124/32
-160.179.166.222/32
-161.118.251.181/32
-162.141.167.14/32
-162.141.167.71/32
-162.243.79.174/32
-163.7.11.71/32
-163.7.12.0/24
-163.7.13.0/24
-164.90.169.163/32
-165.22.147.152/32
-165.22.235.3/32
-165.22.24.78/32
-165.245.185.158/32
-165.245.190.205/32
-167.172.192.26/32
-167.172.31.131/32
-167.172.53.28/32
-167.71.0.47/32
-167.71.39.61/32
-167.86.82.167/32
-167.99.181.249/32
-167.99.182.39/32
-167.99.204.6/32
-167.99.210.137/32
-168.144.100.227/32
-168.144.106.212/32
-168.144.167.238/32
-168.144.174.42/32
-168.144.83.150/32
-168.62.180.116/32
-170.64.144.116/32
-170.64.156.114/32
-171.104.81.195/32
-172.170.254.20/32
-172.171.198.214/32
-172.196.32.64/32
-172.200.0.0/13
-172.208.0.0/13
-172.213.216.126/32
-172.232.175.132/32
-172.245.102.64/32
-172.86.119.249/32
-172.86.122.106/32
-172.98.33.251/32
-173.239.196.33/32
-173.239.196.35/32
-173.239.240.222/32
-173.249.34.197/32
-176.227.240.17/32
-176.227.240.61/32
-178.128.176.132/32
-178.16.54.137/32
-178.62.196.238/32
-179.43.134.114/32
-18.142.90.109/32
-18.191.56.164/32
-18.227.73.7/32
-18.236.136.230/32
-180.95.231.78/32
-182.119.230.145/32
-182.88.226.154/32
-185.151.21.124/32
-185.177.72.0/24
-185.177.72.29/32
-185.192.71.0/24
-185.198.240.238/32
-185.213.175.168/32
-185.223.152.0/24
-185.238.231.167/32
-185.251.19.30/32
-185.93.89.147/32
-185.93.89.167/32
-188.166.245.173/32
-188.213.202.0/24
-191.232.0.0/13
-192.133.76.0/23
-192.133.77.15/32
-192.253.248.142/32
-192.253.248.7/32
-192.34.61.236/32
-193.143.1.112/32
-193.148.56.148/32
-193.148.57.33/32
-193.149.176.55/32
-193.19.109.0/24
-193.36.224.57/32
-193.36.225.0/24
-193.37.32.0/24
-193.37.33.0/24
-193.47.62.168/32
-194.102.104.29/32
-194.156.79.68/32
-194.233.91.21/32
-194.26.192.156/32
-194.32.120.151/32
-194.5.82.0/24
-195.178.110.0/24
-195.178.110.211/32
-195.178.110.223/32
-198.98.52.229/32
-199.16.157.0/24
-2.57.122.13/32
-2.57.122.173/32
-2.58.56.101/32
-20.0.0.0/11
-20.150.0.0/15
-20.160.0.0/12
-20.184.150.240/32
-20.192.0.0/10
-20.215.189.213/32
-20.46.121.148/32
-20.46.252.52/32
-20.48.0.0/12
-20.64.0.0/10
-200.162.144.0/20
-201.78.128.89/32
-203.159.90.217/32
-206.189.19.19/32
-206.189.233.36/32
-206.189.95.232/32
-206.81.12.187/32
-206.81.24.227/32
-207.154.197.113/32
-207.241.173.229/32
-208.84.102.195/32
-209.126.2.173/32
-209.87.169.0/24
-209.97.166.251/32
-209.99.184.0/21
-211.175.22.56/32
-213.111.158.220/32
-213.35.113.47/32
-216.24.210.46/32
-216.24.212.0/24
-216.26.233.39/32
-216.73.163.0/24
-216.73.216.0/22
-216.73.217.176/32
-216.81.248.20/32
-217.217.252.197/32
-220.202.112.100/32
-221.13.93.71/32
-222.176.200.21/32
-222.176.200.251/32
-223.199.187.171/32
-23.102.123.231/32
-23.161.169.127/32
-23.161.169.62/32
-23.178.112.0/24
-23.180.120.0/24
-23.234.98.171/32
-23.94.155.57/32
-23.95.226.104/32
-24.144.85.55/32
-3.101.54.4/32
-3.107.107.18/32
-3.237.185.144/32
-3.250.52.216/32
-3.27.206.50/32
-3.77.92.222/32
-3.79.41.12/32
-31.184.236.8/32
-31.56.58.0/24
-31.58.51.130/32
-31.58.51.184/32
-31.59.129.193/32
-34.105.30.251/32
-34.11.108.163/32
-34.11.127.22/32
-34.116.255.65/32
-34.118.184.2/32
-34.125.213.146/32
-34.125.219.56/32
-34.125.42.15/32
-34.131.251.163/32
-34.139.141.56/32
-34.141.160.140/32
-34.145.62.28/32
-34.146.12.12/32
-34.148.8.63/32
-34.148.98.62/32
-34.154.122.183/32
-34.158.83.192/32
-34.17.159.73/32
-34.171.75.208/32
-34.175.82.8/32
-34.176.154.83/32
-34.18.38.137/32
-34.181.134.189/32
-34.186.83.134/32
-34.187.20.70/32
-34.187.200.119/32
-34.187.82.183/32
-34.21.144.169/32
-34.23.104.90/32
-34.24.155.203/32
-34.242.226.103/32
-34.244.177.56/32
-34.26.191.118/32
-34.26.219.248/32
-34.31.88.250/32
-34.39.210.94/32
-34.40.240.124/32
-34.42.48.100/32
-34.47.183.174/32
-34.47.194.183/32
-34.55.200.45/32
-34.57.157.110/32
-34.59.187.134/32
-34.62.38.199/32
-34.64.173.205/32
-34.65.138.195/32
-34.65.14.246/32
-34.65.18.71/32
-34.69.107.154/32
-34.7.160.235/32
-34.7.193.250/32
-34.73.128.17/32
-34.74.203.130/32
-34.75.4.161/32
-34.82.10.34/32
-34.83.69.198/32
-34.84.59.119/32
-34.86.126.198/32
-34.86.217.208/32
-34.90.87.32/32
-34.90.88.37/32
-34.91.243.226/32
-34.94.172.72/32
-34.95.238.121/32
-34.95.239.96/32
-35.157.251.190/32
-35.182.17.8/32
-35.184.157.121/32
-35.185.21.88/32
-35.187.221.219/32
-35.187.9.30/32
-35.192.232.168/32
-35.196.22.146/32
-35.198.180.224/32
-35.199.111.12/32
-35.199.115.51/32
-35.199.183.190/32
-35.201.159.247/32
-35.204.141.252/32
-35.204.88.252/32
-35.230.52.27/32
-35.231.0.0/17
-35.233.191.179/32
-35.237.138.104/32
-35.237.178.50/32
-35.243.173.211/32
-35.243.223.147/32
-35.255.40.65/32
-35.95.56.230/32
-36.106.166.223/32
-36.106.166.92/32
-37.140.223.58/32
-37.140.223.6/32
-38.248.95.248/32
-4.176.0.0/12
-4.192.0.0/12
-4.208.0.0/12
-4.224.0.0/12
-4.246.70.112/32
-41.143.154.1/32
-43.228.157.102/32
-43.228.157.191/32
-43.98.201.177/32
-44.224.0.0/11
-45.130.203.154/32
-45.130.203.177/32
-45.131.193.139/32
-45.131.195.0/24
-45.134.20.116/32
-45.134.20.128/32
-45.138.16.114/32
-45.139.104.172/32
-45.146.54.137/32
-45.146.54.80/32
-45.146.55.0/24
-45.148.10.0/24
-45.153.34.16/32
-45.153.34.182/32
-45.153.34.43/32
-45.154.138.181/32
-45.154.138.232/32
-45.154.98.0/24
-45.45.237.214/32
-45.45.237.69/32
-45.61.150.67/32
-45.61.187.30/32
-45.67.85.190/32
-45.8.196.200/32
-45.88.138.44/32
-46.23.108.234/32
-46.232.235.0/24
-47.85.160.47/32
-5.161.225.116/32
-5.161.62.209/32
-5.253.247.16/32
-5.255.119.161/32
-5.34.213.142/32
-51.103.133.211/32
-51.103.152.96/32
-51.116.233.22/32
-51.120.0.0/16
-51.79.208.246/32
-52.139.36.144/32
-52.159.244.176/32
-52.160.0.0/11
-52.224.0.0/11
-52.64.227.143/32
-54.163.186.39/32
-54.169.103.174/32
-54.179.129.22/32
-54.206.147.115/32
-54.210.219.163/32
-54.216.227.171/32
-54.251.19.144/32
-54.82.69.43/32
-58.212.237.231/32
-62.60.130.128/32
-62.60.130.227/32
-62.93.179.146/32
-64.225.63.129/32
-64.225.75.246/32
-64.226.123.37/32
-64.227.32.66/32
-64.23.218.208/32
-64.89.161.82/32
-64.89.161.85/32
-65.0.122.218/32
-66.187.5.228/32
-66.187.6.102/32
-66.228.38.221/32
-66.249.64.0/20
-68.183.180.73/32
-68.183.9.16/32
-68.220.0.0/15
-69.164.220.232/32
-69.164.220.85/32
-69.30.85.194/32
-72.146.20.230/32
-72.146.33.199/32
-73.116.194.16/32
-74.208.40.151/32
-74.248.0.0/15
-74.248.33.172/32
-74.7.0.0/16
-77.90.185.230/32
-8.229.1.130/32
-8.229.130.50/32
-8.231.78.19/32
-8.234.157.73/32
-8.234.207.247/32
-8.235.114.162/32
-80.91.86.169/32
-80.94.92.167/32
-81.171.72.135/32
-81.171.72.93/32
-81.171.74.60/32
-82.102.18.0/24
-82.102.18.190/32
-82.165.82.154/32
-82.39.206.154/32
-84.17.60.251/32
-85.11.167.225/32
-85.204.70.0/24
-87.121.79.42/32
-88.184.210.4/32
-9.205.16.190/32
-91.124.88.29/32
-91.124.88.36/32
-91.134.35.95/32
-91.148.244.131/32
-91.148.245.81/32
-91.192.10.243/32
-91.192.10.86/32
-91.193.232.0/24
-91.224.92.159/32
-91.224.92.190/32
-91.230.225.0/24
-91.239.79.146/32
-91.92.241.196/32
-91.92.243.245/32
-91.92.41.115/32
-91.92.42.86/32
-91.92.47.0/24
-92.118.39.126/32
-92.118.39.29/32
-92.119.121.196/32
-92.165.170.72/32
-93.123.109.0/24
-93.152.221.13/32
-94.154.43.0/24
-95.85.238.18/32
-98.159.36.147/32
-98.159.37.147/32

reports/app/data/blocklist_ipv6.txt

@@ -1,24 +0,0 @@
-2001:4ca0:108:42::24/128
-2001:ba0:24c:a500::1/128
-2400:8901::2000:62ff:fe9a:e1c2/128
-2400:8901::2000:beff:fea3:5a44/128
-2406:da18:525:0:a23e:e8f6:3ea1:4c2c/128
-2600:3000::/32
-2600:3c09::2000:5cff:fe99:c802/128
-2600:3c15::2000:53ff:fe31:8c0b/128
-2600:3c15::2000:9cff:fef0:8ad6/128
-2602:fa59::/40
-2a00:6340:2008:ec80:64cc:9116:beb2:1931/128
-2a00:6340:2008:ec80:be24:11ff:fe3b:4058/128
-2a00:6340:2008:ec80:be24:11ff:fe82:2775/128
-2a01:2a8:a0dd:1::1012/128
-2a01:4f8:c014:7f9e::1/128
-2a04:4e40:4400:0:537:cdd2:7250:95db/128
-2a04:4e40:e000::/48
-2a04:c300::/29
-2a05:4140:3::37cd/128
-2a06:a880:5:7fb3::1/128
-2a09:bac1:76c0:460::5e:6a/128
-2a09:bac5::/32
-2a0a:4cc0:2000:8528:c4e8:3ff:febb:3ee9/128
-2a13:adc0::2ed:89ff:fe58:19c5/128

requirements.txt

@@ -1,3 +1,3 @@
+flask
 paramiko
 python-dotenv
-flask

start.ps1

@@ -0,0 +1,21 @@
+<#
+.SYNOPSIS
+    Starts the flask application in a virtual environment.
+
+.DESCRIPTION
+    This script activates the virtual environment and starts the flask application.
+    It only runs a single threaded instance of the application, so it is not suitable for production use.
+#>
+
+## Check if the virtual environment exists
+if (-Not (Test-Path -Path ".\.venv\Scripts\Activate.ps1")) {
+    Write-Host "Virtual environment not found. Please create it first."
+    exit 1
+}
+
+## Activate the virtual environment
+& .\.venv\Scripts\Activate.ps1
+
+## Start the Flask application
+Write-Host "Starting the Flask application..."
+& .\.venv\Scripts\python.exe .\app.py

tests/test_deployment.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python3
+"""Deployment invariant tests (D001).
+
+Proves that running a scan never creates or modifies git-tracked runtime data,
+which is what makes ``git pull`` conflict-free on a deployed server.
+
+Run with:
+    python -m unittest discover -s tests -p 'test_*.py'
+or (if pytest is installed):
+    python -m pytest tests/
+"""
+
+from __future__ import annotations
+
+import os
+import subprocess
+import sys
+import unittest
+
+ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+# The paloalto scanner-ips.txt seed is deliberately tracked (read-only); every
+# other file under the runtime dirs must remain untracked.
+ALLOWED_TRACKED_RUNTIME = {os.path.join("app", "data", "paloalto",
+                                        "scanner-ips.txt")}
+
+
+def _git(*args: str) -> str:
+    return subprocess.run(["git", *args], cwd=ROOT, check=True,
+                          capture_output=True, text=True).stdout
+
+
+class RuntimeGitTest(unittest.TestCase):
+    def test_no_runtime_data_tracked(self):
+        """No runtime file (other than the read-only paloalto seed) is tracked."""
+        tracked = _git("ls-files", "app/data", "app/reports").splitlines()
+        unexpected = [f for f in tracked
+                      if f not in ALLOWED_TRACKED_RUNTIME]
+        self.assertEqual(
+            unexpected, [],
+            "git-tracked runtime files would make `git pull` conflict; "
+            "add them to .gitignore: %s" % unexpected)
+
+    def test_runtime_write_paths_ignored(self):
+        """Every runtime write path is ignored by git (won't dirty the tree)."""
+        stats = subprocess.run(
+            ["git", "check-ignore", "-v",
+             "app/data/scan_state.json", "app/data/scan_jobs.json",
+             "app/data/whois_cache.json", "app/data/ids.db",
+             "app/data/ids.db-wal", "app/reports/app/data/blocklist_ipv4.txt",
+             ], cwd=ROOT, capture_output=True, text=True).stdout
+        self.assertIn("scan_state.json", stats)
+        self.assertIn("ids.db", stats)
+        self.assertIn("blocklist_ipv4.txt", stats)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_firewall_sync.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Tests for evidence-first firewall sync behavior."""
+
+from __future__ import annotations
+
+import unittest
+from unittest.mock import patch
+
+from app.firewall import MikrotikFirewall
+
+
+class _FakeFirewall(MikrotikFirewall):
+    def __init__(self):
+        super().__init__(router_ip="0.0.0.0", username="x", password="x")
+        self.calls: list[str] = []
+
+    def get_address_list_ipv4(self, name: str) -> set[str]:
+        self.calls.append(f"v4:{name}")
+        if name == "Blocked":
+            return {"9.9.9.9/32"}
+        if name == "Init7":
+            return {"77.74.80.0/21"}
+        return set()
+
+    def get_address_list_ipv6(self, name: str) -> set[str]:
+        self.calls.append(f"v6:{name}")
+        if name == "Blocked":
+            return set()
+        if name == "Init7":
+            return {"2001:1620::/30"}
+        return set()
+
+
+class FirewallSyncEvidenceTest(unittest.TestCase):
+    def test_sync_loads_router_status_before_computing_changes(self):
+        fw = _FakeFirewall()
+        # type: ignore[assignment]
+        fw.save_state_report = lambda *args, **kwargs: "noop"
+
+        with patch("app.firewall.cumulative_blocklist",
+                   return_value=({"8.8.8.8/32"}, set())):
+            changes = fw.sync_blocklist(["8.8.8.8/32"], [], dry_run=True)
+
+        self.assertIn("v4:Blocked", fw.calls)
+        self.assertIn("v6:Blocked", fw.calls)
+        self.assertIn("v4:Init7", fw.calls)
+        self.assertIn("v6:Init7", fw.calls)
+        self.assertEqual(changes["added_ipv4"], ["8.8.8.8/32"])
+        self.assertEqual(changes["removed_ipv4"], ["9.9.9.9/32"])
+
+    def test_sync_evaluates_against_loaded_current_status(self):
+        fw = _FakeFirewall()
+        # type: ignore[assignment]
+        fw.save_state_report = lambda *args, **kwargs: "noop"
+
+        with patch("app.firewall.cumulative_blocklist",
+                   return_value=({"9.9.9.9/32"}, set())):
+            changes = fw.sync_blocklist(["9.9.9.9/32"], [], dry_run=True)
+
+        self.assertEqual(changes["added_ipv4"], [])
+        self.assertEqual(changes["removed_ipv4"], [])
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_report_backfill.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""Tests for legacy report artifact backfill."""
+
+from __future__ import annotations
+
+import os
+import tempfile
+import unittest
+from pathlib import Path
+
+from app.db import connect
+from app.db.backfill import backfill_report_artifacts
+from app.db import repositories as repo
+
+
+class ReportBackfillTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp_dir.name)
+        self.conn = connect(str(self.root / "test.db"))
+        self.roots = {
+            "copyparty": str(self.root / "copyparty"),
+            "npm": str(self.root / "npm"),
+            "app": str(self.root / "app"),
+            "mikrotik": str(self.root / "mikrotik"),
+        }
+        for path in self.roots.values():
+            Path(path).mkdir(parents=True)
+
+    def tearDown(self):
+        self.conn.close()
+        self.temp_dir.cleanup()
+
+    def test_dry_run_and_import_are_idempotent(self):
+        Path(self.roots["copyparty"], "2026-11-06-11-30-10.txt").write_text(
+            "copyparty summary\n", encoding="utf-8")
+        Path(self.roots["npm"], "2026-11-06.csv").write_text(
+            "ip,count\n192.0.2.1,3\n", encoding="utf-8")
+        Path(self.roots["app"], "2026-11-06.json").write_text(
+            '{"summary":{"blocked":1}}', encoding="utf-8")
+        Path(self.roots["app"], "data").mkdir()
+        Path(self.roots["app"], "data", "requests.csv").write_text(
+            "ip\n192.0.2.2\n", encoding="utf-8")
+
+        preview = backfill_report_artifacts(
+            self.conn, dry_run=True, roots=self.roots)["reports"]
+        self.assertEqual(preview["would_import"], 3)
+        self.assertEqual(preview["skipped_canonical"], [
+            {"source": "app", "path": "data/requests.csv"}])
+        self.assertEqual(preview["by_source"]["copyparty"]["would_import"], 1)
+        self.assertEqual(preview["by_format"]["csv"]["would_import"], 1)
+        self.assertEqual(repo.list_report_artifacts(self.conn)["items"], [])
+
+        imported = backfill_report_artifacts(
+            self.conn, roots=self.roots)["reports"]
+        self.assertEqual(imported["imported"], 3)
+        self.assertEqual(imported["unmapped"], [])
+
+        repeated = backfill_report_artifacts(
+            self.conn, roots=self.roots)["reports"]
+        self.assertEqual(repeated["imported"], 0)
+        self.assertEqual(repeated["already_imported"], 3)
+
+        items = repo.list_report_artifacts(
+            self.conn, source="copyparty")["items"]
+        self.assertEqual(items[0]["original_path"],
+                         "copyparty/2026-11-06-11-30-10.txt")
+        self.assertEqual(items[0]["generated_at"],
+                         "2026-11-06T11:30:10+00:00")
+
+    def test_malformed_json_is_preserved_with_parse_error(self):
+        Path(self.roots["app"], "2026-11-06.json").write_text(
+            '{"incomplete":', encoding="utf-8")
+
+        result = backfill_report_artifacts(
+            self.conn, roots=self.roots)["reports"]
+        self.assertEqual(result["imported"], 1)
+        artifact = repo.list_report_artifacts(
+            self.conn, source="app")["items"][0]
+        detail = repo.get_report_artifact(self.conn, artifact["id"])
+        self.assertEqual(detail["payload"]["content"], '{"incomplete":')
+        self.assertIn("parse_error", detail["payload"])
+
+    def test_artifacts_survive_sqlite_backup_and_restore(self):
+        artifact_id = repo.insert_report_artifact(
+            self.conn, source="app", report_type="summary",
+            generated_at="2026-11-06T00:00:00+00:00",
+            payload={"summary": {"count": 4}},
+            original_path="app/2026-11-06.json", content_sha256="c" * 64,
+        )
+        restored = connect(str(self.root / "restored.db"))
+        try:
+            self.conn.backup(restored)
+            artifact = repo.get_report_artifact(restored, artifact_id)
+            self.assertEqual(artifact["payload"], {"summary": {"count": 4}})
+        finally:
+            restored.close()
+
+    def test_oversized_file_is_reported_without_deleting_source(self):
+        source = Path(self.roots["npm"], "large.txt")
+        source.write_text("keep", encoding="utf-8")
+        from unittest.mock import patch
+
+        with patch("app.db.backfill.MAX_REPORT_ARTIFACT_BYTES", 1):
+            result = backfill_report_artifacts(
+                self.conn, roots=self.roots)["reports"]
+
+        self.assertEqual(len(result["unmapped"]), 1)
+        self.assertTrue(source.exists())
+        self.assertEqual(result["unmapped"][0]["path"], "large.txt")
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_report_persistence.py

@@ -0,0 +1,67 @@
+#!/usr/bin/env python3
+"""Tests for dual-writing generated reports to SQLite."""
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+import app.scan
+from app.analyzer import Analyzer
+from app.db import connect
+from app.db import repositories as repo
+from app.utils.blocklist_history import write_blocklist_report
+
+
+class ReportPersistenceTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.root = Path(self.temp_dir.name)
+        self.conn = connect(str(self.root / "test.db"))
+        self.connection_patch = patch("app.db.get_connection",
+                                      return_value=self.conn)
+        self.connection_patch.start()
+
+    def tearDown(self):
+        self.connection_patch.stop()
+        self.conn.close()
+        self.temp_dir.cleanup()
+
+    def test_analyzer_summary_is_stored_with_file_provenance(self):
+        report_dir = self.root / "copyparty"
+        report_dir.mkdir()
+        path = report_dir / "2026-11-06-11-30-10.txt"
+        path.write_text("summary body", encoding="utf-8")
+        analyzer = Analyzer("logs", str(report_dir), quiet=True)
+
+        analyzer.persist_report_artifact(str(path), scan_id=14)
+
+        item = repo.list_report_artifacts(
+            self.conn, source="ids")["items"][0]
+        detail = repo.get_report_artifact(self.conn, item["id"])
+        self.assertEqual(item["original_path"], "ids/2026-11-06-11-30-10.txt")
+        self.assertEqual(detail["scan_id"], 14)
+        self.assertEqual(detail["payload"]["content"], "summary body")
+
+    def test_blocklist_report_is_stored_as_structured_artifact(self):
+        path = write_blocklist_report(
+            str(self.root / "reports"), history_v4=set(), history_v6=set(),
+            prev_v4=set(), prev_v6=set(),
+            complete_v4={"192.0.2.1/32"}, complete_v6=set(),
+            firewall_changes={"added_ipv4": ["192.0.2.1/32"]},
+            dry_run=True, scan_id=22,
+        )
+
+        items = repo.list_report_artifacts(
+            self.conn, source="app", report_type="blocklist_change")["items"]
+        self.assertEqual(len(items), 1)
+        detail = repo.get_report_artifact(self.conn, items[0]["id"])
+        self.assertEqual(detail["scan_id"], 22)
+        self.assertEqual(detail["payload"]["blocked_ipv4"], ["192.0.2.1/32"])
+        self.assertTrue(Path(path).exists())
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_report_routes.py

@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+"""Tests for database-backed report API routes."""
+
+from __future__ import annotations
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import patch
+
+from app.db import connect
+from app.db import repositories as repo
+from app.web import create_app
+
+
+class ReportRoutesTest(unittest.TestCase):
+    def setUp(self):
+        self.temp_dir = tempfile.TemporaryDirectory()
+        self.conn = connect(str(Path(self.temp_dir.name) / "test.db"))
+        self.app = create_app()
+        self.app.config["TESTING"] = True
+        self.client = self.app.test_client()
+        self.connection_patch = patch("app.db.get_connection",
+                                      return_value=self.conn)
+        self.connection_patch.start()
+        self.artifact_id = repo.insert_report_artifact(
+            self.conn, source="copyparty", report_type="text_report",
+            generated_at="2026-11-06T10:00:00+00:00",
+            payload={"format": "text", "content": "summary"},
+            original_path="copyparty/2026-11-06.txt",
+            content_sha256="b" * 64,
+        )
+
+    def tearDown(self):
+        self.connection_patch.stop()
+        self.conn.close()
+        self.temp_dir.cleanup()
+
+    def test_list_detail_and_text_export(self):
+        listed = self.client.get("/api/reports?source=copyparty&limit=1")
+        self.assertEqual(listed.status_code, 200)
+        self.assertEqual(listed.json["items"][0]["id"], self.artifact_id)
+
+        detail = self.client.get(
+            f"/api/report-artifacts/{self.artifact_id}")
+        self.assertEqual(detail.status_code, 200)
+        self.assertEqual(detail.json["payload"]["content"], "summary")
+
+        exported = self.client.get(
+            f"/api/report-artifacts/{self.artifact_id}/export?format=txt")
+        self.assertEqual(exported.status_code, 200)
+        self.assertEqual(exported.data.decode(), "summary")
+        self.assertIn("attachment", exported.headers["Content-Disposition"])
+        self.assertNotIn(".txt.txt", exported.headers["Content-Disposition"])
+
+    def test_invalid_source_cursor_and_missing_id(self):
+        self.assertEqual(
+            self.client.get("/api/reports?source=unknown").status_code, 400)
+        self.assertEqual(
+            self.client.get("/api/reports?cursor=invalid").status_code, 400)
+        self.assertEqual(
+            self.client.get("/api/report-artifacts/9999").status_code, 404)
+
+    def test_legacy_file_routes_remain_registered(self):
+        response = self.client.get("/api/reports/copyparty")
+        self.assertEqual(response.status_code, 200)
+        self.assertIn("files", response.json)
+
+
+if __name__ == "__main__":
+    unittest.main()

tests/test_repositories.py

@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+"""Tests for the SQLite repositories + schema migrations (T001).
+
+Run with:
+    python -m unittest discover -s tests -p 'test_*.py'
+or (if pytest is installed):
+    python -m pytest tests/
+"""
+
+from __future__ import annotations
+
+import os
+import sqlite3
+import tempfile
+import unittest
+from datetime import datetime, timedelta, timezone
+
+from app.db import connect
+from app.db import repositories as repo
+from app.db.schema import SCHEMA_VERSION, apply_migrations
+
+
+def _new_conn() -> sqlite3.Connection:
+    fd, path = tempfile.mkstemp(suffix=".db")
+    os.close(fd)
+    os.unlink(path)
+    return connect(path)
+
+
+class SchemaTest(unittest.TestCase):
+    def test_migrations_apply_and_are_idempotent(self):
+        conn = _new_conn()
+        self.assertEqual(conn.execute("PRAGMA user_version").fetchone()[0],
+                         SCHEMA_VERSION)
+        # Re-applying should be a no-op (no exception, version unchanged).
+        apply_migrations(conn)
+        self.assertEqual(conn.execute("PRAGMA user_version").fetchone()[0],
+                         SCHEMA_VERSION)
+        conn.close()
+
+    def test_core_tables_exist(self):
+        conn = _new_conn()
+        tables = {r["name"] for r in conn.execute(
+            "SELECT name FROM sqlite_master WHERE type='table'")}
+        for t in ("requests", "ip_stats", "scans", "scans_sources",
+                  "blocklist_entries", "blocklist_events",
+                  "blocklist_reasons", "whitelist_entries",
+                  "whitelist_sources", "firewall_snapshots",
+                  "firewall_address_list_entries", "firewall_rule_entries",
+                  "whois_info", "log_sources", "report_artifacts"):
+            self.assertIn(t, tables, f"missing table {t}")
+        conn.close()
+
+
+class ScanAndRequestTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_scan_lifecycle(self):
+        sid = repo.start_scan(self.conn, "ids", params={"full": True})
+        self.assertEqual(repo.get_scan(self.conn, sid)["status"], "running")
+        repo.finish_scan(self.conn, sid, status="done",
+                         result={"ok": True}, request_count=3)
+        s = repo.get_scan(self.conn, sid)
+        self.assertEqual(s["status"], "done")
+        self.assertEqual(s["request_count"], 3)
+        self.assertEqual([x["id"] for x in repo.latest_scans(self.conn)],
+                         [sid])
+
+    def test_insert_and_query_requests(self):
+        sid = repo.start_scan(self.conn, "ids")
+        rows = [
+            {"source": "npm", "scan_id": sid, "log_file": "a.log",
+             "ts": "2026-09-01 10:00:00", "ip": "1.2.3.4", "version": 4,
+             "method": "GET", "path": "/.env", "status": 404,
+             "host": "h", "user": None, "ua": "curl", "referer": None,
+             "denied": False, "banned": False, "is_error": True, "extra": None},
+            {"source": "copyparty", "scan_id": sid, "log_file": "f.txt",
+             "ts": "2026-09-01 10:00:01", "ip": "1.2.3.4", "version": 4,
+             "method": "GET", "path": "/admin", "status": 403,
+             "host": None, "user": None, "ua": None, "referer": None,
+             "denied": True, "banned": False, "is_error": True, "extra": None},
+        ]
+        self.assertEqual(repo.insert_requests(self.conn, rows), 2)
+        self.assertEqual(repo.count_requests(self.conn), 2)
+        # cross-source
+        cs = repo.correlate_across_sources(self.conn)
+        self.assertEqual(cs[0]["ip"], "1.2.3.4")
+        self.assertEqual(cs[0]["sources"], 2)
+        # timeline
+        tl = repo.ip_timeline(self.conn, "1.2.3.4")
+        self.assertEqual(len(tl), 2)
+        # top ips
+        top = repo.top_ips(self.conn)
+        self.assertEqual(top[0]["total"], 2)
+        repo.finish_scan(self.conn, sid, status="done")
+
+
+class BlocklistTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_grow_only_record(self):
+        r1 = repo.record_blocklist(self.conn, ["5.5.5.5/32", "6.6.6.6/32"],
+                                   added_by="scan:1")
+        self.assertEqual(r1["active"], 2)
+        # a later scan only sees 5.5.5.5; 6.6.6.6 stays (cumulative).
+        r2 = repo.record_blocklist(
+            self.conn, ["5.5.5.5/32"], added_by="scan:2")
+        active = [e["cidr"] for e in repo.active_blocklist(self.conn)]
+        self.assertIn("6.6.6.6/32", active)
+        self.assertEqual(r2["active"], 2)
+
+    def test_unblock(self):
+        repo.record_blocklist(self.conn, ["7.7.7.7/32"])
+        self.assertTrue(repo.unblock_entry(self.conn, "7.7.7.7/32"))
+        self.assertNotIn("7.7.7.7/32",
+                         [e["cidr"] for e in repo.active_blocklist(self.conn)])
+        self.assertFalse(repo.unblock_entry(self.conn, "7.7.7.7/32"))
+
+    def test_prune_old_single_ip_blocks_keeps_ranges(self):
+        old = (datetime.now(timezone.utc) - timedelta(days=31)).isoformat(
+            timespec="seconds")
+        with repo.transaction(self.conn):
+            repo._block_entry_inner(self.conn, "1.2.3.4/32", last_seen=old)
+            repo._block_entry_inner(
+                self.conn, "2001:db8::1/128", last_seen=old)
+            repo._block_entry_inner(self.conn, "1.2.3.0/24", last_seen=old)
+            repo._block_entry_inner(self.conn, "2001:db8::/64", last_seen=old)
+
+        removed = repo.prune_old_single_ip_blocks(
+            self.conn, older_than_days=30)
+        self.assertCountEqual(removed, ["1.2.3.4/32", "2001:db8::1/128"])
+
+        active = {e["cidr"] for e in repo.active_blocklist(self.conn)}
+        self.assertIn("1.2.3.0/24", active)
+        self.assertIn("2001:db8::/64", active)
+        self.assertNotIn("1.2.3.4/32", active)
+        self.assertNotIn("2001:db8::1/128", active)
+
+    def test_prune_old_single_ip_blocks_respects_recency(self):
+        fresh = (datetime.now(timezone.utc) - timedelta(days=5)).isoformat(
+            timespec="seconds")
+        with repo.transaction(self.conn):
+            repo._block_entry_inner(self.conn, "9.9.9.9/32", last_seen=fresh)
+
+        removed = repo.prune_old_single_ip_blocks(
+            self.conn, older_than_days=30)
+        self.assertEqual(removed, [])
+        self.assertIn("9.9.9.9/32",
+                      {e["cidr"] for e in repo.active_blocklist(self.conn)})
+
+
+class WhitelistTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_whitelist_lookup(self):
+        repo.set_whitelist(self.conn, "Cloudflare", ["173.245.48.0/20"],
+                           source="test")
+        self.assertTrue(repo.is_whitelisted_db(self.conn, "173.245.48.5"))
+        self.assertFalse(repo.is_whitelisted_db(self.conn, "8.8.8.8"))
+        # provider replace deactivates old ranges
+        repo.set_whitelist(self.conn, "Cloudflare", ["104.16.0.0/13"])
+        self.assertFalse(repo.is_whitelisted_db(self.conn, "173.245.48.5"))
+
+
+class FirewallTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_firewall_snapshot(self):
+        sid = repo.save_firewall_snapshot(
+            self.conn,
+            address_lists_v4=[{"list": "Blocked", "address": "5.5.5.5/32"}],
+            address_lists_v6=[{"list": "Init7", "address": "2001:1620::/30"}],
+            rules_v4=[{"chain": "input", "action": "drop", "comment": "x"}],
+            rules_v6=[])
+        self.assertEqual(sid, 1)
+        self.assertEqual(self.conn.execute(
+            "SELECT COUNT(*) FROM firewall_address_list_entries").fetchone()[0], 2)
+        self.assertEqual(self.conn.execute(
+            "SELECT COUNT(*) FROM firewall_rule_entries").fetchone()[0], 1)
+
+
+class ReportArtifactTest(unittest.TestCase):
+    def setUp(self):
+        self.conn = _new_conn()
+
+    def tearDown(self):
+        self.conn.close()
+
+    def test_artifact_insert_is_idempotent_and_detail_decodes_payload(self):
+        kwargs = {
+            "source": "copyparty", "report_type": "analyzer_summary",
+            "generated_at": "2026-11-01T10:00:00+00:00",
+            "payload": {"summary": {"requests": 12}},
+            "original_path": "reports/copyparty/2026-11-01.txt",
+            "content_sha256": "a" * 64,
+        }
+        first_id = repo.insert_report_artifact(self.conn, **kwargs)
+        duplicate_id = repo.insert_report_artifact(self.conn, **kwargs)
+
+        self.assertEqual(first_id, duplicate_id)
+        self.assertEqual(self.conn.execute(
+            "SELECT COUNT(*) FROM report_artifacts").fetchone()[0], 1)
+        self.assertEqual(repo.get_report_artifact(self.conn, first_id)["payload"],
+                         kwargs["payload"])
+
+    def test_artifact_cursor_orders_stably_and_filters_source(self):
+        for day, source in (("01", "npm"), ("02", "copyparty"),
+                            ("03", "npm")):
+            repo.insert_report_artifact(
+                self.conn, source=source, report_type="summary",
+                generated_at=f"2026-11-{day}T00:00:00+00:00",
+                payload={"day": day}, original_path=f"reports/{day}.txt",
+                content_sha256=day * 64,
+            )
+
+        first = repo.list_report_artifacts(self.conn, source="npm", limit=1)
+        self.assertEqual(len(first["items"]), 1)
+        self.assertIsNotNone(first["next_cursor"])
+        second = repo.list_report_artifacts(
+            self.conn, source="npm", limit=1, cursor=first["next_cursor"])
+        self.assertEqual(len(second["items"]), 1)
+        self.assertIsNone(second["next_cursor"])
+        self.assertGreater(first["items"][0]["generated_at"],
+                           second["items"][0]["generated_at"])
+
+    def test_artifact_cursor_rejects_invalid_token(self):
+        with self.assertRaises(ValueError):
+            repo.list_report_artifacts(self.conn, cursor="invalid")
+
+
+if __name__ == "__main__":
+    unittest.main()