feat: Implement SQLite schema and migrations for intrusion detection system

- Added schema definition and migration logic in app/db/schema.py.
- Created tables for log sources, requests, IP stats, scans, blocklist, whitelist, and firewall rules.
- Introduced functions to apply migrations and ensure database integrity.

feat: Enhance scan functionality with database integration

- Added _sync_blocklist_to_db function to synchronize blocklist with SQLite.
- Updated run_scan function to handle database connections and scan records.
- Implemented error handling for database operations during scans.

feat: Introduce log analysis page and API endpoints

- Created analysis.html template for displaying log analysis results.
- Added routes for fetching top IPs, IP request timelines, and cross-source IPs.
- Integrated analysis data into the dashboard and status endpoints.

fix: Ensure whitelists are synchronized with the database

- Implemented sync_whitelists_to_db function to persist in-memory whitelists.
- Updated constants.py to define the path for the SQLite database.

refactor: Modularize analysis helpers for better maintainability

- Created _analysis_helpers function to encapsulate SQLite query logic.
- Updated routes.py to utilize new analysis helper functions for API responses.

app/scan/__init__.py

@@ -40,6 +40,22 @@ from ..mikrotik import split_cidr_versions
 from ..utils.whois import consolidate_by_subnet
 
 
+def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
+                          desired_v6: list[str], added_by: str | None = None) -> None:
+    """Best-effort sync of the consolidated blocklist targets into SQLite.
+
+    Keep the DB ledger in sync regardless of whether a router sync happened so
+    analysis always sees the current policy."""
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        conn = get_connection()
+        repo.sync_blocklist(conn, list(desired_v4) + list(desired_v6),
+                            added_by=added_by)
+    except Exception as exc:  # never break the scan for a DB issue
+        print(f"WARN: could not sync blocklist to db ({exc})")
+
+
 @dataclass
 class ScanParams:
     """Configuration for a consolidated scan.
@@ -93,6 +109,24 @@ def run_scan(params: ScanParams) -> ScanResult:
     """
     errors: list[str] = []
 
+    # --- Step 0: open the SQLite store for this consolidated run -------------
+    db_scan_id: int | None = None
+    try:
+        from ..db import get_connection
+        from ..db import repositories as repo
+        from ..utils.whitelists import sync_whitelists_to_db
+        conn = get_connection()
+        sync_whitelists_to_db()
+        params_map = {
+            "full": params.full, "dry_run": params.dry_run,
+            "update_firewall": params.update_firewall,
+            "quiet": params.quiet, "top": params.top,
+        }
+        db_scan_id = repo.start_scan(conn, "ids", params=params_map,
+                                     full=params.full, dry_run=params.dry_run)
+    except Exception as exc:
+        print(f"WARN: sqlite store unavailable ({exc})")
+
     # --- Step 1-3: connect to MikroTik RouterOS and capture current state ---
     # If the router is unreachable we fall back to scan-only mode: still
     # generate reports and blocklists, but skip the firewall sync.
@@ -134,7 +168,7 @@ def run_scan(params: ScanParams) -> ScanResult:
     print("=== Running copyparty scan ===")
     cp_exists = True
     try:
-        cp_analyzer.run()
+        cp_analyzer.run(db_scan_id=db_scan_id)
     except Exception as e:
         errors.append(f"copyparty scan failed: {e}")
         print(f"Error running copyparty scan: {e}")
@@ -143,7 +177,7 @@ def run_scan(params: ScanParams) -> ScanResult:
     print("\n=== Running NPM scan ===")
     npm_exists = True
     try:
-        npm_analyzer.run()
+        npm_analyzer.run(db_scan_id=db_scan_id)
     except Exception as e:
         errors.append(f"NPM scan failed: {e}")
         print(f"Error running NPM scan: {e}")
@@ -155,6 +189,14 @@ def run_scan(params: ScanParams) -> ScanResult:
         print(f"\n{message}")
         if firewall is not None:
             firewall.disconnect()
+        if db_scan_id is not None:
+            try:
+                from ..db import get_connection
+                from ..db import repositories as repo
+                repo.finish_scan(get_connection(), db_scan_id, status="error",
+                                 error="; ".join(errors))
+            except Exception:
+                pass
         return ScanResult(ok=False, errors=errors)
 
     # Consolidate block candidates
@@ -240,6 +282,35 @@ def run_scan(params: ScanParams) -> ScanResult:
           f"(keep {REPORT_RETENTION_DAYS}d / "
           f"max {REPORT_RETENTION_MAX_FILES} per dir)")
 
+    # --- Step 9: persist consolidated results to SQLite ---------------------
+    # Sync the blocklist ledger and close out the scan record regardless of
+    # whether the router sync was attempted.
+    _sync_blocklist_to_db(
+        firewall is not None, blocked_ipv4, blocked_ipv6,
+        added_by=("scan" if db_scan_id is None else f"scan:{db_scan_id}"))
+    if db_scan_id is not None:
+        try:
+            from ..db import get_connection
+            from ..db import repositories as repo
+            conn = get_connection()
+            result_payload = {
+                "ok": True,
+                "cp_candidates": cp_candidates,
+                "npm_candidates": npm_candidates,
+                "blocked_ips": len(final_blocked_ips),
+                "blocked_targets": len(final_blocked_targets),
+                "firewall_connected": firewall is not None,
+            }
+            repo.finish_scan(
+                conn, db_scan_id, status="done",
+                result=result_payload, log_files=0,
+                request_count=(cp_analyzer and len(cp_analyzer.requests) or 0)
+                + (npm_analyzer and len(npm_analyzer.requests) or 0),
+                error="; ".join(errors) or None,
+            )
+        except Exception as exc:
+            print(f"WARN: could not finalize scan in db ({exc})")
+
     return ScanResult(
         ok=True,
         cp_exists=cp_exists,