feat: Implement SQLite schema and migrations for intrusion detection system

- Added schema definition and migration logic in app/db/schema.py.
- Created tables for log sources, requests, IP stats, scans, blocklist, whitelist, and firewall rules.
- Introduced functions to apply migrations and ensure database integrity.

feat: Enhance scan functionality with database integration

- Added _sync_blocklist_to_db function to synchronize blocklist with SQLite.
- Updated run_scan function to handle database connections and scan records.
- Implemented error handling for database operations during scans.

feat: Introduce log analysis page and API endpoints

- Created analysis.html template for displaying log analysis results.
- Added routes for fetching top IPs, IP request timelines, and cross-source IPs.
- Integrated analysis data into the dashboard and status endpoints.

fix: Ensure whitelists are synchronized with the database

- Implemented sync_whitelists_to_db function to persist in-memory whitelists.
- Updated constants.py to define the path for the SQLite database.

refactor: Modularize analysis helpers for better maintainability

- Created _analysis_helpers function to encapsulate SQLite query logic.
- Updated routes.py to utilize new analysis helper functions for API responses.

app/analyzer.py

@@ -60,11 +60,66 @@ class Analyzer:
         self.flagged: dict[str, IPStats] = {}
         self.per_file: dict[str, int] = {}
 
+    # --- sqlite persistence ------------------------------------------------
+
+    def persist_to_db(self, scan_id: int | None = None) -> None:
+        """Ingest this analyzer's requests + per-IP stats into SQLite.
+
+        Lazily imported to avoid cycles. When ``scan_id`` is None a standalone
+        scan record is created for this analyzer so individual
+        (non-consolidated) runs are still tracked. Never raises: a DB failure
+        logs a warning but does not break the scan.
+        """
+        if not self.requests and not self.stats:
+            return
+        try:
+            from .db import get_connection
+            from .db import repositories as repo
+            from .utils.ip_utils import ip_version as _ip_ver
+            conn = get_connection()
+            if scan_id is None:
+                scan_id = repo.start_scan(conn, self.name,
+                                          params={"full": self.full})
+            source = self.name
+            rows: list[dict] = []
+            for r in self.requests:
+                rows.append({
+                    "source": source, "scan_id": scan_id,
+                    "log_file": ((r.extra or {}).get("log_file")
+                                  or (self.files[0] if self.files else None)),
+                    "ts": r.ts, "ip": r.ip,
+                    "version": _ip_ver(r.ip) or 0,
+                    "method": r.method, "path": r.path,
+                    "status": r.status,
+                    "upstream_status": r.upstream_status,
+                    "host": r.host, "user": r.user, "ua": r.ua,
+                    "referer": r.referer, "denied": r.denied,
+                    "banned": r.banned, "is_error": r.is_error,
+                    "extra": r.extra,
+                })
+            repo.insert_requests(conn, rows)
+            repo.insert_ip_stats(conn, scan_id,
+                                 list(self.stats.values()),
+                                 set(self.block_candidates()))
+            repo.set_scan_source_counts(
+                conn, scan_id, source, len(self.requests), len(self.files))
+            repo.finish_scan(conn, scan_id, status="done",
+                             log_files=len(self.files),
+                             request_count=len(self.requests))
+        except Exception as exc:  # never break the scan because of DB issues
+            if not self.quiet:
+                print(f"WARN: sqlite ingest skipped ({exc})")
     # --- hooks -------------------------------------------------------------
 
     def parse_line(self, line: str) -> Request | None:
         raise NotImplementedError
 
+    def attr_source(self, req: Request, log_file: str) -> None:
+        """Record the originating log file onto a parsed request."""
+        if req.extra is None:
+            req.extra = {}
+        req.extra["log_file"] = req.extra.get("log_file", log_file)
+
     def flag(self, stats: dict[str, IPStats]) -> dict[str, IPStats]:
         raise NotImplementedError
 
@@ -111,6 +166,7 @@ class Analyzer:
                 for line in read_lines(full_path):
                     req = self.parse_line(line)
                     if req is not None:
+                        self.attr_source(req, name)
                         self.requests.append(req)
                         count += 1
             else:
@@ -125,6 +181,7 @@ class Analyzer:
                 for line, resume in read_lines_from(full_path, start):
                     req = self.parse_line(line)
                     if req is not None:
+                        self.attr_source(req, name)
                         self.requests.append(req)
                         count += 1
                     last_resume = resume
@@ -262,7 +319,7 @@ class Analyzer:
 
     # --- entrypoint --------------------------------------------------------
 
-    def run(self) -> dict[str, Any]:
+    def run(self, db_scan_id: int | None = None) -> dict[str, Any]:
         validate_dir(self.log_path)
         ensure_dirs(self.report_dir, self.data_dir)
         refresh_whitelists()
@@ -273,6 +330,10 @@ class Analyzer:
         if not self.quiet:
             print(self.text_report())
 
+        # Ingest into the SQLite store (creates its own scan record when no
+        # consolidated ``db_scan_id`` is supplied).
+        self.persist_to_db(scan_id=db_scan_id)
+
         self.write_data()
         report_path = self.write_report()
         print(f"data files:     {self.data_dir}")