intrusion-detection/app/db/backfill.py

#!/usr/bin/env python3
"""R001 — one-time backfill of legacy runtime data into the SQLite store.

Imports existing JSON/CSV/git-history data so the DB becomes the authoritative
source without losing anything already accumulated on a deployed instance:

- cumulative blocklist from the git-history reconstruction (old source of
  truth) -> ``blocklist_entries`` (grow-only ledger)
- firewall snapshots from ``data/mikrotik/*.json`` -> ``firewall_snapshots``
- per-source requests from ``reports/<source>/data/requests.csv`` (best effort)

Idempotent: skips rows already present. Safe to run repeatedly.

Usage:
    python -m app.db.backfill [--dry-run]
"""

from __future__ import annotations

import argparse
import csv
import json
import os
import sys
from datetime import datetime, timezone

from ..utils.constants import (
    APP_REPORT_DATA_DIR,
    DATA_DIR,
    MIKROTIK_DATA_DIR,
    NPM_REPORT_DIR,
    COPYPARTY_REPORT_DIR,
)


def utcnow() -> str:
    return datetime.now(timezone.utc).isoformat(timespec="seconds")


def _read_cidr_lines(path: str) -> set[str]:
    try:
        with open(path, "r", encoding="utf-8") as f:
            return {line.strip() for line in f if line.strip()}
    except OSError:
        return set()


def backfill_blocklist(conn, dry_run: bool = False) -> dict:
    """Import the legacy cumulative blocklist into the ledger."""
    from . import repositories as repo
    from ..utils.blocklist_history import (
        committed_lines, current_file_lines,
    )

    v4_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv4.txt")
    v6_path = os.path.join(APP_REPORT_DATA_DIR, "blocklist_ipv6.txt")
    # Legacy source of truth: union of all git revisions + working file +
    # (there is no earlier scan target to add here).
    complete_v4 = committed_lines(v4_path) | current_file_lines(v4_path)
    complete_v6 = committed_lines(v6_path) | current_file_lines(v6_path)
    cidrs = sorted(complete_v4 | complete_v6)
    if not cidrs:
        return {"blocklist": {"cidrs": 0, "added": 0}}
    if dry_run:
        return {"blocklist": {"cidrs": len(cidrs), "added": None, "dry_run": True}}
    result = repo.record_blocklist(conn, cidrs, added_by="backfill")
    return {"blocklist": result}


def backfill_firewall(conn, dry_run: bool = False) -> dict:
    """Import the latest firewall snapshot from ``data/mikrotik/*.json``."""
    from . import repositories as repo

    def _load(name: str) -> list:
        try:
            with open(os.path.join(MIKROTIK_DATA_DIR, name), "r",
                      encoding="utf-8") as f:
                data = json.load(f)
            return data if isinstance(data, list) else []
        except (OSError, ValueError):
            return []

    addr_v4 = _load("address_list_ipv4.json")
    addr_v6 = _load("address_list_ipv6.json")
    fw_v4 = _load("firewall_ipv4.json")
    fw_v6 = _load("firewall_ipv6.json")
    if not (addr_v4 or addr_v6 or fw_v4 or fw_v6):
        return {"firewall": {"dry_run": dry_run, "snapshot": None}}
    # Idempotent: if a firewall snapshot already exists (previous backfill),
    # don't create another copy of the same latest state.
    n = conn.execute(
        "SELECT COUNT(*) FROM firewall_snapshots").fetchone()[0]
    if n:
        return {"firewall": {"dry_run": dry_run, "snapshot": "exists",
                              "skipped": True}}
    if dry_run:
        return {"firewall": {"dry_run": True, "snapshot": None}}
    sid = repo.save_firewall_snapshot(
        conn, address_lists_v4=addr_v4, address_lists_v6=addr_v6,
        rules_v4=fw_v4, rules_v6=fw_v6)
    return {"firewall": {"snapshot": sid}}


def backfill_requests(conn, dry_run: bool = False) -> dict:
    """Best-effort import of per-source ``requests.csv`` files.

    Only ingests rows that don't already exist (keyed on source+log_file+ts+ip).
    Legacy CSV columns vary by source; we keep a conservative mapping and never
    fail the whole backfill over a single bad file.
    """
    from . import repositories as repo

    seen = set()
    for r in conn.execute(
            "SELECT source, log_file, ts, ip FROM requests"):
        seen.add((r["source"], r["log_file"], r["ts"], r["ip"]))

    candidates = [
        (os.path.join(COPYPARTY_REPORT_DIR, "data", "requests.csv"), "copyparty"),
        (os.path.join(NPM_REPORT_DIR, "data", "requests.csv"), "npm"),
    ]
    total = 0
    for path, source in candidates:
        if not os.path.exists(path):
            continue
        try:
            with open(path, "r", encoding="utf-8", errors="replace") as f:
                reader = csv.DictReader(f)
                rows = []
                for row in reader:
                    ip = (row.get("ip") or "").strip()
                    ts = row.get("ts") or row.get("timestamp") or ""
                    log_file = row.get("log_file") or row.get("file") or ""
                    if not ip:
                        continue
                    key = (source, log_file, ts, ip)
                    if key in seen:
                        continue
                    seen.add(key)
                    rows.append({
                        "source": source, "scan_id": None, "log_file": log_file,
                        "ts": ts, "ip": ip,
                        "version": 4 if ":" not in ip else 6,
                        "method": row.get("method"), "path": row.get("path"),
                        "status": _int_or_none(row.get("status")),
                        "upstream_status": _int_or_none(
                            row.get("upstream_status")),
                        "host": row.get("host"), "user": row.get("user"),
                        "ua": row.get("ua") or row.get("user_agent"),
                        "referer": row.get("referer"),
                        "denied": _bool_str(row.get("denied")),
                        "banned": _bool_str(row.get("banned")),
                        "is_error": _bool_str(row.get("is_error")),
                        "extra": None,
                    })
                    total += 1
            if not dry_run and total:
                repo.insert_requests(conn, rows)
        except Exception as exc:
            print(f"  WARN: requests.csv backfill {source}: {exc}")
    return {"requests": {"imported": total, "dry_run": dry_run}}


def _int_or_none(v):
    try:
        if v in (None, "", "-"):
            return None
        return int(v)
    except (TypeError, ValueError):
        return None


def _bool_str(v):
    if isinstance(v, str):
        return int(v.strip().lower() in ("1", "true", "yes", "t"))
    return int(bool(v))


def main(argv: list[str] | None = None) -> None:
    p = argparse.ArgumentParser(prog="python -m app.db.backfill",
                                description="Backfill legacy runtime data into SQLite.")
    p.add_argument("--dry-run", action="store_true",
                   help="report what would be imported without writing")
    opts = p.parse_args(argv)

    from . import get_connection
    conn = get_connection()
    print(f"== backfill (dry_run={opts.dry_run}) ==")
    print("blocklist:", backfill_blocklist(conn, dry_run=opts.dry_run))
    print("firewall :", backfill_firewall(conn, dry_run=opts.dry_run))
    print("requests :", backfill_requests(conn, dry_run=opts.dry_run))
    print("== done ==")


if __name__ == "__main__":
    sys.exit(main())