Restructure sources into app package with unified entry point and update documentation

app/__init__.py

@@ -0,0 +1 @@
+# Intrusion Detection System — app package

app/analyzer.py

@@ -0,0 +1,254 @@
+#!/usr/bin/env python3
+"""Analysis pipeline for the intrusion-detection system.
+
+`Analyzer` is the log-source-agnostic base class.  Subclasses implement
+`parse_line()` and `flag()`.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections import Counter
+from datetime import datetime
+from typing import Any, Sequence
+
+from .files import ensure_dirs, find_logs, read_lines, validate_dir, write_csv, write_json, write_lines
+from .flags import by_total, flag_malicious_patterns
+from .ip_utils import is_whitelisted, split_versions
+from .mikrotik import write_mikrotik
+from .models import IPStats, Request, aggregate
+from .reporting import Column, columns_header, columns_row, render_kv, render_table, timestamp_name
+from .whitelists import refresh_whitelists
+
+
+class Analyzer:
+    """Log-source-agnostic analysis pipeline.
+
+    Subclasses must set the class attributes below and implement
+    `parse_line()` and `flag()`.
+    """
+
+    name = "ids"                       # short id, used in file names / tags
+    title = "IDS log analysis report"
+    log_pattern: re.Pattern | None = None
+    log_suffixes: tuple[str, ...] = ()
+    exclude_prefixes: tuple[str, ...] = ()
+    ip_columns: Sequence[Column] = ()
+    request_columns: Sequence[Column] = ()
+    paths_file = "error_paths.csv"
+    paths_label = "error paths"
+    flag_files = ("flagged_ips.txt", "flagged_ipv6.txt")
+    flag_label = "Flagged IPs"
+    flag_note = ""
+
+    def __init__(self, log_path: str, report_dir: str, top: int = 30,
+                 quiet: bool = False, write_rsc: bool = True):
+        self.log_path = log_path
+        self.report_dir = report_dir
+        self.data_dir = os.path.join(report_dir, "data")
+        self.top = top
+        self.quiet = quiet
+        self.write_rsc = write_rsc
+        self.files: list[str] = []
+        self.requests: list[Request] = []
+        self.stats: dict[str, IPStats] = {}
+        self.flagged: dict[str, IPStats] = {}
+        self.per_file: dict[str, int] = {}
+
+    # --- hooks -------------------------------------------------------------
+
+    def parse_line(self, line: str) -> Request | None:
+        raise NotImplementedError
+
+    def flag(self, stats: dict[str, IPStats]) -> dict[str, IPStats]:
+        raise NotImplementedError
+
+    def flag_comment(self, s: IPStats) -> str:
+        return (f"errors={s.errors} total={s.total} ratio={s.error_ratio} "
+                f"first={s.first} last={s.last}")
+
+    def block_candidates(self) -> set[str]:
+        malicious = flag_malicious_patterns(self.stats)
+        return set(self.flagged) | set(malicious)
+
+    def extra_summary_rows(self) -> list[tuple[str, Any]]:
+        return []
+
+    def write_extra(self, data_dir: str) -> list[str]:
+        return []
+
+    # --- pipeline ----------------------------------------------------------
+
+    def discover(self) -> list[str]:
+        self.files = find_logs(self.log_path, self.log_pattern,
+                               self.log_suffixes, self.exclude_prefixes)
+        if not self.files:
+            raise FileNotFoundError(f"no log files found in: {self.log_path}")
+        return self.files
+
+    def load(self) -> list[Request]:
+        for name in self.files:
+            count = 0
+            for line in read_lines(os.path.join(self.log_path, name)):
+                req = self.parse_line(line)
+                if req is not None:
+                    self.requests.append(req)
+                    count += 1
+            self.per_file[name] = count
+        return self.requests
+
+    def analyze(self) -> dict[str, IPStats]:
+        self.stats = aggregate(self.requests)
+        self.flagged = self.flag(self.stats)
+        return self.stats
+
+    # --- derived data ------------------------------------------------------
+
+    def error_paths(self) -> Counter:
+        return Counter(r.path for r in self.requests if r.is_error and r.path)
+
+    def flagged_split(self) -> tuple[list[IPStats], list[IPStats]]:
+        v4 = [s for s in self.flagged.values() if s.version == 4]
+        v6 = [s for s in self.flagged.values() if s.version == 6]
+        def key(s): return (-s.errors, s.ip)
+        return sorted(v4, key=key), sorted(v6, key=key)
+
+    def summary_rows(self) -> list[tuple[str, Any]]:
+        v4, v6 = split_versions(self.stats)
+        rows = [
+            ("log path", self.log_path),
+            ("log files", len(self.files)),
+            ("total requests", len(self.requests)),
+            ("unique IPs",
+             f"{len(self.stats)} (IPv4 {len(v4)} / IPv6 {len(v6)})"),
+            ("error requests", sum(1 for r in self.requests if r.is_error)),
+            (f"unique {self.paths_label}", len(self.error_paths())),
+        ]
+        rows += self.extra_summary_rows()
+        rows += [
+            (self.flag_label.lower(), len(self.flagged)),
+            ("whitelisted IPs seen",
+             sum(1 for ip in self.stats if is_whitelisted(ip))),
+            ("block candidates", len(self.block_candidates())),
+        ]
+        return rows
+
+    # --- reporting ---------------------------------------------------------
+
+    def text_report(self) -> str:
+        out = [self.title,
+               f"generated: {datetime.now().isoformat(timespec='seconds')}",
+               "",
+               render_kv(self.summary_rows()), ""]
+
+        out.append(f"== top {self.top} IPs ==")
+        rows = [columns_row(self.ip_columns, s)
+                for s in by_total(self.stats)[:self.top]]
+        out.append(render_table(columns_header(self.ip_columns), rows))
+        if len(self.stats) > self.top:
+            out.append(f"... {len(self.stats) - self.top} more IPs omitted")
+        out.append("")
+
+        out.append(f"== top 20 {self.paths_label} ==")
+        for path, n in self.error_paths().most_common(20):
+            out.append(f"  {n:>5}  {path}")
+        out.append("")
+
+        v4, v6 = self.flagged_split()
+        out.append(f"== {self.flag_label} ({len(self.flagged)}) ==")
+        if self.flag_note:
+            out.append(self.flag_note)
+        for label, group in (("IPv4", v4), ("IPv6", v6)):
+            if not group:
+                continue
+            out.append(f"-- {label} ({len(group)}) --")
+            rows = [columns_row(self.ip_columns, s) for s in group]
+            out.append(render_table(columns_header(self.ip_columns), rows))
+        out.append("")
+        return "\n".join(out)
+
+    # --- writing -----------------------------------------------------------
+
+    def write_data(self) -> list[str]:
+        ensure_dirs(self.report_dir, self.data_dir)
+        written: list[str] = []
+        d = self.data_dir
+        header = columns_header(self.ip_columns)
+        ordered = by_total(self.stats)
+
+        for version, fname in ((4, "ips_ipv4.csv"), (6, "ips_ipv6.csv")):
+            rows = [columns_row(self.ip_columns, s)
+                    for s in ordered if s.version == version]
+            written.append(write_csv(os.path.join(d, fname), header, rows))
+
+        written.append(write_json(
+            os.path.join(d, "ips.json"),
+            {s.ip: s.to_dict() for s in ordered}))
+
+        written.append(write_csv(
+            os.path.join(d, self.paths_file), ["path", "count"],
+            self.error_paths().most_common()))
+
+        v4, v6 = self.flagged_split()
+        for group, fname in zip((v4, v6), self.flag_files):
+            written.append(write_lines(
+                os.path.join(d, fname),
+                [f"{s.ip}  # {self.flag_comment(s)}" for s in group]))
+
+        blocked_v4, blocked_v6 = split_versions(
+            ip for ip in self.block_candidates() if not is_whitelisted(ip))
+        written.append(write_lines(
+            os.path.join(d, "blocked_ipv4.txt"), blocked_v4))
+        written.append(write_lines(
+            os.path.join(d, "blocked_ipv6.txt"), blocked_v6))
+
+        written.append(write_csv(
+            os.path.join(d, "requests.csv"),
+            columns_header(self.request_columns),
+            (columns_row(self.request_columns, r) for r in self.requests)))
+
+        written += self.write_extra(d)
+        return written
+
+    def write_report(self) -> str:
+        ensure_dirs(self.report_dir)
+        path = os.path.join(self.report_dir, f"{timestamp_name()}.txt")
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(self.text_report())
+        return path
+
+    def write_rsc_file(self) -> tuple[str, int, int]:
+        ensure_dirs(self.data_dir)
+        return write_mikrotik(
+            os.path.join(self.data_dir, "mikrotik-block.rsc"),
+            f"{self.name}-ids", self.block_candidates())
+
+    # --- entrypoint --------------------------------------------------------
+
+    def run(self) -> dict[str, Any]:
+        validate_dir(self.log_path)
+        ensure_dirs(self.report_dir, self.data_dir)
+        refresh_whitelists()
+        self.discover()
+        self.load()
+        self.analyze()
+
+        if not self.quiet:
+            print(self.text_report())
+
+        self.write_data()
+        report_path = self.write_report()
+        print(f"data files:     {self.data_dir}")
+        print(f"summary report: {report_path}")
+
+        result: dict[str, Any] = {"report": report_path,
+                                  "data_dir": self.data_dir}
+        if self.write_rsc:
+            rsc, n_blocked, n_whitelisted = self.write_rsc_file()
+            print(f"mikrotik script: {rsc}")
+            print(f"  {n_blocked} IPs blocked, "
+                  f"{n_whitelisted} CIDR ranges whitelisted")
+            result.update(rsc=rsc, blocked=n_blocked,
+                          whitelisted=n_whitelisted)
+        return result

app/bad_patterns.py

@@ -0,0 +1,190 @@
+#!/usr/bin/env python3
+"""Malicious-pattern detection rules for the intrusion-detection system.
+
+Defines compiled regex patterns and classifiers that identify
+probing/scanning URLs (e.g. env files, wp-admin, webshells) and
+suspicious file names.  Imported by analyzers to flag IPs hitting
+these patterns as blocklist candidates.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+from typing import ClassVar, Pattern
+
+# ---------------------------------------------------------------------------
+# Pattern categories
+# ---------------------------------------------------------------------------
+
+
+@dataclass(frozen=True)
+class BadPattern:
+    """A single detection rule.
+
+    Attributes:
+        name: Short identifier (e.g. "env_file", "wp_admin").
+        label: Human-readable description for reports.
+        regex: Compiled regex matched against the request path.
+        severity: 1 = moderate, 2 = high, 3 = critical (auto-block).
+    """
+
+    name: str
+    label: str
+    regex: Pattern[str]
+    severity: int = 2
+
+    def matches(self, path: str) -> bool:
+        return bool(self.regex.search(path))
+
+
+# ---------------------------------------------------------------------------
+# Compiled pattern bank
+# ---------------------------------------------------------------------------
+
+# -- env / config file probes -------------------------------------------------
+
+ENV_FILE = BadPattern(
+    "env_file",
+    "env/config file probe (.env, .aws/credentials, config.json)",
+    re.compile(
+        r"/\.(?:env|config)(?:\.\w+)*|"
+        r"/\.aws/credentials|"
+        r"/config\.json",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- version control probes ---------------------------------------------------
+
+VCS_PROBE = BadPattern(
+    "vcs_probe",
+    "version-control probe (.git, .svn)",
+    re.compile(
+        r"/\.(?:git|svn)(?:/|$)",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- WordPress / CMS probes ---------------------------------------------------
+
+WP_PROBE = BadPattern(
+    "wp_probe",
+    "WordPress/CMS probe (wp-admin, wp-login, wp-content, wp-includes)",
+    re.compile(
+        r"/wp-(?:admin|login|content|includes|load|cron|xmlrpc)(?:/|\.php)",
+        re.IGNORECASE,
+    ),
+    severity=2,
+)
+
+# -- well-known / common probe paths ------------------------------------------
+
+WELL_KNOWN_PROBE = BadPattern(
+    "well_known_probe",
+    ".well-known probe",
+    re.compile(
+        r"/\.well-known/",
+        re.IGNORECASE,
+    ),
+    severity=1,
+)
+
+# -- webshell / backdoor probes -----------------------------------------------
+
+WEBSHELL_PROBE = BadPattern(
+    "webshell_probe",
+    "webshell/backdoor probe (known shell filenames, phpinfo, file managers)",
+    re.compile(
+        r"/(?:"
+        r"phpinfo|info|shell|cmd|upload|"
+        r"wp_filemanager|filemanager|"
+        r"xstelth|adminfuns|hellopress|"
+        r"fpwch|crgio|coffexium|"
+        r"ccou|biufile|"
+        r"ms-edit|classwithtostring|"
+        r"this_is_a_new_hello_world|"
+        r"chosen|ops|simple|inputs"
+        r")\.php",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- suspicious generic PHP file requests -------------------------------------
+
+GENERIC_PHP_PROBE = BadPattern(
+    "generic_php_probe",
+    "generic/random PHP file probe (single-char, numeric names)",
+    re.compile(
+        r"/(?:"
+        r"[a-z]\.php|"  # single letter: a.php, k.php, x.php
+        r"\d{1,3}\.php|"  # numeric: 1.php, 222.php, 155.php
+        r"[a-z]{2}\.php|"  # two-letter: aa.php, dx.php
+        r"a\d\.php|"  # a1.php
+        r"\w+\.php"  # generic: anything.php
+        r")",
+        re.IGNORECASE,
+    ),
+    severity=1,
+)
+
+# -- sensitive file probes ----------------------------------------------------
+
+SENSITIVE_FILE = BadPattern(
+    "sensitive_file",
+    "sensitive file probe (id_rsa, id_dsa, .pem, .key)",
+    re.compile(
+        r"/(?:id_rsa|id_dsa|id_ecdsa|id_ed25519|.*\.(?:pem|key|ppk|pfx|p12))",
+        re.IGNORECASE,
+    ),
+    severity=3,
+)
+
+# -- suspicious user-agent (optional, path-based only for now) -----------------
+
+
+# ---------------------------------------------------------------------------
+# All active patterns (ordered by severity, highest first)
+# ---------------------------------------------------------------------------
+
+ALL_PATTERNS: list[BadPattern] = [
+    ENV_FILE,
+    VCS_PROBE,
+    SENSITIVE_FILE,
+    WEBSHELL_PROBE,
+    WP_PROBE,
+    GENERIC_PHP_PROBE,
+    WELL_KNOWN_PROBE,
+]
+
+# Quick lookup by name
+PATTERNS_BY_NAME: dict[str, BadPattern] = {p.name: p for p in ALL_PATTERNS}
+
+
+# ---------------------------------------------------------------------------
+# Classification API
+# ---------------------------------------------------------------------------
+
+
+def classify_path(path: str) -> list[BadPattern]:
+    """Return all patterns that match a given request path."""
+    return [p for p in ALL_PATTERNS if p.matches(path)]
+
+
+def is_malicious(path: str) -> bool:
+    """Return True if any pattern matches the path."""
+    return any(p.matches(path) for p in ALL_PATTERNS)
+
+
+def malicious_severity(path: str) -> int:
+    """Return the highest severity among matching patterns, or 0 if none."""
+    matched = classify_path(path)
+    return max((p.severity for p in matched), default=0)
+
+
+def pattern_names(path: str) -> list[str]:
+    """Return names of all patterns matching the path."""
+    return [p.name for p in classify_path(path)]

app/copyparty.py

@@ -0,0 +1,180 @@
+#!/usr/bin/env python3
+"""copyparty access-log analyzer.
+
+Parses copyparty server logs, aggregates per-IP behaviour and flags
+"denied-only" clients: IPs that never authenticated and only ever hit
+`@* has no access to '...'`. Those plus copyparty's own auto-bans become
+the MikroTik block list.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+
+from .analyzer import Analyzer
+from .ip_utils import split_versions
+from .files import write_json, write_lines
+from .flags import flag_denied_only
+from .models import IPStats, Request
+from .utils import arg_parser
+
+
+DEFAULT_LOG_PATH = os.environ.get("COPYPARTY_LOG_PATH", r"P:\copyparty")
+DEFAULT_REPORT_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "reports", "copyparty")
+
+# "HH:MM:SS.mmm  <ip>  <port>  LEVEL: message"
+# System lines ("hsrv", "root", ...) have no port and are skipped.
+LINE_RE = re.compile(
+    r"^(\d{2}:\d{2}:\d{2}\.\d{3})\s+"                 # timestamp
+    r"(\S+)\s+"                                        # ip
+    r"(\d+)\s+"                                        # port
+    r"(LOG|WARN|CRIT|BTW|INFO|ERR|DEBUG):\s"           # level
+    r"(.*)$"                                           # message
+)
+
+METHOD_RE = re.compile(
+    r"^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS|PROPFIND|PROPPATCH|MKCOL|"
+    r"MOVE|COPY|LOCK|UNLOCK)\s+(\S+)(?:\s+@(\S+))?")
+DENY_RE = re.compile(r"^@(\S+) has no access to '([^']+)'")
+CLIENT_BAN_RE = re.compile(r"client banned")
+BAN_SECS_RE = re.compile(r"banned for (\d+) sec")
+
+
+class CopypartyAnalyzer(Analyzer):
+    name = "copyparty"
+    title = "copyparty log analysis report"
+    log_pattern = re.compile(r"\d{4}-\d{2}-\d{2}\.txt(\.\d+)?$")
+    paths_file = "denied_paths.csv"
+    paths_label = "denied paths"
+    flag_files = ("denied_ips.txt", "denied_ipv6.txt")
+    flag_label = "Denied-only IPs"
+    flag_note = "never authenticated -- ban candidates"
+
+    ip_columns = (
+        ("ip", lambda s: s.ip),
+        ("total", lambda s: s.total),
+        ("denied", lambda s: s.denied),
+        ("banned", lambda s: s.banned),
+        ("first", lambda s: s.first),
+        ("last", lambda s: s.last),
+        ("users", lambda s: s.users_str),
+        ("whitelist", lambda s: s.whitelist),
+    )
+
+    request_columns = (
+        ("time", lambda r: r.ts),
+        ("ip", lambda r: r.ip),
+        ("port", lambda r: r.extra.get("port")),
+        ("level", lambda r: r.extra.get("level")),
+        ("method", lambda r: r.method),
+        ("path", lambda r: r.path),
+        ("user", lambda r: r.user),
+        ("denied", lambda r: r.denied),
+        ("banned", lambda r: r.banned),
+        ("ban_secs", lambda r: r.extra.get("ban_secs")),
+    )
+
+    def __init__(self, *args, min_denied: int = 1, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.min_denied = min_denied
+
+    # --- parsing -----------------------------------------------------------
+
+    def parse_line(self, line: str) -> Request | None:
+        if not line:
+            return None
+        m = LINE_RE.match(line)
+        if not m:
+            return None
+        ts, ip, port, level, msg = m.groups()
+        req = Request(ts=ts, ip=ip, user=None,
+                      extra={"port": int(port), "level": level})
+
+        mm = METHOD_RE.match(msg)
+        if mm:
+            req.method, req.path, user = mm.groups()
+            req.user = user or "*"
+            return req
+
+        dm = DENY_RE.match(msg)
+        if dm:
+            req.user, req.path = dm.groups()
+            req.denied = True
+            return req
+
+        if CLIENT_BAN_RE.search(msg):
+            req.banned = True
+            bs = BAN_SECS_RE.search(msg)
+            if bs:
+                req.extra["ban_secs"] = int(bs.group(1))
+            return req
+
+        bs = BAN_SECS_RE.search(msg)
+        if bs:
+            req.banned = True
+            req.extra["ban_secs"] = int(bs.group(1))
+        return req
+
+    # --- flagging ----------------------------------------------------------
+
+    def flag(self, stats):
+        return flag_denied_only(stats, min_denied=self.min_denied)
+
+    def flag_comment(self, s):
+        return (f"denied={s.denied} total={s.total} "
+                f"first={s.first} last={s.last}")
+
+    def block_candidates(self) -> set[str]:
+        banned = {ip for ip, s in self.stats.items() if s.banned}
+        return set(self.flagged) | banned
+
+    # --- extras ------------------------------------------------------------
+
+    def banned_ips(self) -> dict[str, IPStats]:
+        return {ip: s for ip, s in self.stats.items() if s.banned}
+
+    def authed_users(self) -> dict[str, int]:
+        counts: dict[str, int] = {}
+        for r in self.requests:
+            if r.user and r.user != "*":
+                counts[r.user] = counts.get(r.user, 0) + 1
+        return dict(sorted(counts.items(), key=lambda kv: -kv[1]))
+
+    def extra_summary_rows(self):
+        return [("denied requests",
+                 sum(1 for r in self.requests if r.denied)),
+                ("banned IPs", len(self.banned_ips())),
+                ("authenticated users", self.authed_users() or "{}")]
+
+    def write_extra(self, data_dir: str) -> list[str]:
+        v4, v6 = split_versions(self.banned_ips())
+        return [
+            write_lines(os.path.join(data_dir, "banned_ips.txt"), v4),
+            write_lines(os.path.join(data_dir, "banned_ipv6.txt"), v6),
+            write_json(os.path.join(data_dir, "authed_users.json"),
+                       self.authed_users()),
+        ]
+
+
+def main() -> None:
+    parser = arg_parser(
+        "analyze copyparty logs, flag denied-only IPs, export MikroTik rules",
+        DEFAULT_LOG_PATH, DEFAULT_REPORT_DIR)
+    parser.add_argument("--min-denied", type=int, default=1,
+                        help="minimum denied hits before flagging (default: 1)")
+    args = parser.parse_args()
+
+    CopypartyAnalyzer(
+        log_path=args.log_path,
+        report_dir=args.report_dir,
+        top=args.top,
+        quiet=args.quiet,
+        write_rsc=not args.no_rsc,
+        min_denied=args.min_denied,
+    ).run()
+
+
+if __name__ == "__main__":
+    main()

app/files.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Filesystem and I/O helpers for the intrusion-detection system."""
+
+from __future__ import annotations
+
+import csv
+import gzip
+import json
+import os
+import re
+from typing import Any, Iterable, Iterator, Sequence
+
+ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
+
+
+# -- directory --------------------------------------------------------------
+
+def validate_dir(path: str, need_write: bool = False) -> str:
+    if not os.path.exists(path):
+        raise FileNotFoundError(f"path does not exist: {path}")
+    if not os.path.isdir(path):
+        raise NotADirectoryError(f"path is not a directory: {path}")
+    if not os.access(path, os.R_OK):
+        raise PermissionError(f"path not readable: {path}")
+    if need_write and not os.access(path, os.W_OK):
+        raise PermissionError(f"path not writable: {path}")
+    return path
+
+
+def ensure_dirs(*paths: str) -> None:
+    for p in paths:
+        os.makedirs(p, exist_ok=True)
+
+
+# -- log discovery ----------------------------------------------------------
+
+def find_logs(path: str, pattern: re.Pattern | None = None,
+              suffixes: Sequence[str] = (),
+              exclude_prefixes: Sequence[str] = ()) -> list[str]:
+    """List matching log file names (not full paths), sorted."""
+    out = []
+    for name in os.listdir(path):
+        if not os.path.isfile(os.path.join(path, name)):
+            continue
+        if any(name.startswith(p) for p in exclude_prefixes):
+            continue
+        if pattern is not None and not pattern.match(name):
+            continue
+        if suffixes and not name.endswith(tuple(suffixes)):
+            continue
+        out.append(name)
+    return sorted(out)
+
+
+def read_lines(path: str) -> Iterator[str]:
+    """Yield ANSI-stripped, newline-stripped lines. Transparently gunzips."""
+    opener = gzip.open if path.endswith(".gz") else open
+    with opener(path, "rt", encoding="utf-8", errors="replace") as f:
+        for line in f:
+            yield ANSI_RE.sub("", line.rstrip("\r\n"))
+
+
+# -- writers ----------------------------------------------------------------
+
+def write_csv(path: str, header: Sequence[str],
+              rows: Iterable[Sequence[Any]]) -> str:
+    with open(path, "w", newline="", encoding="utf-8") as f:
+        w = csv.writer(f)
+        w.writerow(header)
+        w.writerows(rows)
+    return path
+
+
+def write_json(path: str, obj: Any) -> str:
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(obj, f, indent=2, default=str)
+    return path
+
+
+def write_lines(path: str, lines: Iterable[str]) -> str:
+    with open(path, "w", encoding="utf-8") as f:
+        for line in lines:
+            f.write(line + "\n")
+    return path

app/firewall.py

@@ -0,0 +1,321 @@
+"""
+firewall.py -
+Module for interacting with the Mikrotik routeros shell to manage firewall rules and IP address lists.
+Provides functions to read the current firewall configuration, add and remove IPs from the blocklist.
+Reads the latest report from the IDS scan, reads the current firewall configuration, and updates the router's blocklist accordingly.
+"""
+
+from __future__ import annotations
+
+import os
+import json
+import paramiko
+from datetime import datetime
+import shlex
+from typing import Iterable
+from .ip_utils import is_whitelisted
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.dirname(
+    os.path.abspath(__file__))), "data", "mikrotik")
+REPORT_DIR = os.path.join(os.path.dirname(
+    os.path.dirname(os.path.abspath(__file__))), "reports")
+MIKROTIK_DATA_DIR = os.path.join(REPORT_DIR, "mikrotik", "data")
+
+SSH_TIMEOUT = 10  # seconds
+SSH_PORT = 22
+ROUTER_IP = os.environ.get("ROUTER_IP", "192.168.88.1")
+ROUTER_USER = os.environ.get("ROUTER_USER", "admin")
+ROUTER_PASSWORD = os.environ.get("ROUTER_PASSWORD", "M11ffpgm")
+BLOCKLIST_NAME = os.environ.get("BLOCKLIST_NAME", "Blocked")
+WHITELIST_NAMES = os.environ.get(
+    "WHITELIST_NAMES", "Cloudflare,Openfactory").split(",")
+
+
+class MikrotikFirewall:
+    """
+    Class to manage Mikrotik firewall rules and IP address lists.
+    Provides methods to read the current blocklist, add/remove IPs, and update the blocklist based on IDS scan reports.
+    """
+
+    def __init__(self, router_ip: str = ROUTER_IP, username: str = ROUTER_USER,
+                 password: str = ROUTER_PASSWORD):
+        self.router_ip = router_ip
+        self.username = username
+        self.password = password
+        self.client: paramiko.SSHClient | None = None
+
+    def connect(self) -> None:
+        """Establish an SSH connection to the Mikrotik router."""
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            # RouterOS 7.20.x exposes an RSA host key and classic SSH ciphers.
+            client.connect(
+                self.router_ip,
+                port=SSH_PORT,
+                username=self.username,
+                password=self.password,
+                timeout=SSH_TIMEOUT,
+                banner_timeout=SSH_TIMEOUT,
+                auth_timeout=SSH_TIMEOUT,
+                allow_agent=False,
+                look_for_keys=False,
+                compress=False,
+                disabled_algorithms={"pubkeys": ["rsa-sha2-512"]},
+            )
+        except Exception:
+            client.close()
+            raise
+        self.client = client
+
+    def disconnect(self) -> None:
+        """Close the SSH connection."""
+        if self.client:
+            self.client.close()
+            self.client = None
+
+    def execute_command(self, command: str) -> str:
+        """Execute a command on the Mikrotik router via SSH and return the output."""
+        if not self.client:
+            raise RuntimeError("SSH client is not connected.")
+        stdin, stdout, stderr = self.client.exec_command(command)
+        output = stdout.read().decode("utf-8")
+        error = stderr.read().decode("utf-8")
+        if error:
+            raise RuntimeError(f"Error executing command '{command}': {error}")
+        return output
+
+    def get_address_list_ipv4(self, name: str) -> set[str]:
+        """Retrieve the current IPv4 address list from the Mikrotik router."""
+        output = self.execute_command("/ip firewall address-list print")
+        ips = set()
+        for line in output.splitlines():
+            parts = line.split()
+            if len(parts) >= 3 and parts[1] == "list" and parts[2] == name:
+                ip = parts[0]
+                ips.add(ip)
+        return ips
+
+    def get_address_list_ipv6(self, name: str) -> set[str]:
+        """Retrieve the current IPv6 address list from the Mikrotik router."""
+        output = self.execute_command("/ipv6 firewall address-list print")
+        ips = set()
+        for line in output.splitlines():
+            parts = line.split()
+            if len(parts) >= 3 and parts[1] == "list" and parts[2] == name:
+                ip = parts[0]
+                ips.add(ip)
+        return ips
+
+    @staticmethod
+    def _parse_terse_output(output: str) -> list[dict[str, str]]:
+        """Parse RouterOS `print terse` records into key/value dictionaries."""
+        records = []
+        for line in output.splitlines():
+            fields = {}
+            for field in shlex.split(line):
+                key, separator, value = field.partition("=")
+                if separator:
+                    fields[key] = value
+            if fields:
+                records.append(fields)
+        return records
+
+    def get_address_lists(self, ipv6: bool = False) -> list[dict[str, str]]:
+        """Retrieve all address-list entries in structured form."""
+        path = "/ipv6" if ipv6 else "/ip"
+        output = self.execute_command(
+            f"{path} firewall address-list print terse")
+        return self._parse_terse_output(output)
+
+    def get_firewall_rules(self, ipv6: bool = False) -> list[dict[str, str]]:
+        """Retrieve all firewall filter rules in structured form."""
+        path = "/ipv6" if ipv6 else "/ip"
+        output = self.execute_command(
+            f"{path} firewall filter print terse")
+        return self._parse_terse_output(output)
+
+    def save_state_report(self, report_dir: str = REPORT_DIR,
+                          changes: dict | None = None) -> str:
+        """Save current address lists and firewall rules for auditing."""
+        state = {
+            "router": self.router_ip,
+            "collected_at": datetime.now().astimezone().isoformat(),
+            "address_lists": {
+                "ipv4": self.get_address_lists(),
+                "ipv6": self.get_address_lists(ipv6=True),
+            },
+            "firewall_rules": {
+                "ipv4": self.get_firewall_rules(),
+                "ipv6": self.get_firewall_rules(ipv6=True),
+            },
+        }
+        if changes is not None:
+            state["changes"] = changes
+        data_dir = os.path.join(report_dir, "mikrotik", "data")
+        os.makedirs(data_dir, exist_ok=True)
+        for version in ("ipv4", "ipv6"):
+            with open(os.path.join(data_dir, f"address_list_{version}.json"), "w", encoding="utf-8") as file:
+                json.dump(state["address_lists"][version],
+                          file, indent=2, sort_keys=True)
+            with open(os.path.join(data_dir, f"firewall_{version}.json"), "w", encoding="utf-8") as file:
+                json.dump(state["firewall_rules"][version],
+                          file, indent=2, sort_keys=True)
+        timestamp = datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
+        report_path = os.path.join(report_dir, "mikrotik", f"{timestamp}.json")
+        with open(report_path, "w", encoding="utf-8") as file:
+            json.dump(state, file, indent=2, sort_keys=True)
+        return report_path
+
+    def sync_blocklist(self, blocked_ipv4: Iterable[str],
+                       blocked_ipv6: Iterable[str],
+                       dry_run: bool = False,
+                       report_dir: str = REPORT_DIR) -> dict[str, list[str]]:
+        """Synchronize the Blocked list with scan targets and audit the result."""
+        desired_ipv4 = set(blocked_ipv4)
+        desired_ipv6 = set(blocked_ipv6)
+        current_ipv4 = self.get_address_list_ipv4(BLOCKLIST_NAME)
+        current_ipv6 = self.get_address_list_ipv6(BLOCKLIST_NAME)
+        whitelisted_ipv4 = set()
+        whitelisted_ipv6 = set()
+        for name in WHITELIST_NAMES:
+            whitelisted_ipv4.update(self.get_address_list_ipv4(name))
+            whitelisted_ipv6.update(self.get_address_list_ipv6(name))
+
+        changes = {
+            "added_ipv4": sorted(desired_ipv4 - current_ipv4 - whitelisted_ipv4),
+            "added_ipv6": sorted(desired_ipv6 - current_ipv6 - whitelisted_ipv6),
+            "removed_ipv4": sorted(current_ipv4 - desired_ipv4),
+            "removed_ipv6": sorted(current_ipv6 - desired_ipv6),
+            "dry_run": dry_run,
+        }
+        if not dry_run:
+            for ip in changes["removed_ipv4"]:
+                self.execute_command(
+                    f"/ip firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+            for ip in changes["removed_ipv6"]:
+                self.execute_command(
+                    f"/ipv6 firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+            self.write_address_list_ipv4(BLOCKLIST_NAME, changes["added_ipv4"])
+            self.write_address_list_ipv6(BLOCKLIST_NAME, changes["added_ipv6"])
+        self.save_state_report(report_dir, changes)
+        return changes
+
+    def write_address_list_ipv4(self, name: str, ips: Iterable[str]) -> None:
+        """Write a list of IPv4 addresses to the Mikrotik router's address list."""
+        for ip in ips:
+            if is_whitelisted(ip):
+                continue  # Skip whitelisted IPs
+            self.execute_command(
+                f"/ip firewall address-list add list={name} address={ip}")
+
+    def write_address_list_ipv6(self, name: str, ips: Iterable[str]) -> None:
+        """Write a list of IPv6 addresses to the Mikrotik router's address list."""
+        for ip in ips:
+            if is_whitelisted(ip):
+                continue  # Skip whitelisted IPs
+            self.execute_command(
+                f"/ipv6 firewall address-list add list={name} address={ip}")
+
+
+def get_scan_report(report_dir: str = REPORT_DIR) -> dict[str, list[str]]:
+    """Read the latest scan report `mikrotik-block.rsc` from the report directory and return the list of blocked IPs."""
+    report_path = os.path.join(report_dir, "mikrotik-block.rsc")
+    if not os.path.exists(report_path):
+        raise FileNotFoundError(f"Scan report not found: {report_path}")
+
+    # add sections, they start with `/ip firewall address-list` or `/ipv6 firewall address-list` and end with a blank line or EOF
+    sections = {
+        "/ip firewall address-list": [],
+        "/ipv6 firewall address-list": [],
+    }
+    # add list=Cloudflare or list=Openfactory to whitelist sections, list=Blocked to blocked section
+    with open(report_path, "r", encoding="utf-8") as f:
+        current_section = None
+        for line in f:
+            line = line.strip()
+            if not line:
+                current_section = None
+                continue
+            if line.startswith("/ip firewall address-list"):
+                current_section = "/ip firewall address-list"
+                continue
+            if line.startswith("/ipv6 firewall address-list"):
+                current_section = "/ipv6 firewall address-list"
+                continue
+            if current_section:
+                sections[current_section].append(line)
+
+    return sections
+
+
+def main():
+    """Legacy entry point for applying the latest generated RouterOS report."""
+    # Read the latest scan report
+    scan_report = get_scan_report(REPORT_DIR)
+
+    # Connect to the Mikrotik router
+    firewall = MikrotikFirewall()
+    firewall.connect()
+
+    # read current blocked IPs from the router
+    current_blocked_ipv4 = firewall.get_address_list_ipv4(BLOCKLIST_NAME)
+    current_blocked_ipv6 = firewall.get_address_list_ipv6(BLOCKLIST_NAME)
+
+    # read whitelisted IPs from the router
+    whitelisted_ipv4 = set()
+    whitelisted_ipv6 = set()
+    for wl_name in WHITELIST_NAMES:
+        whitelisted_ipv4.update(firewall.get_address_list_ipv4(wl_name))
+        whitelisted_ipv6.update(firewall.get_address_list_ipv6(wl_name))
+
+    # read blocked IPs from the scan report
+    blocked_ipv4 = set()
+    blocked_ipv6 = set()
+    for line in scan_report["/ip firewall address-list"]:
+        if "list=Blocked" in line:
+            parts = line.split()
+            for part in parts:
+                if part.startswith("address="):
+                    ip = part.split("=")[1]
+                    blocked_ipv4.add(ip)
+    for line in scan_report["/ipv6 firewall address-list"]:
+        if "list=Blocked" in line:
+            parts = line.split()
+            for part in parts:
+                if part.startswith("address="):
+                    ip = part.split("=")[1]
+                    blocked_ipv6.add(ip)
+
+    # Determine which IPs to add and remove
+    to_add_ipv4 = blocked_ipv4 - current_blocked_ipv4 - whitelisted_ipv4
+    to_add_ipv6 = blocked_ipv6 - current_blocked_ipv6 - whitelisted_ipv6
+    to_remove_ipv4 = current_blocked_ipv4 - blocked_ipv4
+    to_remove_ipv6 = current_blocked_ipv6 - blocked_ipv6
+
+    # Remove old blocked IPs
+    for ip in to_remove_ipv4:
+        firewall.execute_command(
+            f"/ip firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+    for ip in to_remove_ipv6:
+        firewall.execute_command(
+            f"/ipv6 firewall address-list remove [find list={BLOCKLIST_NAME} address={ip}]")
+
+    # Add new blocked IPs
+    firewall.write_address_list_ipv4(BLOCKLIST_NAME, to_add_ipv4)
+    firewall.write_address_list_ipv6(BLOCKLIST_NAME, to_add_ipv6)
+
+    firewall.save_state_report(REPORT_DIR)
+
+    # Disconnect from the router
+    firewall.disconnect()
+
+    # Print summary
+    print(
+        f"Removed {len(to_remove_ipv4)} IPv4 and {len(to_remove_ipv6)} IPv6 addresses from the blocklist.")
+    print(
+        f"Added {len(to_add_ipv4)} IPv4 and {len(to_add_ipv6)} IPv6 addresses to the blocklist.")
+
+
+if __name__ == "__main__":
+    main()

app/flags.py

@@ -0,0 +1,46 @@
+#!/usr/bin/env python3
+"""Flagging rules for the intrusion-detection system.
+
+Detection functions that identify hostile IPs based on aggregated stats.
+"""
+
+from __future__ import annotations
+
+from . import bad_patterns
+from .models import IPStats
+
+
+def flag_scanners(stats: dict[str, IPStats], min_errors: int = 5,
+                  min_ratio: float = 0.5) -> dict[str, IPStats]:
+    """IPs with many errors AND a high error ratio."""
+    return {
+        ip: s for ip, s in stats.items()
+        if s.errors >= min_errors and s.error_ratio >= min_ratio
+    }
+
+
+def flag_denied_only(stats: dict[str, IPStats],
+                     min_denied: int = 1) -> dict[str, IPStats]:
+    """IPs that only ever hit denied paths and never authenticated."""
+    return {
+        ip: s for ip, s in stats.items()
+        if s.denied >= min_denied and not s.authenticated
+    }
+
+
+def flag_malicious_patterns(stats: dict[str, IPStats],
+                            min_severity: int = 2,
+                            min_hits: int = 1) -> dict[str, IPStats]:
+    """IPs with hits against malicious path patterns at or above severity threshold."""
+    return {
+        ip: s for ip, s in stats.items()
+        if s.malicious_hits >= min_hits and any(
+            bad_patterns.PATTERNS_BY_NAME[name].severity >= min_severity
+            for name in s.malicious_patterns
+        )
+    }
+
+
+def by_total(stats: dict[str, IPStats]) -> list[IPStats]:
+    """Stats sorted by request count, descending."""
+    return sorted(stats.values(), key=lambda s: (-s.total, s.ip))

app/ip_utils.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""IP helper functions for the intrusion-detection system."""
+
+from __future__ import annotations
+
+import ipaddress
+from typing import Iterable
+
+from .whitelists import WHITELIST_NETS as _WHITELIST_NETS
+
+
+def parse_ip(value: str):
+    """Return an ip_address object, or None if `value` is not an IP."""
+    try:
+        return ipaddress.ip_address(value)
+    except (ValueError, TypeError):
+        return None
+
+
+def ip_version(value: str) -> int | None:
+    addr = parse_ip(value)
+    return addr.version if addr else None
+
+
+def is_ipv6(value: str) -> bool:
+    return ip_version(value) == 6
+
+
+def is_ipv4(value: str) -> bool:
+    return ip_version(value) == 4
+
+
+def whitelist_of(value: str) -> str | None:
+    """Return the whitelist name an IP belongs to, else None."""
+    addr = parse_ip(value)
+    if addr is None:
+        return None
+    for name, nets in _WHITELIST_NETS.items():
+        for net in nets:
+            if addr.version == net.version and addr in net:
+                return name
+    return None
+
+
+def is_whitelisted(value: str) -> bool:
+    return whitelist_of(value) is not None
+
+
+def sort_ips(ips: Iterable[str]) -> list[str]:
+    """Numeric sort, IPv4 before IPv6, unparseable last."""
+    def key(ip: str):
+        addr = parse_ip(ip)
+        if addr is None:
+            return (9, b"", ip)
+        return (addr.version, addr.packed, "")
+    return sorted(ips, key=key)
+
+
+def split_versions(ips: Iterable[str]) -> tuple[list[str], list[str]]:
+    """Split an IP iterable into (sorted IPv4, sorted IPv6)."""
+    ips = list(ips)
+    return (sort_ips(ip for ip in ips if is_ipv4(ip)),
+            sort_ips(ip for ip in ips if is_ipv6(ip)))

app/mikrotik.py

@@ -0,0 +1,110 @@
+#!/usr/bin/env python3
+"""MikroTik RouterOS script generation."""
+
+from __future__ import annotations
+
+import ipaddress
+from datetime import datetime
+from typing import Iterable
+
+from .ip_utils import is_ipv4, is_ipv6, is_whitelisted, parse_ip
+from .whitelists import WHITELISTS
+
+
+def split_cidr_versions(targets: Iterable[str]) -> tuple[list[str], list[str]]:
+    """Split target CIDRs/IPs into (sorted IPv4 list, sorted IPv6 list)."""
+    v4 = []
+    v6 = []
+    for t in targets:
+        ip = t.split("/")[0]
+        if is_ipv4(ip):
+            v4.append(t)
+        elif is_ipv6(ip):
+            v6.append(t)
+
+    def key_cidr(cidr: str):
+        ip = cidr.split("/")[0]
+        addr = parse_ip(ip)
+        if addr is None:
+            return (9, b"", cidr)
+        return (addr.version, addr.packed, cidr)
+
+    return sorted(v4, key=key_cidr), sorted(v6, key=key_cidr)
+
+
+def mikrotik_script(tag: str, blocked_ips_or_subnets: Iterable[str],
+                    whitelists: dict[str, list[str]] | None = None,
+                    generated: str | None = None) -> tuple[str, int, int]:
+    """Build a RouterOS .rsc script.
+
+    Whitelisted ranges become `accept` rules placed above the `drop` rules.
+    Blocked targets can be individual IPs (e.g. 1.2.3.4/32) or CIDR blocks
+    (e.g. 1.2.3.0/24).
+
+    Returns (script_text, blocked_count, whitelisted_cidr_count).
+    """
+    whitelists = WHITELISTS if whitelists is None else whitelists
+    generated = generated or datetime.now().isoformat(timespec="seconds")
+
+    def clean_blocked(targets: Iterable[str]) -> list[str]:
+        cleaned = []
+        for t in targets:
+            ip = t.split("/")[0]
+            if not is_whitelisted(ip):
+                cleaned.append(t)
+        return cleaned
+
+    blocked = clean_blocked(set(blocked_ips_or_subnets))
+    block_v4, block_v6 = split_cidr_versions(blocked)
+
+    lines: list[str] = [f"# {tag} firewall export - generated {generated}", ""]
+    wl_count = 0
+
+    for name, cidrs in whitelists.items():
+        nets = [ipaddress.ip_network(c) for c in cidrs]
+        wl_count += len(nets)
+        for version, cmd in ((4, "/ip firewall address-list"),
+                             (6, "/ipv6 firewall address-list")):
+            chunk = [n for n in nets if n.version == version]
+            if not chunk:
+                continue
+            lines.append(f"# {name} IPv{version} ranges (whitelisted)")
+            lines.append(cmd)
+            lines += [f"add list={name} address={n}" for n in chunk]
+            lines.append("")
+
+    if block_v4:
+        lines.append(f"# Blocked IPv4 ({len(block_v4)}, non-whitelisted)")
+        lines.append("/ip firewall address-list")
+        lines += [f"add list=Blocked address={t}" for t in block_v4]
+        lines.append("")
+    if block_v6:
+        lines.append(f"# Blocked IPv6 ({len(block_v6)}, non-whitelisted)")
+        lines.append("/ipv6 firewall address-list")
+        lines += [f"add list=Blocked address={t}" for t in block_v6]
+        lines.append("")
+
+    for version, cmd, blocks in ((4, "/ip firewall filter", block_v4),
+                                 (6, "/ipv6 firewall filter", block_v6)):
+        lines.append(cmd)
+        for name in whitelists:
+            lines.append(
+                f"add chain=input action=accept src-address-list={name} "
+                f'comment="{tag} {name.lower()} whitelist (v{version})"')
+        if blocks:
+            for chain in ("input", "forward"):
+                lines.append(
+                    f"add chain={chain} action=drop src-address-list=Blocked "
+                    f'comment="{tag} blocked (v{version})"')
+        lines.append("")
+
+    return "\n".join(lines), len(blocked), wl_count
+
+
+def write_mikrotik(path: str, tag: str,
+                   blocked_ips_or_subnets: Iterable[str]) -> tuple[str, int, int]:
+    text, n_blocked, n_whitelisted = mikrotik_script(
+        tag, blocked_ips_or_subnets)
+    with open(path, "w", encoding="utf-8") as f:
+        f.write(text)
+    return path, n_blocked, n_whitelisted

app/models.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Data models for the intrusion-detection system.
+
+`Request` and `IPStats` plus the `aggregate()` fold function.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Iterable
+
+from . import bad_patterns
+from .ip_utils import ip_version, whitelist_of
+
+
+@dataclass(slots=True)
+class Request:
+    """One normalised request line, produced by any log parser."""
+
+    ts: str = ""
+    ip: str = ""
+    method: str | None = None
+    path: str | None = None
+    status: int | None = None
+    upstream_status: int | None = None
+    host: str | None = None
+    user: str | None = None
+    ua: str | None = None
+    referer: str | None = None
+    denied: bool = False
+    banned: bool = False
+    extra: dict = field(default_factory=dict)
+
+    @property
+    def is_client_error(self) -> bool:
+        return self.status is not None and 400 <= self.status < 500
+
+    @property
+    def is_server_error(self) -> bool:
+        return self.status is not None and 500 <= self.status < 600
+
+    @property
+    def is_error(self) -> bool:
+        return self.denied or self.is_client_error or self.is_server_error
+
+
+@dataclass
+class IPStats:
+    """Aggregated per-IP counters."""
+
+    ip: str
+    version: int = 0
+    total: int = 0
+    errors: int = 0
+    c4xx: int = 0
+    c5xx: int = 0
+    denied: int = 0
+    banned: int = 0
+    first: str = ""
+    last: str = ""
+    paths: Counter = field(default_factory=Counter)
+    methods: Counter = field(default_factory=Counter)
+    hosts: Counter = field(default_factory=Counter)
+    uas: Counter = field(default_factory=Counter)
+    users: set[str] = field(default_factory=set)
+    malicious_hits: int = 0
+    malicious_patterns: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def error_ratio(self) -> float:
+        return round(self.errors / self.total, 3) if self.total else 0.0
+
+    @property
+    def authenticated(self) -> bool:
+        """True if the IP ever presented a real (non-anonymous) user."""
+        return any(u and u != "*" for u in self.users)
+
+    @property
+    def whitelist(self) -> str:
+        return whitelist_of(self.ip) or ""
+
+    @property
+    def users_str(self) -> str:
+        return ",".join(sorted(self.users)) or "-"
+
+    @property
+    def methods_str(self) -> str:
+        return ",".join(sorted(self.methods)) or "-"
+
+    def to_dict(self) -> dict:
+        return {
+            "ip": self.ip,
+            "version": self.version,
+            "total": self.total,
+            "errors": self.errors,
+            "error_ratio": self.error_ratio,
+            "status_4xx": self.c4xx,
+            "status_5xx": self.c5xx,
+            "denied": self.denied,
+            "banned": self.banned,
+            "first": self.first,
+            "last": self.last,
+            "whitelist": self.whitelist or None,
+            "users": sorted(self.users),
+            "methods": dict(self.methods),
+            "hosts": dict(self.hosts),
+            "paths": dict(self.paths.most_common()),
+            "uas": dict(self.uas.most_common(10)),
+            "malicious_hits": self.malicious_hits,
+            "malicious_patterns": dict(self.malicious_patterns),
+        }
+
+
+def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]:
+    """Fold request records into per-IP statistics."""
+    stats: dict[str, IPStats] = {}
+    for r in requests:
+        s = stats.get(r.ip)
+        if s is None:
+            s = stats[r.ip] = IPStats(
+                ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts)
+        s.total += 1
+        s.last = r.ts
+        if r.denied:
+            s.denied += 1
+        if r.banned:
+            s.banned += 1
+        if r.is_client_error:
+            s.c4xx += 1
+        if r.is_server_error:
+            s.c5xx += 1
+        if r.is_error:
+            s.errors += 1
+        if r.path:
+            s.paths[r.path] += 1
+        if r.method:
+            s.methods[r.method] += 1
+        if r.host:
+            s.hosts[r.host] += 1
+        if r.ua and r.ua != "-":
+            s.uas[r.ua] += 1
+        if r.user:
+            s.users.add(r.user)
+        if r.path and bad_patterns.is_malicious(r.path):
+            s.malicious_hits += 1
+            for p in bad_patterns.classify_path(r.path):
+                s.malicious_patterns[p.name] = s.malicious_patterns.get(
+                    p.name, 0) + 1
+    return stats

app/npm.py

@@ -0,0 +1,167 @@
+#!/usr/bin/env python3
+"""Nginx Proxy Manager access-log analyzer.
+
+Parses NPM proxy access logs (plain + gzipped), aggregates per-IP
+behaviour and flags scanner IPs: clients with many error responses and a
+high error ratio. Flagged IPs become the MikroTik block list.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+from collections import Counter
+
+from .analyzer import Analyzer
+from .files import write_csv
+from .flags import flag_scanners
+from .models import Request
+from .utils import arg_parser
+
+
+DEFAULT_LOG_PATH = os.environ.get("NPM_LOG_PATH", r"Y:\npm\logs")
+DEFAULT_REPORT_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "reports", "npm")
+
+# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path"
+# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer"
+LINE_RE = re.compile(
+    r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+"  # [datetime]
+    r"-\s+"
+    r"(\d+|-)\s+"                                                  # upstream
+    r"(\d+)\s+"                                                    # status
+    r"-\s+"
+    r"(\S+)\s+"                                                    # method
+    r"(\S+)\s+"                                                    # scheme
+    r"(\S+)\s+"                                                    # host
+    r'"([^"]*)"\s+'                                                # "path"
+    r"\[Client\s+(\S+)\]\s+"
+    r"\[Length\s+(\d+|-)\]\s+"
+    r"\[Gzip\s+(\S+)\]\s+"
+    r"\[Sent-to\s+(\S+)\]\s+"
+    r'"([^"]*)"\s+'                                                # "UA"
+    r'"([^"]*)"'                                                   # "Referer"
+)
+
+
+class NpmAnalyzer(Analyzer):
+    name = "npm"
+    title = "Nginx Proxy Manager log analysis report"
+    log_suffixes = (".log", ".gz")
+    exclude_prefixes = ("letsencrypt",)
+    paths_file = "error_paths.csv"
+    paths_label = "error paths"
+    flag_files = ("scanner_ips.txt", "scanner_ipv6.txt")
+    flag_label = "Scanner IPs"
+    flag_note = "high error rate -- probable scanners/attackers"
+
+    ip_columns = (
+        ("ip", lambda s: s.ip),
+        ("total", lambda s: s.total),
+        ("4xx", lambda s: s.c4xx),
+        ("5xx", lambda s: s.c5xx),
+        ("ratio", lambda s: s.error_ratio),
+        ("first", lambda s: s.first),
+        ("last", lambda s: s.last),
+        ("methods", lambda s: s.methods_str),
+        ("whitelist", lambda s: s.whitelist),
+    )
+
+    request_columns = (
+        ("datetime", lambda r: r.ts),
+        ("ip", lambda r: r.ip),
+        ("method", lambda r: r.method),
+        ("scheme", lambda r: r.extra.get("scheme")),
+        ("host", lambda r: r.host),
+        ("path", lambda r: r.path),
+        ("status", lambda r: r.status),
+        ("upstream_status", lambda r: r.upstream_status),
+        ("length", lambda r: r.extra.get("length")),
+        ("gzip", lambda r: r.extra.get("gzip")),
+        ("sent_to", lambda r: r.extra.get("sent_to")),
+        ("referer", lambda r: r.referer),
+        ("ua", lambda r: r.ua),
+    )
+
+    def __init__(self, *args, min_errors: int = 5,
+                 error_ratio: float = 0.5, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.min_errors = min_errors
+        self.min_ratio = error_ratio
+
+    # --- parsing -----------------------------------------------------------
+
+    def parse_line(self, line: str) -> Request | None:
+        if not line:
+            return None
+        m = LINE_RE.match(line)
+        if not m:
+            return None
+        (ts, upstream, status, method, scheme, host, path, ip,
+         length, gz, sent_to, ua, referer) = m.groups()
+        return Request(
+            ts=ts,
+            ip=ip,
+            method=method,
+            path=path,
+            status=int(status),
+            upstream_status=int(upstream) if upstream.isdigit() else None,
+            host=host,
+            ua=ua,
+            referer=referer,
+            extra={"scheme": scheme, "length": length,
+                   "gzip": gz, "sent_to": sent_to},
+        )
+
+    # --- flagging ----------------------------------------------------------
+
+    def flag(self, stats):
+        return flag_scanners(stats, min_errors=self.min_errors,
+                             min_ratio=self.min_ratio)
+
+    # --- extras ------------------------------------------------------------
+
+    def extra_summary_rows(self):
+        c4 = sum(1 for r in self.requests if r.is_client_error)
+        c5 = sum(1 for r in self.requests if r.is_server_error)
+        return [
+            ("4xx / 5xx", f"{c4} / {c5}"),
+            ("hosts seen", len({r.host for r in self.requests if r.host})),
+            ("scanner threshold",
+             f">= {self.min_errors} errors and ratio >= {self.min_ratio}"),
+        ]
+
+    def write_extra(self, data_dir: str) -> list[str]:
+        uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-")
+        hosts = Counter(r.host for r in self.requests if r.host)
+        return [
+            write_csv(os.path.join(data_dir, "ua_summary.csv"),
+                      ["user_agent", "count"], uas.most_common()),
+            write_csv(os.path.join(data_dir, "host_summary.csv"),
+                      ["host", "count"], hosts.most_common()),
+        ]
+
+
+def main() -> None:
+    parser = arg_parser(
+        "analyze NPM proxy logs, flag scanner IPs, export MikroTik rules",
+        DEFAULT_LOG_PATH, DEFAULT_REPORT_DIR)
+    parser.add_argument("--min-errors", type=int, default=5,
+                        help="minimum error responses to flag (default: 5)")
+    parser.add_argument("--error-ratio", type=float, default=0.5,
+                        help="minimum error ratio to flag (default: 0.5)")
+    args = parser.parse_args()
+
+    NpmAnalyzer(
+        log_path=args.log_path,
+        report_dir=args.report_dir,
+        top=args.top,
+        quiet=args.quiet,
+        write_rsc=not args.no_rsc,
+        min_errors=args.min_errors,
+        error_ratio=args.error_ratio,
+    ).run()
+
+
+if __name__ == "__main__":
+    main()

app/reporting.py

@@ -0,0 +1,55 @@
+#!/usr/bin/env python3
+"""Text rendering helpers for the intrusion-detection system."""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any, Callable, Sequence
+
+#: (header, getter) pair used for CSV + console tables
+Column = tuple[str, Callable[[Any], Any]]
+
+
+def columns_header(columns: Sequence[Column]) -> list[str]:
+    return [c[0] for c in columns]
+
+
+def columns_row(columns: Sequence[Column], obj: Any) -> list[Any]:
+    return [fn(obj) for _, fn in columns]
+
+
+def render_table(headers: Sequence[str], rows: Sequence[Sequence[Any]],
+                 max_width: int = 48) -> str:
+    """Aligned fixed-width text table. Numeric columns right-aligned."""
+    def cell(v: Any) -> str:
+        s = "" if v is None else str(v)
+        return s if len(s) <= max_width else s[:max_width - 1] + "~"
+
+    body = [[cell(v) for v in row] for row in rows]
+    aligns = []
+    for i in range(len(headers)):
+        numeric = bool(rows) and all(
+            isinstance(row[i], (int, float)) and not isinstance(row[i], bool)
+            for row in rows)
+        aligns.append(">" if numeric else "<")
+    widths = [len(h) for h in headers]
+    for row in body:
+        for i, v in enumerate(row):
+            widths[i] = max(widths[i], len(v))
+
+    def line(values: Sequence[str]) -> str:
+        return "  ".join(
+            f"{v:{a}{w}}" for v, a, w in zip(values, aligns, widths)).rstrip()
+
+    out = [line(list(headers)), "-" * (sum(widths) + 2 * (len(widths) - 1))]
+    out += [line(row) for row in body]
+    return "\n".join(out)
+
+
+def render_kv(rows: Sequence[tuple[str, Any]]) -> str:
+    width = max((len(k) for k, _ in rows), default=0) + 1
+    return "\n".join(f"{k + ':':<{width + 1}} {v}" for k, v in rows)
+
+
+def timestamp_name() -> str:
+    return datetime.now().strftime("%Y-%m-%d-%H-%M-%S")

app/scan.py

@@ -0,0 +1,149 @@
+#!/usr/bin/env python3
+"""Consolidated intrusion detection log scanner.
+
+Runs both copyparty and Nginx Proxy Manager (NPM) log analyzers,
+and outputs a single consolidated MikroTik RouterOS blocklist script.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+from datetime import datetime
+
+from . import utils
+from .firewall import MikrotikFirewall
+from .copyparty import CopypartyAnalyzer, DEFAULT_LOG_PATH as CP_DEFAULT_LOG, DEFAULT_REPORT_DIR as CP_DEFAULT_REPORT
+from .npm import NpmAnalyzer, DEFAULT_LOG_PATH as NPM_DEFAULT_LOG, DEFAULT_REPORT_DIR as NPM_DEFAULT_REPORT
+from .ip_utils import is_whitelisted, split_versions
+from .whois import consolidate_by_subnet
+
+DEFAULT_CONSOLIDATED_REPORT_DIR = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "reports")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Consolidated IDS Scanner: run copyparty + NPM analyzers and generate unified blocklist."
+    )
+
+    # copyparty arguments
+    parser.add_argument("--copyparty-log-path", default=CP_DEFAULT_LOG,
+                        help=f"copyparty log directory (default: {CP_DEFAULT_LOG})")
+    parser.add_argument("--copyparty-report-dir", default=CP_DEFAULT_REPORT,
+                        help=f"copyparty report directory (default: {CP_DEFAULT_REPORT})")
+    parser.add_argument("--min-denied", type=int, default=1,
+                        help="minimum copyparty denied hits before flagging (default: 1)")
+
+    # NPM arguments
+    parser.add_argument("--npm-log-path", default=NPM_DEFAULT_LOG,
+                        help=f"NPM log directory (default: {NPM_DEFAULT_LOG})")
+    parser.add_argument("--npm-report-dir", default=NPM_DEFAULT_REPORT,
+                        help=f"NPM report directory (default: {NPM_DEFAULT_REPORT})")
+    parser.add_argument("--min-errors", type=int, default=5,
+                        help="minimum NPM error responses to flag (default: 5)")
+    parser.add_argument("--error-ratio", type=float, default=0.5,
+                        help="minimum NPM error ratio to flag (default: 0.5)")
+
+    # Shared arguments
+    parser.add_argument("--report-dir", default=DEFAULT_CONSOLIDATED_REPORT_DIR,
+                        help=f"consolidated reports directory (default: {DEFAULT_CONSOLIDATED_REPORT_DIR})")
+    parser.add_argument("--top", type=int, default=30,
+                        help="rows in the top-IP tables (default: 30)")
+    parser.add_argument("--quiet", action="store_true",
+                        help="write files, suppress console reports")
+    parser.add_argument("--update-firewall", action="store_true",
+                        help="synchronize the MikroTik Blocked list after scanning")
+    parser.add_argument("--dry-run", action="store_true",
+                        help="preview firewall changes without applying them")
+
+    args = parser.parse_args()
+
+    # 1. Initialize Analyzers
+    # We turn off inner write_rsc of individual analyzers to avoid duplicate/redundant rsc writes,
+    # or let them write their individual ones if they want to. Writing individual scripts is harmless.
+    # We'll match the individual tools' default write behavior, but generate the consolidated one.
+    cp_analyzer = CopypartyAnalyzer(
+        log_path=args.copyparty_log_path,
+        report_dir=args.copyparty_report_dir,
+        top=args.top,
+        quiet=args.quiet,
+        write_rsc=False,
+        min_denied=args.min_denied,
+    )
+
+    npm_analyzer = NpmAnalyzer(
+        log_path=args.npm_log_path,
+        report_dir=args.npm_report_dir,
+        top=args.top,
+        quiet=args.quiet,
+        write_rsc=False,
+        min_errors=args.min_errors,
+        error_ratio=args.error_ratio,
+    )
+
+    # 2. Run copyparty analysis
+    print("=== Running copyparty scan ===")
+    cp_exists = True
+    try:
+        cp_analyzer.run()
+    except Exception as e:
+        print(f"Error running copyparty scan: {e}")
+        cp_exists = False
+
+    # 3. Run NPM analysis
+    print("\n=== Running NPM scan ===")
+    npm_exists = True
+    try:
+        npm_analyzer.run()
+    except Exception as e:
+        print(f"Error running NPM scan: {e}")
+        npm_exists = False
+
+    if not cp_exists and not npm_exists:
+        print("\nBoth scans failed. Unable to generate consolidated blocklist.")
+        return
+
+    # 4. Consolidate block candidates
+    consolidated_candidates: set[str] = set()
+    if cp_exists:
+        consolidated_candidates.update(cp_analyzer.block_candidates())
+    if npm_exists:
+        consolidated_candidates.update(npm_analyzer.block_candidates())
+
+    # Exclude whitelisted IPs
+    final_blocked_ips = {
+        ip for ip in consolidated_candidates if not is_whitelisted(ip)}
+
+    # Group into subnets via WHOIS if multiple malicious IPs in same subnet (threshold = 3)
+    print("\nGrouping blocked IPs into subnets using WHOIS...")
+    final_blocked_targets = consolidate_by_subnet(
+        final_blocked_ips, threshold=3)
+    blocked_ipv4, blocked_ipv6 = split_versions(final_blocked_targets)
+
+    print("\n=== Consolidated Summary ===")
+    print(
+        f"copyparty block candidates: {len(cp_analyzer.block_candidates()) if cp_exists else 0}")
+    print(
+        f"NPM block candidates:       {len(npm_analyzer.block_candidates()) if npm_exists else 0}")
+    print(f"Total unified blocked IPs:  {len(final_blocked_ips)}")
+    print(f"Total target blocks/subnets: {len(final_blocked_targets)}")
+
+    if args.update_firewall:
+        firewall = MikrotikFirewall()
+        firewall.connect()
+        try:
+            changes = firewall.sync_blocklist(
+                blocked_ipv4, blocked_ipv6, dry_run=args.dry_run,
+                report_dir=args.report_dir)
+        finally:
+            firewall.disconnect()
+        action = "Would" if args.dry_run else ""
+        print(f"{action} add {len(changes['added_ipv4'])} IPv4 and "
+              f"{len(changes['added_ipv6'])} IPv6 addresses; "
+              f"remove {len(changes['removed_ipv4'])} IPv4 and "
+              f"{len(changes['removed_ipv6'])} IPv6 addresses.")
+
+
+if __name__ == "__main__":
+    main()

app/utils.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+"""Shared core for the local-dev intrusion-detection scripts.
+
+`copyparty.py` and `npm.py` are thin adapters. They only provide:
+
+  * a log-file matcher (glob-ish pattern / suffixes)
+  * a line parser producing `Request` objects
+  * a flagging rule (which IPs look hostile)
+  * column definitions for their CSV output
+
+Everything else -- IP aggregation, whitelisting, console + file reports,
+MikroTik RouterOS export -- lives in this module.
+"""
+
+from __future__ import annotations
+
+import argparse
+
+
+def arg_parser(description: str, default_log_path: str,
+               default_report_dir: str) -> argparse.ArgumentParser:
+    p = argparse.ArgumentParser(description=description)
+    p.add_argument("--log-path", default=default_log_path,
+                   help=f"log directory (default: {default_log_path})")
+    p.add_argument("--report-dir", default=default_report_dir,
+                   help=f"report output dir (default: {default_report_dir})")
+    p.add_argument("--top", type=int, default=30,
+                   help="rows in the top-IP table (default: 30)")
+    p.add_argument("--quiet", action="store_true",
+                   help="write files, print nothing but the paths")
+    p.add_argument("--no-rsc", action="store_true",
+                   help="skip MikroTik .rsc generation")
+    return p

app/whitelists.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Whitelist management for the intrusion-detection system.
+
+Owns provider CIDR constants, remote download/refresh logic, and
+CIDR extraction/validation.  Imported by `utils.py` which provides
+the convenience lookups `whitelist_of()` and `is_whitelisted()`.
+
+Whitelist loading order (per provider):
+  1. Local file in `data/<provider>/` (if present and readable)
+  2. Remote download (if network available)
+  3. Built-in hardcoded fallback
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import os
+import re
+from typing import Any
+
+# ---------------------------------------------------------------------------
+# Paths
+# ---------------------------------------------------------------------------
+
+_HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+_DATA_DIR = os.path.join(_HERE, "data")
+
+# ---------------------------------------------------------------------------
+# CIDR extraction regex
+# ---------------------------------------------------------------------------
+
+CIDR_RE = re.compile(
+    r"(?<![0-9A-Fa-f:.])([0-9A-Fa-f:.]+/\d{1,3})(?![0-9A-Fa-f:.])")
+
+# ---------------------------------------------------------------------------
+# Provider URLs
+# ---------------------------------------------------------------------------
+
+OPENFACTORY_URL = "https://openfactory.net/geofeed.csv"
+CLOUDFLARE_URL_V4 = "https://www.cloudflare.com/ips-v4"
+CLOUDFLARE_URL_V6 = "https://www.cloudflare.com/ips-v6"
+PALOALTO_URL = "https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity"
+
+# ---------------------------------------------------------------------------
+# Local file paths (checked first, before remote download)
+# ---------------------------------------------------------------------------
+
+OPENFACTORY_FILE = os.path.join(_DATA_DIR, "openfactory", "geofeed.csv")
+CLOUDFLARE_V4_FILE = os.path.join(_DATA_DIR, "cloudflare", "ips-v4.txt")
+CLOUDFLARE_V6_FILE = os.path.join(_DATA_DIR, "cloudflare", "ips-v6.txt")
+PALOALTO_FILE = os.path.join(_DATA_DIR, "paloalto", "scanner-ips.txt")
+
+# ---------------------------------------------------------------------------
+# Built-in fallback CIDR lists
+# ---------------------------------------------------------------------------
+
+#: Openfactory AS58299 colocation IP range (https://openfactory.net/geofeed.csv)
+OPENFACTORY_IPV4 = ["5.226.144.0/21"]
+
+#: https://www.cloudflare.com/ips-v4
+CLOUDFLARE_IPV4 = [
+    "173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22",
+    "103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18",
+    "190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22",
+    "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13",
+    "104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
+]
+
+#: https://www.cloudflare.com/ips-v6
+CLOUDFLARE_IPV6 = [
+    "2400:cb00::/32", "2606:4700::/32", "2803:f800::/32",
+    "2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29",
+    "2c0f:f248::/32",
+]
+
+#: https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity
+PALOALTO_IPV4 = [
+    "35.203.210.0/23", "144.86.173.0/24", "147.185.132.0/23",
+    "162.216.149.0/24", "162.216.150.0/24", "172.105.147.0/24",
+    "198.235.24.0/24", "205.210.31.0/24", "216.25.88.0/21",
+]
+
+PALOALTO_IPV6 = [
+    "2604:a940:300:5b6::/64",
+    "2604:a940:301:225::/64",
+    "2604:a940:302:118::/64",
+]
+
+# ---------------------------------------------------------------------------
+# Active whitelist state (mutated by refresh)
+# ---------------------------------------------------------------------------
+
+#: address-list name -> CIDR ranges (v4 + v6 mixed, split at render time)
+WHITELISTS: dict[str, list[str]] = {
+    "Openfactory": OPENFACTORY_IPV4,
+    "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
+    "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
+}
+
+#: pre-parsed networks, so matching does not re-parse CIDRs per IP
+WHITELIST_NETS: dict[str, tuple[Any, ...]] = {
+    name: tuple(ipaddress.ip_network(c) for c in cidrs)
+    for name, cidrs in WHITELISTS.items()
+}
+
+#: guard so refresh runs at most once per process
+_refreshed = False
+
+
+# ---------------------------------------------------------------------------
+# CIDR extraction / validation
+# ---------------------------------------------------------------------------
+
+def extract_cidrs(text: str) -> list[str]:
+    """Extract valid CIDR strings from plain text, CSV, or HTML."""
+    seen: set[str] = set()
+    cidrs: list[str] = []
+    for match in CIDR_RE.finditer(text):
+        candidate = match.group(1).strip()
+        try:
+            network = ipaddress.ip_network(candidate, strict=False)
+        except ValueError:
+            continue
+        rendered = str(network)
+        if rendered not in seen:
+            seen.add(rendered)
+            cidrs.append(rendered)
+    return cidrs
+
+
+# ---------------------------------------------------------------------------
+# Local file loading
+# ---------------------------------------------------------------------------
+
+def _read_local_file(path: str) -> list[str]:
+    """Read a local whitelist file and extract valid CIDRs."""
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            return extract_cidrs(f.read())
+    except Exception:
+        return []
+
+
+def _load_local_whitelists() -> dict[str, list[str]]:
+    """Load whitelist CIDRs from local data/ files.
+
+    Returns a dict of {name: [CIDR strings]}. Only keys with
+    successfully parsed CIDRs are included.
+    """
+    out: dict[str, list[str]] = {}
+    cidrs = _read_local_file(OPENFACTORY_FILE)
+    if cidrs:
+        out["Openfactory"] = cidrs
+    cf_v4 = _read_local_file(CLOUDFLARE_V4_FILE)
+    cf_v6 = _read_local_file(CLOUDFLARE_V6_FILE)
+    if cf_v4 or cf_v6:
+        out["Cloudflare"] = cf_v4 + cf_v6
+    cidrs = _read_local_file(PALOALTO_FILE)
+    if cidrs:
+        out["PaloAlto"] = cidrs
+    return out
+
+
+# ---------------------------------------------------------------------------
+# HTTP download helpers
+# ---------------------------------------------------------------------------
+
+def _download_text(url: str) -> str:
+    """Download a text file from a URL, return as string."""
+    import urllib.request
+    request = urllib.request.Request(
+        url,
+        headers={
+            "User-Agent": "Mozilla/5.0 (compatible; intrusion-detection/1.0)",
+            "Accept": "text/plain,text/csv,text/html;q=0.9,*/*;q=0.8",
+        },
+    )
+    with urllib.request.urlopen(request) as response:
+        return response.read().decode("utf-8", errors="replace")
+
+
+def _download_cidrs(url: str) -> list[str]:
+    """Download a whitelist source and extract valid CIDRs from it."""
+    return extract_cidrs(_download_text(url))
+
+
+# ---------------------------------------------------------------------------
+# Refresh pipeline
+# ---------------------------------------------------------------------------
+
+def _download_whitelists() -> dict[str, list[str]]:
+    """Download the latest whitelists from the Internet and save to local files.
+
+    Returns a dict of {name: [CIDR strings]}.  Only keys with
+    successfully parsed CIDRs are included; failures are logged
+    and skipped so built-in fallbacks remain untouched.
+    Saves each download to the corresponding local file on success.
+    """
+    out: dict[str, list[str]] = {}
+    try:
+        text = _download_text(OPENFACTORY_URL)
+        cidrs = extract_cidrs(text)
+        if cidrs:
+            out["Openfactory"] = cidrs
+            _save_local_file(OPENFACTORY_FILE, text)
+    except Exception as e:
+        print(f"Error downloading Openfactory whitelist: {e}")
+    try:
+        text_v4 = _download_text(CLOUDFLARE_URL_V4)
+        text_v6 = _download_text(CLOUDFLARE_URL_V6)
+        cidrs = extract_cidrs(text_v4) + extract_cidrs(text_v6)
+        if cidrs:
+            out["Cloudflare"] = cidrs
+            _save_local_file(CLOUDFLARE_V4_FILE, text_v4)
+            _save_local_file(CLOUDFLARE_V6_FILE, text_v6)
+    except Exception as e:
+        print(f"Error downloading Cloudflare whitelist: {e}")
+    # Do not download PaloAlto whitelist for now, as it is not a reliable source of CIDRs and often returns HTML instead of plain text.
+    # try:
+    #     text = _download_text(PALOALTO_URL)
+    #     cidrs = extract_cidrs(text)
+    #     if cidrs and len(cidrs) > 0:
+    #         out["PaloAlto"] = cidrs
+    #         _save_local_file(PALOALTO_FILE, text)
+    # except Exception as e:
+    #     print(f"Error downloading PaloAlto whitelist: {e}")
+    return out
+
+
+def _save_local_file(path: str, text: str) -> None:
+    """Save downloaded text to a local file, creating parent dirs as needed."""
+    try:
+        os.makedirs(os.path.dirname(path), exist_ok=True)
+        with open(path, "w", encoding="utf-8") as f:
+            f.write(text)
+    except Exception as e:
+        print(f"Error saving whitelist file {path}: {e}")
+
+
+def update_whitelists() -> None:
+    """Load whitelists (local files first, then remote, then built-in fallback).
+
+    Priority per provider:
+      1. Local file in data/<provider>/
+      2. Remote download (saves to local file on success)
+      3. Built-in hardcoded constants (always available)
+    """
+    global WHITELISTS, WHITELIST_NETS
+
+    # Start from built-in fallbacks, then layer local, then remote
+    WHITELISTS = {
+        "Openfactory": OPENFACTORY_IPV4,
+        "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
+        "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
+    }
+
+    # Layer 1: local files
+    local = _load_local_whitelists()
+    if local:
+        WHITELISTS.update(local)
+
+    # Layer 2: remote download (highest priority, saves local)
+    remote = _download_whitelists()
+    if remote:
+        WHITELISTS.update(remote)
+
+    # Rebuild parsed networks
+    WHITELIST_NETS = {
+        name: tuple(ipaddress.ip_network(c) for c in cidrs)
+        for name, cidrs in WHITELISTS.items()
+    }
+
+
+def refresh_whitelists(force: bool = False) -> bool:
+    """Refresh provider whitelists once per process, keeping built-in fallbacks.
+
+    Returns True if a refresh was attempted, False if already done.
+    """
+    global _refreshed
+    if _refreshed and not force:
+        return False
+    update_whitelists()
+    _refreshed = True
+    return True

app/whois.py

@@ -0,0 +1,130 @@
+#!/usr/bin/env python3
+"""WHOIS subnet querying and IP consolidation."""
+
+from __future__ import annotations
+
+import ipaddress
+import json
+import os
+import re
+from typing import Iterable
+
+from .ip_utils import ip_version, parse_ip
+
+# WHOIS Cache file path
+_WHOIS_CACHE_PATH = os.path.join(
+    os.path.dirname(os.path.dirname(os.path.abspath(__file__))
+                    ), "data", "whois_cache.json"
+)
+
+_whois_cache: dict[str, str] = {}
+
+
+def load_whois_cache() -> None:
+    global _whois_cache
+    if not _whois_cache and os.path.exists(_WHOIS_CACHE_PATH):
+        try:
+            with open(_WHOIS_CACHE_PATH, "r", encoding="utf-8") as f:
+                _whois_cache = json.load(f)
+        except Exception:
+            _whois_cache = {}
+
+
+def save_whois_cache() -> None:
+    if _whois_cache:
+        os.makedirs(os.path.dirname(_WHOIS_CACHE_PATH), exist_ok=True)
+        try:
+            with open(_WHOIS_CACHE_PATH, "w", encoding="utf-8") as f:
+                json.dump(_whois_cache, f, indent=2)
+        except Exception:
+            pass
+
+
+def query_whois_subnet(ip: str) -> str | None:
+    """Query whois.radb.net for the route/subnet of an IP address."""
+    addr = parse_ip(ip)
+    if addr is None:
+        return None
+
+    load_whois_cache()
+    if ip in _whois_cache:
+        return _whois_cache[ip] or None
+
+    import socket
+    try:
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        s.settimeout(3.0)
+        s.connect(("whois.radb.net", 43))
+        s.sendall(f"{ip}\r\n".encode())
+        res = ""
+        while True:
+            chunk = s.recv(4096).decode(errors="ignore")
+            if not chunk:
+                break
+            res += chunk
+        s.close()
+
+        routes = re.findall(r"^(?:route|route6):\s*(\S+)",
+                            res, re.IGNORECASE | re.MULTILINE)
+        if routes:
+            routes_parsed = []
+            for r in routes:
+                try:
+                    routes_parsed.append(ipaddress.ip_network(r))
+                except ValueError:
+                    continue
+            if routes_parsed:
+                best_route = max(routes_parsed, key=lambda n: n.prefixlen)
+                subnet_str = str(best_route)
+                _whois_cache[ip] = subnet_str
+                save_whois_cache()
+                return subnet_str
+
+    except Exception:
+        pass
+
+    subnet_str = f"{ip}/32" if addr.version == 4 else f"{ip}/128"
+    _whois_cache[ip] = subnet_str
+    save_whois_cache()
+    return subnet_str
+
+
+def consolidate_by_subnet(ips: Iterable[str], threshold: int = 3) -> set[str]:
+    """Group IPs by WHOIS subnet; if >= threshold IPs in a subnet, block the subnet.
+
+    Otherwise, block individual IPs.
+    """
+    ip_list = list(ips)
+    subnet_to_ips: dict[str, set[str]] = {}
+    ip_to_subnet: dict[str, str] = {}
+
+    for ip in ip_list:
+        subnet = query_whois_subnet(ip)
+        if subnet:
+            try:
+                net = ipaddress.ip_network(subnet, strict=False)
+                addr = ipaddress.ip_address(ip)
+                if addr in net:
+                    subnet_to_ips.setdefault(subnet, set()).add(ip)
+                    ip_to_subnet[ip] = subnet
+                    continue
+            except ValueError:
+                pass
+        fallback_net = f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128"
+        subnet_to_ips.setdefault(fallback_net, set()).add(ip)
+        ip_to_subnet[ip] = fallback_net
+
+    blocked_targets: set[str] = set()
+    for subnet, mapped_ips in subnet_to_ips.items():
+        if len(mapped_ips) >= threshold:
+            blocked_targets.add(subnet)
+        else:
+            for ip in mapped_ips:
+                blocked_targets.add(
+                    f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128")
+
+    return blocked_targets
+
+
+# Re-export path for backward compat
+WHOIS_CACHE_PATH = _WHOIS_CACHE_PATH