Refactor monolith utils into domain modules and update docs

models.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Data models for the intrusion-detection system.
+
+`Request` and `IPStats` plus the `aggregate()` fold function.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Iterable
+
+import bad_patterns
+from ip_utils import ip_version, whitelist_of
+
+
+@dataclass(slots=True)
+class Request:
+    """One normalised request line, produced by any log parser."""
+
+    ts: str = ""
+    ip: str = ""
+    method: str | None = None
+    path: str | None = None
+    status: int | None = None
+    upstream_status: int | None = None
+    host: str | None = None
+    user: str | None = None
+    ua: str | None = None
+    referer: str | None = None
+    denied: bool = False
+    banned: bool = False
+    extra: dict = field(default_factory=dict)
+
+    @property
+    def is_client_error(self) -> bool:
+        return self.status is not None and 400 <= self.status < 500
+
+    @property
+    def is_server_error(self) -> bool:
+        return self.status is not None and 500 <= self.status < 600
+
+    @property
+    def is_error(self) -> bool:
+        return self.denied or self.is_client_error or self.is_server_error
+
+
+@dataclass
+class IPStats:
+    """Aggregated per-IP counters."""
+
+    ip: str
+    version: int = 0
+    total: int = 0
+    errors: int = 0
+    c4xx: int = 0
+    c5xx: int = 0
+    denied: int = 0
+    banned: int = 0
+    first: str = ""
+    last: str = ""
+    paths: Counter = field(default_factory=Counter)
+    methods: Counter = field(default_factory=Counter)
+    hosts: Counter = field(default_factory=Counter)
+    uas: Counter = field(default_factory=Counter)
+    users: set[str] = field(default_factory=set)
+    malicious_hits: int = 0
+    malicious_patterns: dict[str, int] = field(default_factory=dict)
+
+    @property
+    def error_ratio(self) -> float:
+        return round(self.errors / self.total, 3) if self.total else 0.0
+
+    @property
+    def authenticated(self) -> bool:
+        """True if the IP ever presented a real (non-anonymous) user."""
+        return any(u and u != "*" for u in self.users)
+
+    @property
+    def whitelist(self) -> str:
+        return whitelist_of(self.ip) or ""
+
+    @property
+    def users_str(self) -> str:
+        return ",".join(sorted(self.users)) or "-"
+
+    @property
+    def methods_str(self) -> str:
+        return ",".join(sorted(self.methods)) or "-"
+
+    def to_dict(self) -> dict:
+        return {
+            "ip": self.ip,
+            "version": self.version,
+            "total": self.total,
+            "errors": self.errors,
+            "error_ratio": self.error_ratio,
+            "status_4xx": self.c4xx,
+            "status_5xx": self.c5xx,
+            "denied": self.denied,
+            "banned": self.banned,
+            "first": self.first,
+            "last": self.last,
+            "whitelist": self.whitelist or None,
+            "users": sorted(self.users),
+            "methods": dict(self.methods),
+            "hosts": dict(self.hosts),
+            "paths": dict(self.paths.most_common()),
+            "uas": dict(self.uas.most_common(10)),
+            "malicious_hits": self.malicious_hits,
+            "malicious_patterns": dict(self.malicious_patterns),
+        }
+
+
+def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]:
+    """Fold request records into per-IP statistics."""
+    stats: dict[str, IPStats] = {}
+    for r in requests:
+        s = stats.get(r.ip)
+        if s is None:
+            s = stats[r.ip] = IPStats(
+                ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts)
+        s.total += 1
+        s.last = r.ts
+        if r.denied:
+            s.denied += 1
+        if r.banned:
+            s.banned += 1
+        if r.is_client_error:
+            s.c4xx += 1
+        if r.is_server_error:
+            s.c5xx += 1
+        if r.is_error:
+            s.errors += 1
+        if r.path:
+            s.paths[r.path] += 1
+        if r.method:
+            s.methods[r.method] += 1
+        if r.host:
+            s.hosts[r.host] += 1
+        if r.ua and r.ua != "-":
+            s.uas[r.ua] += 1
+        if r.user:
+            s.users.add(r.user)
+        if r.path and bad_patterns.is_malicious(r.path):
+            s.malicious_hits += 1
+            for p in bad_patterns.classify_path(r.path):
+                s.malicious_patterns[p.name] = s.malicious_patterns.get(
+                    p.name, 0) + 1
+    return stats