151 lines
4.5 KiB
Python
151 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Data models for the intrusion-detection system.
|
|
|
|
`Request` and `IPStats` plus the `aggregate()` fold function.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections import Counter
|
|
from dataclasses import dataclass, field
|
|
from typing import Iterable
|
|
|
|
from .. import bad_patterns
|
|
from .ip_utils import ip_version, whitelist_of
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class Request:
|
|
"""One normalised request line, produced by any log parser."""
|
|
|
|
ts: str = ""
|
|
ip: str = ""
|
|
method: str | None = None
|
|
path: str | None = None
|
|
status: int | None = None
|
|
upstream_status: int | None = None
|
|
host: str | None = None
|
|
user: str | None = None
|
|
ua: str | None = None
|
|
referer: str | None = None
|
|
denied: bool = False
|
|
banned: bool = False
|
|
extra: dict = field(default_factory=dict)
|
|
|
|
@property
|
|
def is_client_error(self) -> bool:
|
|
return self.status is not None and 400 <= self.status < 500
|
|
|
|
@property
|
|
def is_server_error(self) -> bool:
|
|
return self.status is not None and 500 <= self.status < 600
|
|
|
|
@property
|
|
def is_error(self) -> bool:
|
|
return self.denied or self.is_client_error or self.is_server_error
|
|
|
|
|
|
@dataclass
|
|
class IPStats:
|
|
"""Aggregated per-IP counters."""
|
|
|
|
ip: str
|
|
version: int = 0
|
|
total: int = 0
|
|
errors: int = 0
|
|
c4xx: int = 0
|
|
c5xx: int = 0
|
|
denied: int = 0
|
|
banned: int = 0
|
|
first: str = ""
|
|
last: str = ""
|
|
paths: Counter = field(default_factory=Counter)
|
|
methods: Counter = field(default_factory=Counter)
|
|
hosts: Counter = field(default_factory=Counter)
|
|
uas: Counter = field(default_factory=Counter)
|
|
users: set[str] = field(default_factory=set)
|
|
malicious_hits: int = 0
|
|
malicious_patterns: dict[str, int] = field(default_factory=dict)
|
|
|
|
@property
|
|
def error_ratio(self) -> float:
|
|
return round(self.errors / self.total, 3) if self.total else 0.0
|
|
|
|
@property
|
|
def authenticated(self) -> bool:
|
|
"""True if the IP ever presented a real (non-anonymous) user."""
|
|
return any(u and u != "*" for u in self.users)
|
|
|
|
@property
|
|
def whitelist(self) -> str:
|
|
return whitelist_of(self.ip) or ""
|
|
|
|
@property
|
|
def users_str(self) -> str:
|
|
return ",".join(sorted(self.users)) or "-"
|
|
|
|
@property
|
|
def methods_str(self) -> str:
|
|
return ",".join(sorted(self.methods)) or "-"
|
|
|
|
def to_dict(self) -> dict:
|
|
return {
|
|
"ip": self.ip,
|
|
"version": self.version,
|
|
"total": self.total,
|
|
"errors": self.errors,
|
|
"error_ratio": self.error_ratio,
|
|
"status_4xx": self.c4xx,
|
|
"status_5xx": self.c5xx,
|
|
"denied": self.denied,
|
|
"banned": self.banned,
|
|
"first": self.first,
|
|
"last": self.last,
|
|
"whitelist": self.whitelist or None,
|
|
"users": sorted(self.users),
|
|
"methods": dict(self.methods),
|
|
"hosts": dict(self.hosts),
|
|
"paths": dict(self.paths.most_common()),
|
|
"uas": dict(self.uas.most_common(10)),
|
|
"malicious_hits": self.malicious_hits,
|
|
"malicious_patterns": dict(self.malicious_patterns),
|
|
}
|
|
|
|
|
|
def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]:
|
|
"""Fold request records into per-IP statistics."""
|
|
stats: dict[str, IPStats] = {}
|
|
for r in requests:
|
|
s = stats.get(r.ip)
|
|
if s is None:
|
|
s = stats[r.ip] = IPStats(
|
|
ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts)
|
|
s.total += 1
|
|
s.last = r.ts
|
|
if r.denied:
|
|
s.denied += 1
|
|
if r.banned:
|
|
s.banned += 1
|
|
if r.is_client_error:
|
|
s.c4xx += 1
|
|
if r.is_server_error:
|
|
s.c5xx += 1
|
|
if r.is_error:
|
|
s.errors += 1
|
|
if r.path:
|
|
s.paths[r.path] += 1
|
|
if r.method:
|
|
s.methods[r.method] += 1
|
|
if r.host:
|
|
s.hosts[r.host] += 1
|
|
if r.ua and r.ua != "-":
|
|
s.uas[r.ua] += 1
|
|
if r.user:
|
|
s.users.add(r.user)
|
|
if r.path and bad_patterns.is_malicious(r.path):
|
|
s.malicious_hits += 1
|
|
for p in bad_patterns.classify_path(r.path):
|
|
s.malicious_patterns[p.name] = s.malicious_patterns.get(
|
|
p.name, 0) + 1
|
|
return stats
|