zwitschi/intrusion-detection
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor web_scan.py for improved readability and maintainability; add cron job script for npm log backups; update start.ps1 for consistent formatting.

Browse Source
This commit is contained in:
zwitschi committed 2026-09-08 19:49:11 +02:00
1 parent 0d1cd21468
commit c2eba88ac8
113 files changed
+772606 -772579
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/utils/__init__.py
+13 -13
View File
@@ -1,13 +1,13 @@
# Intrusion Detection System — shared utilities
#
# Modules:
# constants.py central configuration, paths, env vars
# files.py filesystem & I/O helpers
# ip_utils.py IP address parsing, validation, whitelist lookup
# models.py data models (Request, IPStats) + aggregate()
# reporting.py text rendering, table formatting
# utils.py shared CLI arg parser
# whitelists.py whitelist CIDR download, caching, validation
# whois.py WHOIS subnet queries, IP consolidation
# blocklist_history.py git-backed cumulative blocklist reconstruction
# retention.py report file retention & pruning
# Intrusion Detection System — shared utilities
#
# Modules:
# constants.py central configuration, paths, env vars
# files.py filesystem & I/O helpers
# ip_utils.py IP address parsing, validation, whitelist lookup
# models.py data models (Request, IPStats) + aggregate()
# reporting.py text rendering, table formatting
# utils.py shared CLI arg parser
# whitelists.py whitelist CIDR download, caching, validation
# whois.py WHOIS subnet queries, IP consolidation
# blocklist_history.py git-backed cumulative blocklist reconstruction
# retention.py report file retention & pruning
app/utils/blocklist_history.py
+186 -186
View File
@@ -1,186 +1,186 @@
#!/usr/bin/env python3
"""Git-backed cumulative blocklist history.
The consolidated blocklist lives in git-tracked files
(``reports/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``). To make the
MikroTik blocklist grow-only, the "complete set of all blocked IPs" is
reconstructed from **every git revision** of those files (the union of all
lines ever present), plus the current working files and the latest scan
targets.
This keeps IPs blocked even after they leave the NPM ~30-day log window, and
only ever removes router entries that were never blocked by the system.
"""
from __future__ import annotations
import ipaddress
import os
import re
import subprocess
from datetime import datetime
from typing import Any, Iterable
from .files import write_json
from .reporting import timestamp_name
#: A ``+ 1.2.3.4/32`` added line in a git patch; ``+++ b/...`` is the file
#: header and is excluded by the negative lookahead.
_ADDED_LINE_RE = re.compile(r"^\+(?!\+)(.*)$")
def _run_git(*args: str) -> subprocess.CompletedProcess[str] | None:
"""Run a git command, returning None on any failure (never raises)."""
try:
return subprocess.run(
["git", *args], capture_output=True, text=True, check=False)
except (OSError, subprocess.SubprocessError):
return None
def _valid_line(line: str) -> str | None:
"""Normalize a blocklist line to a canonical CIDR string, or None."""
line = line.strip()
if not line:
return None
try:
return str(ipaddress.ip_network(line, strict=False))
except ValueError:
return None
def committed_lines(path: str) -> set[str]:
"""Return the union of all lines ever present in ``path`` across git.
Uses a single ``git log --all -p -- <path>`` and collects the ``+`` added
lines (excluding the ``+++`` file header). A line present in any revision
appears as an added line in some commit's diff, so the union equals "all
lines ever present".
Returns an empty set when git is unavailable, the path is untracked, or the
repository has no history — callers then fall back to the working file.
"""
proc = _run_git("log", "--all", "-p", "--", path)
if proc is None or proc.returncode != 0:
return set()
lines: set[str] = set()
for raw in proc.stdout.splitlines():
m = _ADDED_LINE_RE.match(raw)
if not m:
continue
clean = _valid_line(m.group(1))
if clean is not None:
lines.add(clean)
return lines
def current_file_lines(path: str) -> set[str]:
"""Return the validated lines currently in ``path`` (missing file -> {})."""
try:
with open(path, "r", encoding="utf-8") as f:
return {clean for clean in (_valid_line(x) for x in f)
if clean is not None}
except OSError:
return set()
def cumulative_blocklist(v4_path: str, v6_path: str,
desired_v4: Iterable[str],
desired_v6: Iterable[str],
) -> tuple[set[str], set[str]]:
"""Complete set of all blocked targets (history | working file | desired).
Returns ``(complete_v4, complete_v6)`` of canonical CIDR strings.
"""
complete_v4 = (
committed_lines(v4_path)
| current_file_lines(v4_path)
| {c for c in (_valid_line(x) for x in desired_v4) if c is not None})
complete_v6 = (
committed_lines(v6_path)
| current_file_lines(v6_path)
| {c for c in (_valid_line(x) for x in desired_v6) if c is not None})
return complete_v4, complete_v6
def compute_changes(current_v4: set[str], current_v6: set[str],
complete_v4: set[str], complete_v6: set[str],
whitelisted_v4: set[str], whitelisted_v6: set[str],
) -> dict[str, Any]:
"""Compute the add/remove diff for a cumulative (grow-only) blocklist.
Entries are added when present in the complete set but not yet on the
router and not whitelisted. Entries are removed only when they sit on the
router but are absent from the complete set — i.e. they were never blocked
by the system (manual/rogue entries). Callers may attach extra keys (e.g.
``dry_run``).
"""
return {
"added_ipv4": sorted(complete_v4 - current_v4 - whitelisted_v4),
"added_ipv6": sorted(complete_v6 - current_v6 - whitelisted_v6),
"removed_ipv4": sorted(current_v4 - complete_v4),
"removed_ipv6": sorted(current_v6 - complete_v6),
}
def blocklist_delta(history_v4: set[str], history_v6: set[str],
prev_v4: set[str], prev_v6: set[str],
complete_v4: set[str], complete_v6: set[str],
) -> dict[str, list[str]]:
"""Complete-set delta between the previous and current blocklist state.
``added`` are entries newly part of the blocklist (this run's targets not
blocked before); ``removed`` are entries no longer part of it — normally
empty in the cumulative model.
"""
before_v4 = history_v4 | prev_v4
before_v6 = history_v6 | prev_v6
return {
"added_ipv4": sorted(complete_v4 - before_v4),
"added_ipv6": sorted(complete_v6 - before_v6),
"removed_ipv4": sorted(before_v4 - complete_v4),
"removed_ipv6": sorted(before_v6 - complete_v6),
}
def write_blocklist_report(report_dir: str,
*,
history_v4: set[str], history_v6: set[str],
prev_v4: set[str], prev_v6: set[str],
complete_v4: set[str], complete_v6: set[str],
firewall_changes: dict | None = None,
dry_run: bool = False,
errors: Iterable[str] = (),
) -> str:
"""Write the timestamped blocklist report to ``<report_dir>/app/``.
Records the complete-set delta (added/removed) plus the full cumulative
blocked list (git history | working files | scan targets), and — when
supplied — the router sync changes. Returns the written file path.
"""
delta = blocklist_delta(
history_v4, history_v6, prev_v4, prev_v6, complete_v4, complete_v6)
payload = {
"generated_at": datetime.now().astimezone().isoformat(
timespec="seconds"),
"report_dir": report_dir,
"dry_run": dry_run,
"summary": {
"added": len(delta["added_ipv4"]) + len(delta["added_ipv6"]),
"removed": len(delta["removed_ipv4"]) + len(delta["removed_ipv6"]),
"total_blocked": len(complete_v4) + len(complete_v6),
},
"added_ipv4": delta["added_ipv4"],
"added_ipv6": delta["added_ipv6"],
"removed_ipv4": delta["removed_ipv4"],
"removed_ipv6": delta["removed_ipv6"],
"blocked_ipv4": sorted(complete_v4),
"blocked_ipv6": sorted(complete_v6),
"firewall_changes": firewall_changes,
"errors": list(errors),
}
app_dir = os.path.join(report_dir, "app")
os.makedirs(app_dir, exist_ok=True)
path = os.path.join(app_dir, f"{timestamp_name()}.json")
write_json(path, payload)
return path
#!/usr/bin/env python3
"""Git-backed cumulative blocklist history.
The consolidated blocklist lives in git-tracked files
(``reports/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``). To make the
MikroTik blocklist grow-only, the "complete set of all blocked IPs" is
reconstructed from **every git revision** of those files (the union of all
lines ever present), plus the current working files and the latest scan
targets.
This keeps IPs blocked even after they leave the NPM ~30-day log window, and
only ever removes router entries that were never blocked by the system.
"""
from __future__ import annotations
import ipaddress
import os
import re
import subprocess
from datetime import datetime
from typing import Any, Iterable
from .files import write_json
from .reporting import timestamp_name
#: A ``+ 1.2.3.4/32`` added line in a git patch; ``+++ b/...`` is the file
#: header and is excluded by the negative lookahead.
_ADDED_LINE_RE = re.compile(r"^\+(?!\+)(.*)$")
def _run_git(*args: str) -> subprocess.CompletedProcess[str] | None:
"""Run a git command, returning None on any failure (never raises)."""
try:
return subprocess.run(
["git", *args], capture_output=True, text=True, check=False)
except (OSError, subprocess.SubprocessError):
return None
def _valid_line(line: str) -> str | None:
"""Normalize a blocklist line to a canonical CIDR string, or None."""
line = line.strip()
if not line:
return None
try:
return str(ipaddress.ip_network(line, strict=False))
except ValueError:
return None
def committed_lines(path: str) -> set[str]:
"""Return the union of all lines ever present in ``path`` across git.
Uses a single ``git log --all -p -- <path>`` and collects the ``+`` added
lines (excluding the ``+++`` file header). A line present in any revision
appears as an added line in some commit's diff, so the union equals "all
lines ever present".
Returns an empty set when git is unavailable, the path is untracked, or the
repository has no history — callers then fall back to the working file.
"""
proc = _run_git("log", "--all", "-p", "--", path)
if proc is None or proc.returncode != 0:
return set()
lines: set[str] = set()
for raw in proc.stdout.splitlines():
m = _ADDED_LINE_RE.match(raw)
if not m:
continue
clean = _valid_line(m.group(1))
if clean is not None:
lines.add(clean)
return lines
def current_file_lines(path: str) -> set[str]:
"""Return the validated lines currently in ``path`` (missing file -> {})."""
try:
with open(path, "r", encoding="utf-8") as f:
return {clean for clean in (_valid_line(x) for x in f)
if clean is not None}
except OSError:
return set()
def cumulative_blocklist(v4_path: str, v6_path: str,
desired_v4: Iterable[str],
desired_v6: Iterable[str],
) -> tuple[set[str], set[str]]:
"""Complete set of all blocked targets (history | working file | desired).
Returns ``(complete_v4, complete_v6)`` of canonical CIDR strings.
"""
complete_v4 = (
committed_lines(v4_path)
| current_file_lines(v4_path)
| {c for c in (_valid_line(x) for x in desired_v4) if c is not None})
complete_v6 = (
committed_lines(v6_path)
| current_file_lines(v6_path)
| {c for c in (_valid_line(x) for x in desired_v6) if c is not None})
return complete_v4, complete_v6
def compute_changes(current_v4: set[str], current_v6: set[str],
complete_v4: set[str], complete_v6: set[str],
whitelisted_v4: set[str], whitelisted_v6: set[str],
) -> dict[str, Any]:
"""Compute the add/remove diff for a cumulative (grow-only) blocklist.
Entries are added when present in the complete set but not yet on the
router and not whitelisted. Entries are removed only when they sit on the
router but are absent from the complete set — i.e. they were never blocked
by the system (manual/rogue entries). Callers may attach extra keys (e.g.
``dry_run``).
"""
return {
"added_ipv4": sorted(complete_v4 - current_v4 - whitelisted_v4),
"added_ipv6": sorted(complete_v6 - current_v6 - whitelisted_v6),
"removed_ipv4": sorted(current_v4 - complete_v4),
"removed_ipv6": sorted(current_v6 - complete_v6),
}
def blocklist_delta(history_v4: set[str], history_v6: set[str],
prev_v4: set[str], prev_v6: set[str],
complete_v4: set[str], complete_v6: set[str],
) -> dict[str, list[str]]:
"""Complete-set delta between the previous and current blocklist state.
``added`` are entries newly part of the blocklist (this run's targets not
blocked before); ``removed`` are entries no longer part of it — normally
empty in the cumulative model.
"""
before_v4 = history_v4 | prev_v4
before_v6 = history_v6 | prev_v6
return {
"added_ipv4": sorted(complete_v4 - before_v4),
"added_ipv6": sorted(complete_v6 - before_v6),
"removed_ipv4": sorted(before_v4 - complete_v4),
"removed_ipv6": sorted(before_v6 - complete_v6),
}
def write_blocklist_report(report_dir: str,
*,
history_v4: set[str], history_v6: set[str],
prev_v4: set[str], prev_v6: set[str],
complete_v4: set[str], complete_v6: set[str],
firewall_changes: dict | None = None,
dry_run: bool = False,
errors: Iterable[str] = (),
) -> str:
"""Write the timestamped blocklist report to ``<report_dir>/app/``.
Records the complete-set delta (added/removed) plus the full cumulative
blocked list (git history | working files | scan targets), and — when
supplied — the router sync changes. Returns the written file path.
"""
delta = blocklist_delta(
history_v4, history_v6, prev_v4, prev_v6, complete_v4, complete_v6)
payload = {
"generated_at": datetime.now().astimezone().isoformat(
timespec="seconds"),
"report_dir": report_dir,
"dry_run": dry_run,
"summary": {
"added": len(delta["added_ipv4"]) + len(delta["added_ipv6"]),
"removed": len(delta["removed_ipv4"]) + len(delta["removed_ipv6"]),
"total_blocked": len(complete_v4) + len(complete_v6),
},
"added_ipv4": delta["added_ipv4"],
"added_ipv6": delta["added_ipv6"],
"removed_ipv4": delta["removed_ipv4"],
"removed_ipv6": delta["removed_ipv6"],
"blocked_ipv4": sorted(complete_v4),
"blocked_ipv6": sorted(complete_v6),
"firewall_changes": firewall_changes,
"errors": list(errors),
}
app_dir = os.path.join(report_dir, "app")
os.makedirs(app_dir, exist_ok=True)
path = os.path.join(app_dir, f"{timestamp_name()}.json")
write_json(path, payload)
return path
app/utils/files.py
+104 -104
View File
@@ -1,104 +1,104 @@
#!/usr/bin/env python3
"""Filesystem and I/O helpers for the intrusion-detection system."""
from __future__ import annotations
import csv
import gzip
import json
import os
import re
from typing import Any, Iterable, Iterator, Sequence
ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
# -- directory --------------------------------------------------------------
def validate_dir(path: str, need_write: bool = False) -> str:
if not os.path.exists(path):
raise FileNotFoundError(f"path does not exist: {path}")
if not os.path.isdir(path):
raise NotADirectoryError(f"path is not a directory: {path}")
if not os.access(path, os.R_OK):
raise PermissionError(f"path not readable: {path}")
if need_write and not os.access(path, os.W_OK):
raise PermissionError(f"path not writable: {path}")
return path
def ensure_dirs(*paths: str) -> None:
for p in paths:
os.makedirs(p, exist_ok=True)
# -- log discovery ----------------------------------------------------------
def find_logs(path: str, pattern: re.Pattern | None = None,
suffixes: Sequence[str] = (),
exclude_prefixes: Sequence[str] = ()) -> list[str]:
"""List matching log file names (not full paths), sorted."""
out = []
for name in os.listdir(path):
if not os.path.isfile(os.path.join(path, name)):
continue
if any(name.startswith(p) for p in exclude_prefixes):
continue
if pattern is not None and not pattern.match(name):
continue
if suffixes and not name.endswith(tuple(suffixes)):
continue
out.append(name)
return sorted(out)
def read_lines(path: str) -> Iterator[str]:
"""Yield ANSI-stripped, newline-stripped lines. Transparently gunzips."""
opener = gzip.open if path.endswith(".gz") else open
with opener(path, "rt", encoding="utf-8", errors="replace") as f:
for line in f:
yield ANSI_RE.sub("", line.rstrip("\r\n"))
def read_lines_from(path: str, start: int = 0) -> Iterator[tuple[str, int]]:
"""Yield ``(line, resume_offset)`` for a plain-text log from byte ``start``.
Only seekable (non-gzip) files can be resumed. ``resume_offset`` points
after the last complete line; a trailing partial line (no newline yet) is
yielded but resets the resume offset to its start so it is re-read once
completed. The last yielded ``resume_offset`` is the value to store.
"""
with open(path, "rb") as f:
f.seek(start)
pos = start
for raw in f:
line_start = pos
pos += len(raw)
complete = raw.endswith(b"\n") or raw.endswith(b"\r")
text = ANSI_RE.sub(
"", raw.decode("utf-8", errors="replace").rstrip("\r\n"))
yield text, (pos if complete else line_start)
# -- writers ----------------------------------------------------------------
def write_csv(path: str, header: Sequence[str],
rows: Iterable[Sequence[Any]]) -> str:
with open(path, "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(header)
w.writerows(rows)
return path
def write_json(path: str, obj: Any) -> str:
with open(path, "w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, default=str)
return path
def write_lines(path: str, lines: Iterable[str]) -> str:
with open(path, "w", encoding="utf-8") as f:
for line in lines:
f.write(line + "\n")
return path
#!/usr/bin/env python3
"""Filesystem and I/O helpers for the intrusion-detection system."""
from __future__ import annotations
import csv
import gzip
import json
import os
import re
from typing import Any, Iterable, Iterator, Sequence
ANSI_RE = re.compile(r"\x1b\[[0-9;]*m")
# -- directory --------------------------------------------------------------
def validate_dir(path: str, need_write: bool = False) -> str:
if not os.path.exists(path):
raise FileNotFoundError(f"path does not exist: {path}")
if not os.path.isdir(path):
raise NotADirectoryError(f"path is not a directory: {path}")
if not os.access(path, os.R_OK):
raise PermissionError(f"path not readable: {path}")
if need_write and not os.access(path, os.W_OK):
raise PermissionError(f"path not writable: {path}")
return path
def ensure_dirs(*paths: str) -> None:
for p in paths:
os.makedirs(p, exist_ok=True)
# -- log discovery ----------------------------------------------------------
def find_logs(path: str, pattern: re.Pattern | None = None,
suffixes: Sequence[str] = (),
exclude_prefixes: Sequence[str] = ()) -> list[str]:
"""List matching log file names (not full paths), sorted."""
out = []
for name in os.listdir(path):
if not os.path.isfile(os.path.join(path, name)):
continue
if any(name.startswith(p) for p in exclude_prefixes):
continue
if pattern is not None and not pattern.match(name):
continue
if suffixes and not name.endswith(tuple(suffixes)):
continue
out.append(name)
return sorted(out)
def read_lines(path: str) -> Iterator[str]:
"""Yield ANSI-stripped, newline-stripped lines. Transparently gunzips."""
opener = gzip.open if path.endswith(".gz") else open
with opener(path, "rt", encoding="utf-8", errors="replace") as f:
for line in f:
yield ANSI_RE.sub("", line.rstrip("\r\n"))
def read_lines_from(path: str, start: int = 0) -> Iterator[tuple[str, int]]:
"""Yield ``(line, resume_offset)`` for a plain-text log from byte ``start``.
Only seekable (non-gzip) files can be resumed. ``resume_offset`` points
after the last complete line; a trailing partial line (no newline yet) is
yielded but resets the resume offset to its start so it is re-read once
completed. The last yielded ``resume_offset`` is the value to store.
"""
with open(path, "rb") as f:
f.seek(start)
pos = start
for raw in f:
line_start = pos
pos += len(raw)
complete = raw.endswith(b"\n") or raw.endswith(b"\r")
text = ANSI_RE.sub(
"", raw.decode("utf-8", errors="replace").rstrip("\r\n"))
yield text, (pos if complete else line_start)
# -- writers ----------------------------------------------------------------
def write_csv(path: str, header: Sequence[str],
rows: Iterable[Sequence[Any]]) -> str:
with open(path, "w", newline="", encoding="utf-8") as f:
w = csv.writer(f)
w.writerow(header)
w.writerows(rows)
return path
def write_json(path: str, obj: Any) -> str:
with open(path, "w", encoding="utf-8") as f:
json.dump(obj, f, indent=2, default=str)
return path
def write_lines(path: str, lines: Iterable[str]) -> str:
with open(path, "w", encoding="utf-8") as f:
for line in lines:
f.write(line + "\n")
return path
app/utils/ip_utils.py
+63 -63
View File
@@ -1,63 +1,63 @@
#!/usr/bin/env python3
"""IP helper functions for the intrusion-detection system."""
from __future__ import annotations
import ipaddress
from typing import Iterable
from .whitelists import WHITELIST_NETS as _WHITELIST_NETS
def parse_ip(value: str):
"""Return an ip_address object, or None if `value` is not an IP."""
try:
return ipaddress.ip_address(value)
except (ValueError, TypeError):
return None
def ip_version(value: str) -> int | None:
addr = parse_ip(value)
return addr.version if addr else None
def is_ipv6(value: str) -> bool:
return ip_version(value) == 6
def is_ipv4(value: str) -> bool:
return ip_version(value) == 4
def whitelist_of(value: str) -> str | None:
"""Return the whitelist name an IP belongs to, else None."""
addr = parse_ip(value)
if addr is None:
return None
for name, nets in _WHITELIST_NETS.items():
for net in nets:
if addr.version == net.version and addr in net:
return name
return None
def is_whitelisted(value: str) -> bool:
return whitelist_of(value) is not None
def sort_ips(ips: Iterable[str]) -> list[str]:
"""Numeric sort, IPv4 before IPv6, unparseable last."""
def key(ip: str):
addr = parse_ip(ip)
if addr is None:
return (9, b"", ip)
return (addr.version, addr.packed, "")
return sorted(ips, key=key)
def split_versions(ips: Iterable[str]) -> tuple[list[str], list[str]]:
"""Split an IP iterable into (sorted IPv4, sorted IPv6)."""
ips = list(ips)
return (sort_ips(ip for ip in ips if is_ipv4(ip)),
sort_ips(ip for ip in ips if is_ipv6(ip)))
#!/usr/bin/env python3
"""IP helper functions for the intrusion-detection system."""
from __future__ import annotations
import ipaddress
from typing import Iterable
from .whitelists import WHITELIST_NETS as _WHITELIST_NETS
def parse_ip(value: str):
"""Return an ip_address object, or None if `value` is not an IP."""
try:
return ipaddress.ip_address(value)
except (ValueError, TypeError):
return None
def ip_version(value: str) -> int | None:
addr = parse_ip(value)
return addr.version if addr else None
def is_ipv6(value: str) -> bool:
return ip_version(value) == 6
def is_ipv4(value: str) -> bool:
return ip_version(value) == 4
def whitelist_of(value: str) -> str | None:
"""Return the whitelist name an IP belongs to, else None."""
addr = parse_ip(value)
if addr is None:
return None
for name, nets in _WHITELIST_NETS.items():
for net in nets:
if addr.version == net.version and addr in net:
return name
return None
def is_whitelisted(value: str) -> bool:
return whitelist_of(value) is not None
def sort_ips(ips: Iterable[str]) -> list[str]:
"""Numeric sort, IPv4 before IPv6, unparseable last."""
def key(ip: str):
addr = parse_ip(ip)
if addr is None:
return (9, b"", ip)
return (addr.version, addr.packed, "")
return sorted(ips, key=key)
def split_versions(ips: Iterable[str]) -> tuple[list[str], list[str]]:
"""Split an IP iterable into (sorted IPv4, sorted IPv6)."""
ips = list(ips)
return (sort_ips(ip for ip in ips if is_ipv4(ip)),
sort_ips(ip for ip in ips if is_ipv6(ip)))
app/utils/models.py
+150 -150
View File
@@ -1,150 +1,150 @@
#!/usr/bin/env python3
"""Data models for the intrusion-detection system.
`Request` and `IPStats` plus the `aggregate()` fold function.
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass, field
from typing import Iterable
from .. import bad_patterns
from .ip_utils import ip_version, whitelist_of
@dataclass(slots=True)
class Request:
"""One normalised request line, produced by any log parser."""
ts: str = ""
ip: str = ""
method: str | None = None
path: str | None = None
status: int | None = None
upstream_status: int | None = None
host: str | None = None
user: str | None = None
ua: str | None = None
referer: str | None = None
denied: bool = False
banned: bool = False
extra: dict = field(default_factory=dict)
@property
def is_client_error(self) -> bool:
return self.status is not None and 400 <= self.status < 500
@property
def is_server_error(self) -> bool:
return self.status is not None and 500 <= self.status < 600
@property
def is_error(self) -> bool:
return self.denied or self.is_client_error or self.is_server_error
@dataclass
class IPStats:
"""Aggregated per-IP counters."""
ip: str
version: int = 0
total: int = 0
errors: int = 0
c4xx: int = 0
c5xx: int = 0
denied: int = 0
banned: int = 0
first: str = ""
last: str = ""
paths: Counter = field(default_factory=Counter)
methods: Counter = field(default_factory=Counter)
hosts: Counter = field(default_factory=Counter)
uas: Counter = field(default_factory=Counter)
users: set[str] = field(default_factory=set)
malicious_hits: int = 0
malicious_patterns: dict[str, int] = field(default_factory=dict)
@property
def error_ratio(self) -> float:
return round(self.errors / self.total, 3) if self.total else 0.0
@property
def authenticated(self) -> bool:
"""True if the IP ever presented a real (non-anonymous) user."""
return any(u and u != "*" for u in self.users)
@property
def whitelist(self) -> str:
return whitelist_of(self.ip) or ""
@property
def users_str(self) -> str:
return ",".join(sorted(self.users)) or "-"
@property
def methods_str(self) -> str:
return ",".join(sorted(self.methods)) or "-"
def to_dict(self) -> dict:
return {
"ip": self.ip,
"version": self.version,
"total": self.total,
"errors": self.errors,
"error_ratio": self.error_ratio,
"status_4xx": self.c4xx,
"status_5xx": self.c5xx,
"denied": self.denied,
"banned": self.banned,
"first": self.first,
"last": self.last,
"whitelist": self.whitelist or None,
"users": sorted(self.users),
"methods": dict(self.methods),
"hosts": dict(self.hosts),
"paths": dict(self.paths.most_common()),
"uas": dict(self.uas.most_common(10)),
"malicious_hits": self.malicious_hits,
"malicious_patterns": dict(self.malicious_patterns),
}
def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]:
"""Fold request records into per-IP statistics."""
stats: dict[str, IPStats] = {}
for r in requests:
s = stats.get(r.ip)
if s is None:
s = stats[r.ip] = IPStats(
ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts)
s.total += 1
s.last = r.ts
if r.denied:
s.denied += 1
if r.banned:
s.banned += 1
if r.is_client_error:
s.c4xx += 1
if r.is_server_error:
s.c5xx += 1
if r.is_error:
s.errors += 1
if r.path:
s.paths[r.path] += 1
if r.method:
s.methods[r.method] += 1
if r.host:
s.hosts[r.host] += 1
if r.ua and r.ua != "-":
s.uas[r.ua] += 1
if r.user:
s.users.add(r.user)
if r.path and bad_patterns.is_malicious(r.path):
s.malicious_hits += 1
for p in bad_patterns.classify_path(r.path):
s.malicious_patterns[p.name] = s.malicious_patterns.get(
p.name, 0) + 1
return stats
#!/usr/bin/env python3
"""Data models for the intrusion-detection system.
`Request` and `IPStats` plus the `aggregate()` fold function.
"""
from __future__ import annotations
from collections import Counter
from dataclasses import dataclass, field
from typing import Iterable
from .. import bad_patterns
from .ip_utils import ip_version, whitelist_of
@dataclass(slots=True)
class Request:
"""One normalised request line, produced by any log parser."""
ts: str = ""
ip: str = ""
method: str | None = None
path: str | None = None
status: int | None = None
upstream_status: int | None = None
host: str | None = None
user: str | None = None
ua: str | None = None
referer: str | None = None
denied: bool = False
banned: bool = False
extra: dict = field(default_factory=dict)
@property
def is_client_error(self) -> bool:
return self.status is not None and 400 <= self.status < 500
@property
def is_server_error(self) -> bool:
return self.status is not None and 500 <= self.status < 600
@property
def is_error(self) -> bool:
return self.denied or self.is_client_error or self.is_server_error
@dataclass
class IPStats:
"""Aggregated per-IP counters."""
ip: str
version: int = 0
total: int = 0
errors: int = 0
c4xx: int = 0
c5xx: int = 0
denied: int = 0
banned: int = 0
first: str = ""
last: str = ""
paths: Counter = field(default_factory=Counter)
methods: Counter = field(default_factory=Counter)
hosts: Counter = field(default_factory=Counter)
uas: Counter = field(default_factory=Counter)
users: set[str] = field(default_factory=set)
malicious_hits: int = 0
malicious_patterns: dict[str, int] = field(default_factory=dict)
@property
def error_ratio(self) -> float:
return round(self.errors / self.total, 3) if self.total else 0.0
@property
def authenticated(self) -> bool:
"""True if the IP ever presented a real (non-anonymous) user."""
return any(u and u != "*" for u in self.users)
@property
def whitelist(self) -> str:
return whitelist_of(self.ip) or ""
@property
def users_str(self) -> str:
return ",".join(sorted(self.users)) or "-"
@property
def methods_str(self) -> str:
return ",".join(sorted(self.methods)) or "-"
def to_dict(self) -> dict:
return {
"ip": self.ip,
"version": self.version,
"total": self.total,
"errors": self.errors,
"error_ratio": self.error_ratio,
"status_4xx": self.c4xx,
"status_5xx": self.c5xx,
"denied": self.denied,
"banned": self.banned,
"first": self.first,
"last": self.last,
"whitelist": self.whitelist or None,
"users": sorted(self.users),
"methods": dict(self.methods),
"hosts": dict(self.hosts),
"paths": dict(self.paths.most_common()),
"uas": dict(self.uas.most_common(10)),
"malicious_hits": self.malicious_hits,
"malicious_patterns": dict(self.malicious_patterns),
}
def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]:
"""Fold request records into per-IP statistics."""
stats: dict[str, IPStats] = {}
for r in requests:
s = stats.get(r.ip)
if s is None:
s = stats[r.ip] = IPStats(
ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts)
s.total += 1
s.last = r.ts
if r.denied:
s.denied += 1
if r.banned:
s.banned += 1
if r.is_client_error:
s.c4xx += 1
if r.is_server_error:
s.c5xx += 1
if r.is_error:
s.errors += 1
if r.path:
s.paths[r.path] += 1
if r.method:
s.methods[r.method] += 1
if r.host:
s.hosts[r.host] += 1
if r.ua and r.ua != "-":
s.uas[r.ua] += 1
if r.user:
s.users.add(r.user)
if r.path and bad_patterns.is_malicious(r.path):
s.malicious_hits += 1
for p in bad_patterns.classify_path(r.path):
s.malicious_patterns[p.name] = s.malicious_patterns.get(
p.name, 0) + 1
return stats
app/utils/reporting.py
+55 -55
View File
@@ -1,55 +1,55 @@
#!/usr/bin/env python3
"""Text rendering helpers for the intrusion-detection system."""
from __future__ import annotations
from datetime import datetime
from typing import Any, Callable, Sequence
#: (header, getter) pair used for CSV + console tables
Column = tuple[str, Callable[[Any], Any]]
def columns_header(columns: Sequence[Column]) -> list[str]:
return [c[0] for c in columns]
def columns_row(columns: Sequence[Column], obj: Any) -> list[Any]:
return [fn(obj) for _, fn in columns]
def render_table(headers: Sequence[str], rows: Sequence[Sequence[Any]],
max_width: int = 48) -> str:
"""Aligned fixed-width text table. Numeric columns right-aligned."""
def cell(v: Any) -> str:
s = "" if v is None else str(v)
return s if len(s) <= max_width else s[:max_width - 1] + "~"
body = [[cell(v) for v in row] for row in rows]
aligns = []
for i in range(len(headers)):
numeric = bool(rows) and all(
isinstance(row[i], (int, float)) and not isinstance(row[i], bool)
for row in rows)
aligns.append(">" if numeric else "<")
widths = [len(h) for h in headers]
for row in body:
for i, v in enumerate(row):
widths[i] = max(widths[i], len(v))
def line(values: Sequence[str]) -> str:
return " ".join(
f"{v:{a}{w}}" for v, a, w in zip(values, aligns, widths)).rstrip()
out = [line(list(headers)), "-" * (sum(widths) + 2 * (len(widths) - 1))]
out += [line(row) for row in body]
return "\n".join(out)
def render_kv(rows: Sequence[tuple[str, Any]]) -> str:
width = max((len(k) for k, _ in rows), default=0) + 1
return "\n".join(f"{k + ':':<{width + 1}} {v}" for k, v in rows)
def timestamp_name() -> str:
return datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
#!/usr/bin/env python3
"""Text rendering helpers for the intrusion-detection system."""
from __future__ import annotations
from datetime import datetime
from typing import Any, Callable, Sequence
#: (header, getter) pair used for CSV + console tables
Column = tuple[str, Callable[[Any], Any]]
def columns_header(columns: Sequence[Column]) -> list[str]:
return [c[0] for c in columns]
def columns_row(columns: Sequence[Column], obj: Any) -> list[Any]:
return [fn(obj) for _, fn in columns]
def render_table(headers: Sequence[str], rows: Sequence[Sequence[Any]],
max_width: int = 48) -> str:
"""Aligned fixed-width text table. Numeric columns right-aligned."""
def cell(v: Any) -> str:
s = "" if v is None else str(v)
return s if len(s) <= max_width else s[:max_width - 1] + "~"
body = [[cell(v) for v in row] for row in rows]
aligns = []
for i in range(len(headers)):
numeric = bool(rows) and all(
isinstance(row[i], (int, float)) and not isinstance(row[i], bool)
for row in rows)
aligns.append(">" if numeric else "<")
widths = [len(h) for h in headers]
for row in body:
for i, v in enumerate(row):
widths[i] = max(widths[i], len(v))
def line(values: Sequence[str]) -> str:
return " ".join(
f"{v:{a}{w}}" for v, a, w in zip(values, aligns, widths)).rstrip()
out = [line(list(headers)), "-" * (sum(widths) + 2 * (len(widths) - 1))]
out += [line(row) for row in body]
return "\n".join(out)
def render_kv(rows: Sequence[tuple[str, Any]]) -> str:
width = max((len(k) for k, _ in rows), default=0) + 1
return "\n".join(f"{k + ':':<{width + 1}} {v}" for k, v in rows)
def timestamp_name() -> str:
return datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
app/utils/retention.py
+107 -107
View File
@@ -1,107 +1,107 @@
#!/usr/bin/env python3
"""Report retention for generated files under ``/reports/``.
Every subdirectory keeps at most ``REPORT_RETENTION_MAX_FILES`` files, all
newer than ``REPORT_RETENTION_DAYS`` days. Older timestamped reports (e.g.
``reports/copyparty/*.txt``) are pruned automatically after each consolidated
scan so the tree does not grow without bound.
The git-tracked blocklist source of truth
(``<report_dir>/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``) is
always protected and never pruned.
"""
from __future__ import annotations
import os
import time
from typing import Iterable
from .constants import (
REPORT_RETENTION_DAYS,
REPORT_RETENTION_MAX_FILES,
)
#: default protected files (never pruned): the cumulative blocklist
BLOCKLIST_FILES = ("blocklist_ipv4.txt", "blocklist_ipv6.txt")
def _age_days(mtime: float) -> float:
return (time.time() - mtime) / 86400.0
def prune_report_tree(root: str,
*,
max_age_days: int = REPORT_RETENTION_DAYS,
max_files: int = REPORT_RETENTION_MAX_FILES,
dry_run: bool = False,
protected: Iterable[str] = ()) -> dict[str, list[str]]:
"""Prune a report tree so each directory keeps ≤ ``max_files`` recent files.
A file is kept only if it is both among the ``max_files`` most recently
modified files in its directory AND newer than ``max_age_days`` days.
Protected paths are never touched. Returns ``{dir: [deleted paths]}``.
"""
protected_abs = {os.path.abspath(p) for p in protected}
removed: dict[str, list[str]] = {}
for dirpath, _dirnames, filenames in os.walk(root):
files: list[tuple[str, float]] = []
for name in filenames:
path = os.path.join(dirpath, name)
if os.path.abspath(path) in protected_abs:
continue # protected files are neither kept-counted nor pruned
try:
mtime = os.path.getmtime(path)
except OSError:
continue
files.append((path, mtime))
# Newest first; only these (within the age window) survive.
files.sort(key=lambda t: t[1], reverse=True)
newest = {path for path, _ in files[:max_files]}
cutoff_days = max_age_days
deleted: list[str] = []
for path, mtime in files:
if path in newest and _age_days(mtime) <= cutoff_days:
continue # keep: recent and within the file cap
deleted.append(path)
if not dry_run:
try:
os.remove(path)
except OSError:
# e.g. file in use on Windows — skip and leave it.
continue
if deleted:
removed[dirpath] = deleted
return removed
def prune_report_dirs(*roots: str,
dry_run: bool = False,
max_age_days: int = REPORT_RETENTION_DAYS,
max_files: int = REPORT_RETENTION_MAX_FILES,
) -> dict[str, list[str]]:
"""Prune each of the given report roots (deduplicated by real path).
The blocklist source-of-truth files are always protected.
Returns ``{dir: [deleted paths]}`` across all roots.
"""
protected = [
os.path.join(root, "app", "data", name)
for root in roots
for name in BLOCKLIST_FILES
]
seen: set[str] = set()
summary: dict[str, list[str]] = {}
for root in roots:
real = os.path.realpath(root)
if real in seen or not os.path.isdir(root):
continue
seen.add(real)
summary.update(prune_report_tree(
root, max_age_days=max_age_days, max_files=max_files,
dry_run=dry_run, protected=protected))
return summary
#!/usr/bin/env python3
"""Report retention for generated files under ``/reports/``.
Every subdirectory keeps at most ``REPORT_RETENTION_MAX_FILES`` files, all
newer than ``REPORT_RETENTION_DAYS`` days. Older timestamped reports (e.g.
``reports/copyparty/*.txt``) are pruned automatically after each consolidated
scan so the tree does not grow without bound.
The git-tracked blocklist source of truth
(``<report_dir>/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``) is
always protected and never pruned.
"""
from __future__ import annotations
import os
import time
from typing import Iterable
from .constants import (
REPORT_RETENTION_DAYS,
REPORT_RETENTION_MAX_FILES,
)
#: default protected files (never pruned): the cumulative blocklist
BLOCKLIST_FILES = ("blocklist_ipv4.txt", "blocklist_ipv6.txt")
def _age_days(mtime: float) -> float:
return (time.time() - mtime) / 86400.0
def prune_report_tree(root: str,
*,
max_age_days: int = REPORT_RETENTION_DAYS,
max_files: int = REPORT_RETENTION_MAX_FILES,
dry_run: bool = False,
protected: Iterable[str] = ()) -> dict[str, list[str]]:
"""Prune a report tree so each directory keeps ≤ ``max_files`` recent files.
A file is kept only if it is both among the ``max_files`` most recently
modified files in its directory AND newer than ``max_age_days`` days.
Protected paths are never touched. Returns ``{dir: [deleted paths]}``.
"""
protected_abs = {os.path.abspath(p) for p in protected}
removed: dict[str, list[str]] = {}
for dirpath, _dirnames, filenames in os.walk(root):
files: list[tuple[str, float]] = []
for name in filenames:
path = os.path.join(dirpath, name)
if os.path.abspath(path) in protected_abs:
continue # protected files are neither kept-counted nor pruned
try:
mtime = os.path.getmtime(path)
except OSError:
continue
files.append((path, mtime))
# Newest first; only these (within the age window) survive.
files.sort(key=lambda t: t[1], reverse=True)
newest = {path for path, _ in files[:max_files]}
cutoff_days = max_age_days
deleted: list[str] = []
for path, mtime in files:
if path in newest and _age_days(mtime) <= cutoff_days:
continue # keep: recent and within the file cap
deleted.append(path)
if not dry_run:
try:
os.remove(path)
except OSError:
# e.g. file in use on Windows — skip and leave it.
continue
if deleted:
removed[dirpath] = deleted
return removed
def prune_report_dirs(*roots: str,
dry_run: bool = False,
max_age_days: int = REPORT_RETENTION_DAYS,
max_files: int = REPORT_RETENTION_MAX_FILES,
) -> dict[str, list[str]]:
"""Prune each of the given report roots (deduplicated by real path).
The blocklist source-of-truth files are always protected.
Returns ``{dir: [deleted paths]}`` across all roots.
"""
protected = [
os.path.join(root, "app", "data", name)
for root in roots
for name in BLOCKLIST_FILES
]
seen: set[str] = set()
summary: dict[str, list[str]] = {}
for root in roots:
real = os.path.realpath(root)
if real in seen or not os.path.isdir(root):
continue
seen.add(real)
summary.update(prune_report_tree(
root, max_age_days=max_age_days, max_files=max_files,
dry_run=dry_run, protected=protected))
return summary
app/utils/utils.py
+35 -35
View File
@@ -1,35 +1,35 @@
#!/usr/bin/env python3
"""Shared core for the local-dev intrusion-detection scripts.
`copyparty.py` and `npm.py` are thin adapters. They only provide:
* a log-file matcher (glob-ish pattern / suffixes)
* a line parser producing `Request` objects
* a flagging rule (which IPs look hostile)
* column definitions for their CSV output
Everything else -- IP aggregation, whitelisting, console + file reports,
MikroTik RouterOS export -- lives in this module.
"""
from __future__ import annotations
import argparse
from .constants import DEFAULT_TOP
def arg_parser(description: str, default_log_path: str,
default_report_dir: str) -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=description)
p.add_argument("--log-path", default=default_log_path,
help=f"log directory (default: {default_log_path})")
p.add_argument("--report-dir", default=default_report_dir,
help=f"report output dir (default: {default_report_dir})")
p.add_argument("--top", type=int, default=DEFAULT_TOP,
help=f"rows in the top-IP table (default: {DEFAULT_TOP})")
p.add_argument("--quiet", action="store_true",
help="write files, print nothing but the paths")
p.add_argument("--no-rsc", action="store_true",
help="skip MikroTik .rsc generation")
return p
#!/usr/bin/env python3
"""Shared core for the local-dev intrusion-detection scripts.
`copyparty.py` and `npm.py` are thin adapters. They only provide:
* a log-file matcher (glob-ish pattern / suffixes)
* a line parser producing `Request` objects
* a flagging rule (which IPs look hostile)
* column definitions for their CSV output
Everything else -- IP aggregation, whitelisting, console + file reports,
MikroTik RouterOS export -- lives in this module.
"""
from __future__ import annotations
import argparse
from .constants import DEFAULT_TOP
def arg_parser(description: str, default_log_path: str,
default_report_dir: str) -> argparse.ArgumentParser:
p = argparse.ArgumentParser(description=description)
p.add_argument("--log-path", default=default_log_path,
help=f"log directory (default: {default_log_path})")
p.add_argument("--report-dir", default=default_report_dir,
help=f"report output dir (default: {default_report_dir})")
p.add_argument("--top", type=int, default=DEFAULT_TOP,
help=f"rows in the top-IP table (default: {DEFAULT_TOP})")
p.add_argument("--quiet", action="store_true",
help="write files, print nothing but the paths")
p.add_argument("--no-rsc", action="store_true",
help="skip MikroTik .rsc generation")
return p
app/utils/whitelists.py
+277 -277
View File
@@ -1,277 +1,277 @@
#!/usr/bin/env python3
"""Whitelist management for the intrusion-detection system.
Owns provider CIDR constants, remote download/refresh logic, and
CIDR extraction/validation. Imported by `utils.py` which provides
the convenience lookups `whitelist_of()` and `is_whitelisted()`.
Whitelist loading order (per provider):
1. Local file in `data/<provider>/` (if present and readable)
2. Remote download (if network available)
3. Built-in hardcoded fallback
"""
from __future__ import annotations
import ipaddress
import os
import re
from typing import Any
from .constants import (
CLOUDFLARE_URL_V4,
CLOUDFLARE_URL_V6,
CLOUDFLARE_V4_FILE,
CLOUDFLARE_V6_FILE,
OPENFACTORY_FILE,
OPENFACTORY_URL,
PALOALTO_FILE,
PALOALTO_URL,
)
# ---------------------------------------------------------------------------
# CIDR extraction regex
# ---------------------------------------------------------------------------
CIDR_RE = re.compile(
r"(?<![0-9A-Fa-f:.])([0-9A-Fa-f:.]+/\d{1,3})(?![0-9A-Fa-f:.])")
# ---------------------------------------------------------------------------
# Local file paths (checked first, before remote download)
# ---------------------------------------------------------------------------
# (paths live in app/constants.py as OPENFACTORY_FILE, CLOUDFLARE_V4_FILE,
# CLOUDFLARE_V6_FILE, PALOALTO_FILE; URLs as OPENFACTORY_URL, ...)
# ---------------------------------------------------------------------------
# Built-in fallback CIDR lists
# ---------------------------------------------------------------------------
#: Openfactory AS58299 colocation IP range (https://openfactory.net/geofeed.csv)
OPENFACTORY_IPV4 = ["5.226.144.0/21"]
#: https://www.cloudflare.com/ips-v4
CLOUDFLARE_IPV4 = [
"173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22",
"103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18",
"190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22",
"198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13",
"104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
]
#: https://www.cloudflare.com/ips-v6
CLOUDFLARE_IPV6 = [
"2400:cb00::/32", "2606:4700::/32", "2803:f800::/32",
"2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29",
"2c0f:f248::/32",
]
#: https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity
PALOALTO_IPV4 = [
"35.203.210.0/23", "144.86.173.0/24", "147.185.132.0/23",
"162.216.149.0/24", "162.216.150.0/24", "172.105.147.0/24",
"198.235.24.0/24", "205.210.31.0/24", "216.25.88.0/21",
]
PALOALTO_IPV6 = [
"2604:a940:300:5b6::/64",
"2604:a940:301:225::/64",
"2604:a940:302:118::/64",
]
# ---------------------------------------------------------------------------
# Active whitelist state (mutated by refresh)
# ---------------------------------------------------------------------------
#: address-list name -> CIDR ranges (v4 + v6 mixed, split at render time)
WHITELISTS: dict[str, list[str]] = {
"Openfactory": OPENFACTORY_IPV4,
"Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
"PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
}
#: pre-parsed networks, so matching does not re-parse CIDRs per IP
WHITELIST_NETS: dict[str, tuple[Any, ...]] = {
name: tuple(ipaddress.ip_network(c) for c in cidrs)
for name, cidrs in WHITELISTS.items()
}
#: guard so refresh runs at most once per process
_refreshed = False
# ---------------------------------------------------------------------------
# CIDR extraction / validation
# ---------------------------------------------------------------------------
def extract_cidrs(text: str) -> list[str]:
"""Extract valid CIDR strings from plain text, CSV, or HTML."""
seen: set[str] = set()
cidrs: list[str] = []
for match in CIDR_RE.finditer(text):
candidate = match.group(1).strip()
try:
network = ipaddress.ip_network(candidate, strict=False)
except ValueError:
continue
rendered = str(network)
if rendered not in seen:
seen.add(rendered)
cidrs.append(rendered)
return cidrs
# ---------------------------------------------------------------------------
# Local file loading
# ---------------------------------------------------------------------------
def _read_local_file(path: str) -> list[str]:
"""Read a local whitelist file and extract valid CIDRs."""
try:
with open(path, "r", encoding="utf-8", errors="replace") as f:
return extract_cidrs(f.read())
except Exception:
return []
def _load_local_whitelists() -> dict[str, list[str]]:
"""Load whitelist CIDRs from local data/ files.
Returns a dict of {name: [CIDR strings]}. Only keys with
successfully parsed CIDRs are included.
"""
out: dict[str, list[str]] = {}
cidrs = _read_local_file(OPENFACTORY_FILE)
if cidrs:
out["Openfactory"] = cidrs
cf_v4 = _read_local_file(CLOUDFLARE_V4_FILE)
cf_v6 = _read_local_file(CLOUDFLARE_V6_FILE)
if cf_v4 or cf_v6:
out["Cloudflare"] = cf_v4 + cf_v6
cidrs = _read_local_file(PALOALTO_FILE)
if cidrs:
out["PaloAlto"] = cidrs
return out
# ---------------------------------------------------------------------------
# HTTP download helpers
# ---------------------------------------------------------------------------
def _download_text(url: str) -> str:
"""Download a text file from a URL, return as string."""
import urllib.request
request = urllib.request.Request(
url,
headers={
"User-Agent": "Mozilla/5.0 (compatible; intrusion-detection/1.0)",
"Accept": "text/plain,text/csv,text/html;q=0.9,*/*;q=0.8",
},
)
with urllib.request.urlopen(request) as response:
return response.read().decode("utf-8", errors="replace")
def _download_cidrs(url: str) -> list[str]:
"""Download a whitelist source and extract valid CIDRs from it."""
return extract_cidrs(_download_text(url))
# ---------------------------------------------------------------------------
# Refresh pipeline
# ---------------------------------------------------------------------------
def _download_whitelists() -> dict[str, list[str]]:
"""Download the latest whitelists from the Internet and save to local files.
Returns a dict of {name: [CIDR strings]}. Only keys with
successfully parsed CIDRs are included; failures are logged
and skipped so built-in fallbacks remain untouched.
Saves each download to the corresponding local file on success.
"""
out: dict[str, list[str]] = {}
try:
text = _download_text(OPENFACTORY_URL)
cidrs = extract_cidrs(text)
if cidrs:
out["Openfactory"] = cidrs
_save_local_file(OPENFACTORY_FILE, text)
except Exception as e:
print(f"Error downloading Openfactory whitelist: {e}")
try:
text_v4 = _download_text(CLOUDFLARE_URL_V4)
text_v6 = _download_text(CLOUDFLARE_URL_V6)
cidrs = extract_cidrs(text_v4) + extract_cidrs(text_v6)
if cidrs:
out["Cloudflare"] = cidrs
_save_local_file(CLOUDFLARE_V4_FILE, text_v4)
_save_local_file(CLOUDFLARE_V6_FILE, text_v6)
except Exception as e:
print(f"Error downloading Cloudflare whitelist: {e}")
# Do not download PaloAlto whitelist for now, as it is not a reliable source of CIDRs and often returns HTML instead of plain text.
# try:
# text = _download_text(PALOALTO_URL)
# cidrs = extract_cidrs(text)
# if cidrs and len(cidrs) > 0:
# out["PaloAlto"] = cidrs
# _save_local_file(PALOALTO_FILE, text)
# except Exception as e:
# print(f"Error downloading PaloAlto whitelist: {e}")
return out
def _save_local_file(path: str, text: str) -> None:
"""Save downloaded text to a local file, creating parent dirs as needed."""
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
except Exception as e:
print(f"Error saving whitelist file {path}: {e}")
def update_whitelists() -> None:
"""Load whitelists (local files first, then remote, then built-in fallback).
Priority per provider:
1. Local file in data/<provider>/
2. Remote download (saves to local file on success)
3. Built-in hardcoded constants (always available)
"""
global WHITELISTS, WHITELIST_NETS
# Start from built-in fallbacks, then layer local, then remote
WHITELISTS = {
"Openfactory": OPENFACTORY_IPV4,
"Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
"PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
}
# Layer 1: local files
local = _load_local_whitelists()
if local:
WHITELISTS.update(local)
# Layer 2: remote download (highest priority, saves local)
remote = _download_whitelists()
if remote:
WHITELISTS.update(remote)
# Rebuild parsed networks
WHITELIST_NETS = {
name: tuple(ipaddress.ip_network(c) for c in cidrs)
for name, cidrs in WHITELISTS.items()
}
def refresh_whitelists(force: bool = False) -> bool:
"""Refresh provider whitelists once per process, keeping built-in fallbacks.
Returns True if a refresh was attempted, False if already done.
"""
global _refreshed
if _refreshed and not force:
return False
update_whitelists()
_refreshed = True
return True
#!/usr/bin/env python3
"""Whitelist management for the intrusion-detection system.
Owns provider CIDR constants, remote download/refresh logic, and
CIDR extraction/validation. Imported by `utils.py` which provides
the convenience lookups `whitelist_of()` and `is_whitelisted()`.
Whitelist loading order (per provider):
1. Local file in `data/<provider>/` (if present and readable)
2. Remote download (if network available)
3. Built-in hardcoded fallback
"""
from __future__ import annotations
import ipaddress
import os
import re
from typing import Any
from .constants import (
CLOUDFLARE_URL_V4,
CLOUDFLARE_URL_V6,
CLOUDFLARE_V4_FILE,
CLOUDFLARE_V6_FILE,
OPENFACTORY_FILE,
OPENFACTORY_URL,
PALOALTO_FILE,
PALOALTO_URL,
)
# ---------------------------------------------------------------------------
# CIDR extraction regex
# ---------------------------------------------------------------------------
CIDR_RE = re.compile(
r"(?<![0-9A-Fa-f:.])([0-9A-Fa-f:.]+/\d{1,3})(?![0-9A-Fa-f:.])")
# ---------------------------------------------------------------------------
# Local file paths (checked first, before remote download)
# ---------------------------------------------------------------------------
# (paths live in app/constants.py as OPENFACTORY_FILE, CLOUDFLARE_V4_FILE,
# CLOUDFLARE_V6_FILE, PALOALTO_FILE; URLs as OPENFACTORY_URL, ...)
# ---------------------------------------------------------------------------
# Built-in fallback CIDR lists
# ---------------------------------------------------------------------------
#: Openfactory AS58299 colocation IP range (https://openfactory.net/geofeed.csv)
OPENFACTORY_IPV4 = ["5.226.144.0/21"]
#: https://www.cloudflare.com/ips-v4
CLOUDFLARE_IPV4 = [
"173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22",
"103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18",
"190.93.240.0/20", "188.114.96.0/20", "197.234.240.0/22",
"198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13",
"104.24.0.0/14", "172.64.0.0/13", "131.0.72.0/22",
]
#: https://www.cloudflare.com/ips-v6
CLOUDFLARE_IPV6 = [
"2400:cb00::/32", "2606:4700::/32", "2803:f800::/32",
"2405:b500::/32", "2405:8100::/32", "2a06:98c0::/29",
"2c0f:f248::/32",
]
#: https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity
PALOALTO_IPV4 = [
"35.203.210.0/23", "144.86.173.0/24", "147.185.132.0/23",
"162.216.149.0/24", "162.216.150.0/24", "172.105.147.0/24",
"198.235.24.0/24", "205.210.31.0/24", "216.25.88.0/21",
]
PALOALTO_IPV6 = [
"2604:a940:300:5b6::/64",
"2604:a940:301:225::/64",
"2604:a940:302:118::/64",
]
# ---------------------------------------------------------------------------
# Active whitelist state (mutated by refresh)
# ---------------------------------------------------------------------------
#: address-list name -> CIDR ranges (v4 + v6 mixed, split at render time)
WHITELISTS: dict[str, list[str]] = {
"Openfactory": OPENFACTORY_IPV4,
"Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
"PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
}
#: pre-parsed networks, so matching does not re-parse CIDRs per IP
WHITELIST_NETS: dict[str, tuple[Any, ...]] = {
name: tuple(ipaddress.ip_network(c) for c in cidrs)
for name, cidrs in WHITELISTS.items()
}
#: guard so refresh runs at most once per process
_refreshed = False
# ---------------------------------------------------------------------------
# CIDR extraction / validation
# ---------------------------------------------------------------------------
def extract_cidrs(text: str) -> list[str]:
"""Extract valid CIDR strings from plain text, CSV, or HTML."""
seen: set[str] = set()
cidrs: list[str] = []
for match in CIDR_RE.finditer(text):
candidate = match.group(1).strip()
try:
network = ipaddress.ip_network(candidate, strict=False)
except ValueError:
continue
rendered = str(network)
if rendered not in seen:
seen.add(rendered)
cidrs.append(rendered)
return cidrs
# ---------------------------------------------------------------------------
# Local file loading
# ---------------------------------------------------------------------------
def _read_local_file(path: str) -> list[str]:
"""Read a local whitelist file and extract valid CIDRs."""
try:
with open(path, "r", encoding="utf-8", errors="replace") as f:
return extract_cidrs(f.read())
except Exception:
return []
def _load_local_whitelists() -> dict[str, list[str]]:
"""Load whitelist CIDRs from local data/ files.
Returns a dict of {name: [CIDR strings]}. Only keys with
successfully parsed CIDRs are included.
"""
out: dict[str, list[str]] = {}
cidrs = _read_local_file(OPENFACTORY_FILE)
if cidrs:
out["Openfactory"] = cidrs
cf_v4 = _read_local_file(CLOUDFLARE_V4_FILE)
cf_v6 = _read_local_file(CLOUDFLARE_V6_FILE)
if cf_v4 or cf_v6:
out["Cloudflare"] = cf_v4 + cf_v6
cidrs = _read_local_file(PALOALTO_FILE)
if cidrs:
out["PaloAlto"] = cidrs
return out
# ---------------------------------------------------------------------------
# HTTP download helpers
# ---------------------------------------------------------------------------
def _download_text(url: str) -> str:
"""Download a text file from a URL, return as string."""
import urllib.request
request = urllib.request.Request(
url,
headers={
"User-Agent": "Mozilla/5.0 (compatible; intrusion-detection/1.0)",
"Accept": "text/plain,text/csv,text/html;q=0.9,*/*;q=0.8",
},
)
with urllib.request.urlopen(request) as response:
return response.read().decode("utf-8", errors="replace")
def _download_cidrs(url: str) -> list[str]:
"""Download a whitelist source and extract valid CIDRs from it."""
return extract_cidrs(_download_text(url))
# ---------------------------------------------------------------------------
# Refresh pipeline
# ---------------------------------------------------------------------------
def _download_whitelists() -> dict[str, list[str]]:
"""Download the latest whitelists from the Internet and save to local files.
Returns a dict of {name: [CIDR strings]}. Only keys with
successfully parsed CIDRs are included; failures are logged
and skipped so built-in fallbacks remain untouched.
Saves each download to the corresponding local file on success.
"""
out: dict[str, list[str]] = {}
try:
text = _download_text(OPENFACTORY_URL)
cidrs = extract_cidrs(text)
if cidrs:
out["Openfactory"] = cidrs
_save_local_file(OPENFACTORY_FILE, text)
except Exception as e:
print(f"Error downloading Openfactory whitelist: {e}")
try:
text_v4 = _download_text(CLOUDFLARE_URL_V4)
text_v6 = _download_text(CLOUDFLARE_URL_V6)
cidrs = extract_cidrs(text_v4) + extract_cidrs(text_v6)
if cidrs:
out["Cloudflare"] = cidrs
_save_local_file(CLOUDFLARE_V4_FILE, text_v4)
_save_local_file(CLOUDFLARE_V6_FILE, text_v6)
except Exception as e:
print(f"Error downloading Cloudflare whitelist: {e}")
# Do not download PaloAlto whitelist for now, as it is not a reliable source of CIDRs and often returns HTML instead of plain text.
# try:
# text = _download_text(PALOALTO_URL)
# cidrs = extract_cidrs(text)
# if cidrs and len(cidrs) > 0:
# out["PaloAlto"] = cidrs
# _save_local_file(PALOALTO_FILE, text)
# except Exception as e:
# print(f"Error downloading PaloAlto whitelist: {e}")
return out
def _save_local_file(path: str, text: str) -> None:
"""Save downloaded text to a local file, creating parent dirs as needed."""
try:
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "w", encoding="utf-8") as f:
f.write(text)
except Exception as e:
print(f"Error saving whitelist file {path}: {e}")
def update_whitelists() -> None:
"""Load whitelists (local files first, then remote, then built-in fallback).
Priority per provider:
1. Local file in data/<provider>/
2. Remote download (saves to local file on success)
3. Built-in hardcoded constants (always available)
"""
global WHITELISTS, WHITELIST_NETS
# Start from built-in fallbacks, then layer local, then remote
WHITELISTS = {
"Openfactory": OPENFACTORY_IPV4,
"Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6,
"PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6,
}
# Layer 1: local files
local = _load_local_whitelists()
if local:
WHITELISTS.update(local)
# Layer 2: remote download (highest priority, saves local)
remote = _download_whitelists()
if remote:
WHITELISTS.update(remote)
# Rebuild parsed networks
WHITELIST_NETS = {
name: tuple(ipaddress.ip_network(c) for c in cidrs)
for name, cidrs in WHITELISTS.items()
}
def refresh_whitelists(force: bool = False) -> bool:
"""Refresh provider whitelists once per process, keeping built-in fallbacks.
Returns True if a refresh was attempted, False if already done.
"""
global _refreshed
if _refreshed and not force:
return False
update_whitelists()
_refreshed = True
return True
app/utils/whois.py
+128 -128
View File
@@ -1,128 +1,128 @@
#!/usr/bin/env python3
"""WHOIS subnet querying and IP consolidation."""
from __future__ import annotations
import ipaddress
import json
import os
import re
from typing import Iterable
from .constants import (
SUBNET_THRESHOLD,
WHOIS_CACHE_PATH,
WHOIS_PORT,
WHOIS_SERVER,
WHOIS_TIMEOUT,
)
from .ip_utils import ip_version, parse_ip
_whois_cache: dict[str, str] = {}
def load_whois_cache() -> None:
global _whois_cache
if not _whois_cache and os.path.exists(WHOIS_CACHE_PATH):
try:
with open(WHOIS_CACHE_PATH, "r", encoding="utf-8") as f:
_whois_cache = json.load(f)
except Exception:
_whois_cache = {}
def save_whois_cache() -> None:
if _whois_cache:
os.makedirs(os.path.dirname(WHOIS_CACHE_PATH), exist_ok=True)
try:
with open(WHOIS_CACHE_PATH, "w", encoding="utf-8") as f:
json.dump(_whois_cache, f, indent=2)
except Exception:
pass
def query_whois_subnet(ip: str) -> str | None:
"""Query whois.radb.net for the route/subnet of an IP address."""
addr = parse_ip(ip)
if addr is None:
return None
load_whois_cache()
if ip in _whois_cache:
return _whois_cache[ip] or None
import socket
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(WHOIS_TIMEOUT)
s.connect((WHOIS_SERVER, WHOIS_PORT))
s.sendall(f"{ip}\r\n".encode())
res = ""
while True:
chunk = s.recv(4096).decode(errors="ignore")
if not chunk:
break
res += chunk
s.close()
routes = re.findall(r"^(?:route|route6):\s*(\S+)",
res, re.IGNORECASE | re.MULTILINE)
if routes:
routes_parsed = []
for r in routes:
try:
routes_parsed.append(ipaddress.ip_network(r))
except ValueError:
continue
if routes_parsed:
best_route = max(routes_parsed, key=lambda n: n.prefixlen)
subnet_str = str(best_route)
_whois_cache[ip] = subnet_str
save_whois_cache()
return subnet_str
except Exception:
pass
subnet_str = f"{ip}/32" if addr.version == 4 else f"{ip}/128"
_whois_cache[ip] = subnet_str
save_whois_cache()
return subnet_str
def consolidate_by_subnet(ips: Iterable[str],
threshold: int = SUBNET_THRESHOLD) -> set[str]:
"""Group IPs by WHOIS subnet; if >= threshold IPs in a subnet, block the subnet.
Otherwise, block individual IPs.
"""
ip_list = list(ips)
subnet_to_ips: dict[str, set[str]] = {}
ip_to_subnet: dict[str, str] = {}
for ip in ip_list:
subnet = query_whois_subnet(ip)
if subnet:
try:
net = ipaddress.ip_network(subnet, strict=False)
addr = ipaddress.ip_address(ip)
if addr in net:
subnet_to_ips.setdefault(subnet, set()).add(ip)
ip_to_subnet[ip] = subnet
continue
except ValueError:
pass
fallback_net = f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128"
subnet_to_ips.setdefault(fallback_net, set()).add(ip)
ip_to_subnet[ip] = fallback_net
blocked_targets: set[str] = set()
for subnet, mapped_ips in subnet_to_ips.items():
if len(mapped_ips) >= threshold:
blocked_targets.add(subnet)
else:
for ip in mapped_ips:
blocked_targets.add(
f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128")
return blocked_targets
#!/usr/bin/env python3
"""WHOIS subnet querying and IP consolidation."""
from __future__ import annotations
import ipaddress
import json
import os
import re
from typing import Iterable
from .constants import (
SUBNET_THRESHOLD,
WHOIS_CACHE_PATH,
WHOIS_PORT,
WHOIS_SERVER,
WHOIS_TIMEOUT,
)
from .ip_utils import ip_version, parse_ip
_whois_cache: dict[str, str] = {}
def load_whois_cache() -> None:
global _whois_cache
if not _whois_cache and os.path.exists(WHOIS_CACHE_PATH):
try:
with open(WHOIS_CACHE_PATH, "r", encoding="utf-8") as f:
_whois_cache = json.load(f)
except Exception:
_whois_cache = {}
def save_whois_cache() -> None:
if _whois_cache:
os.makedirs(os.path.dirname(WHOIS_CACHE_PATH), exist_ok=True)
try:
with open(WHOIS_CACHE_PATH, "w", encoding="utf-8") as f:
json.dump(_whois_cache, f, indent=2)
except Exception:
pass
def query_whois_subnet(ip: str) -> str | None:
"""Query whois.radb.net for the route/subnet of an IP address."""
addr = parse_ip(ip)
if addr is None:
return None
load_whois_cache()
if ip in _whois_cache:
return _whois_cache[ip] or None
import socket
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(WHOIS_TIMEOUT)
s.connect((WHOIS_SERVER, WHOIS_PORT))
s.sendall(f"{ip}\r\n".encode())
res = ""
while True:
chunk = s.recv(4096).decode(errors="ignore")
if not chunk:
break
res += chunk
s.close()
routes = re.findall(r"^(?:route|route6):\s*(\S+)",
res, re.IGNORECASE | re.MULTILINE)
if routes:
routes_parsed = []
for r in routes:
try:
routes_parsed.append(ipaddress.ip_network(r))
except ValueError:
continue
if routes_parsed:
best_route = max(routes_parsed, key=lambda n: n.prefixlen)
subnet_str = str(best_route)
_whois_cache[ip] = subnet_str
save_whois_cache()
return subnet_str
except Exception:
pass
subnet_str = f"{ip}/32" if addr.version == 4 else f"{ip}/128"
_whois_cache[ip] = subnet_str
save_whois_cache()
return subnet_str
def consolidate_by_subnet(ips: Iterable[str],
threshold: int = SUBNET_THRESHOLD) -> set[str]:
"""Group IPs by WHOIS subnet; if >= threshold IPs in a subnet, block the subnet.
Otherwise, block individual IPs.
"""
ip_list = list(ips)
subnet_to_ips: dict[str, set[str]] = {}
ip_to_subnet: dict[str, str] = {}
for ip in ip_list:
subnet = query_whois_subnet(ip)
if subnet:
try:
net = ipaddress.ip_network(subnet, strict=False)
addr = ipaddress.ip_address(ip)
if addr in net:
subnet_to_ips.setdefault(subnet, set()).add(ip)
ip_to_subnet[ip] = subnet
continue
except ValueError:
pass
fallback_net = f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128"
subnet_to_ips.setdefault(fallback_net, set()).add(ip)
ip_to_subnet[ip] = fallback_net
blocked_targets: set[str] = set()
for subnet, mapped_ips in subnet_to_ips.items():
if len(mapped_ips) >= threshold:
blocked_targets.add(subnet)
else:
for ip in mapped_ips:
blocked_targets.add(
f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128")
return blocked_targets