zwitschi/intrusion-detection
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(prune): add functionality to remove old single-host IP blocks while keeping CIDR ranges intact

Browse Source
This commit is contained in:
zwitschi committed 2026-11-06 11:33:12 +01:00
1 parent 714aaf901e
commit 5ece4cd113
3 files changed
+536 -428
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/db/repositories.py
+71 -3
View File
@@ -11,7 +11,8 @@ from __future__ import annotations
import json import json
import sqlite3 import sqlite3
from contextlib import contextmanager from contextlib import contextmanager
from datetime import datetime, timezone from datetime import datetime, timedelta, timezone
import ipaddress
from typing import Any, Iterator, Sequence from typing import Any, Iterator, Sequence
from ..utils.models import IPStats from ..utils.models import IPStats
@@ -104,7 +105,8 @@ def _parse_ts(ts: str) -> tuple[str, float | None]:
cleaned = ts.strip() cleaned = ts.strip()
# Accept a trailing timezone like +HH:MM or Z; fall back to naive parse. # Accept a trailing timezone like +HH:MM or Z; fall back to naive parse.
try: try:
epoch = datetime.fromisoformat(cleaned.replace("Z", "+00:00")).timestamp() epoch = datetime.fromisoformat(
cleaned.replace("Z", "+00:00")).timestamp()
except ValueError: except ValueError:
try: try:
epoch = datetime.fromisoformat(cleaned).timestamp() epoch = datetime.fromisoformat(cleaned).timestamp()
@@ -339,7 +341,7 @@ def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *,
with transaction(conn): with transaction(conn):
for cidr in to_add: for cidr in to_add:
_block_entry_inner(conn, cidr, added_by=added_by, last_seen=now, _block_entry_inner(conn, cidr, added_by=added_by, last_seen=now,
reasons=reasons) reasons=reasons)
# Refresh last_seen on entries that are still desired so the ledger # Refresh last_seen on entries that are still desired so the ledger
# reflects recency for existing rows too. # reflects recency for existing rows too.
conn.executemany( conn.executemany(
@@ -348,6 +350,52 @@ def record_blocklist(conn: sqlite3.Connection, cidrs: Sequence[str], *,
return {"added": to_add, "active": len(before) + len(to_add)} return {"added": to_add, "active": len(before) + len(to_add)}
def prune_old_single_ip_blocks(conn: sqlite3.Connection, *,
older_than_days: int = 30,
actor: str | None = "retention") -> list[str]:
"""Remove active single-host entries older than ``older_than_days``.
Keeps CIDR ranges/subnets intact. Host entries are detected via
``ipaddress.ip_network(..., strict=False)`` and only removed when the
network is exactly one address (/32 for IPv4, /128 for IPv6).
"""
cutoff = datetime.now(timezone.utc) - \
timedelta(days=max(0, older_than_days))
rows = conn.execute(
"SELECT cidr, added_at, last_seen FROM blocklist_entries "
"WHERE status='active'").fetchall()
to_remove: list[str] = []
for row in rows:
cidr = str(row["cidr"])
if not _is_single_host_cidr(cidr):
continue
seen = _parse_iso_ts(
row["last_seen"]) or _parse_iso_ts(row["added_at"])
if seen is None:
continue
if seen < cutoff:
to_remove.append(cidr)
if not to_remove:
return []
now = utcnow()
with transaction(conn):
conn.executemany(
"UPDATE blocklist_entries SET status='removed', removed_at=? "
"WHERE cidr=? AND status='active'",
[(now, cidr) for cidr in to_remove],
)
conn.executemany(
"INSERT INTO blocklist_events (action, cidr, actor, fired_at, detail) "
"VALUES ('remove', ?, ?, ?, ?)",
[(cidr, actor, now, f"single-ip-retention>{older_than_days}d")
for cidr in to_remove],
)
return sorted(to_remove)
def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *, def sync_blocklist(conn: sqlite3.Connection, desired_cidrs: Sequence[str], *,
added_by: str | None = None) -> dict[str, Any]: added_by: str | None = None) -> dict[str, Any]:
"""Reconcile active blocklist to ``desired_cidrs``. """Reconcile active blocklist to ``desired_cidrs``.
@@ -402,6 +450,26 @@ def _cidr_version(cidr: str) -> int:
return 4 if ":" not in str(cidr) else 6 return 4 if ":" not in str(cidr) else 6
def _is_single_host_cidr(cidr: str) -> bool:
try:
net = ipaddress.ip_network(cidr, strict=False)
return net.num_addresses == 1
except ValueError:
return False
def _parse_iso_ts(value: str | None) -> datetime | None:
if not value:
return None
try:
parsed = datetime.fromisoformat(str(value).replace("Z", "+00:00"))
except ValueError:
return None
if parsed.tzinfo is None:
return parsed.replace(tzinfo=timezone.utc)
return parsed.astimezone(timezone.utc)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Whitelists # Whitelists
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
app/scan/__init__.py
+430 -424
View File
@@ -1,424 +1,430 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
"""Consolidated intrusion detection log scanner. """Consolidated intrusion detection log scanner.
Runs both copyparty and Nginx Proxy Manager (NPM) log analyzers, Runs both copyparty and Nginx Proxy Manager (NPM) log analyzers,
and outputs a single consolidated MikroTik RouterOS blocklist. and outputs a single consolidated MikroTik RouterOS blocklist.
""" """
from __future__ import annotations from __future__ import annotations
import argparse import argparse
import os import os
from dataclasses import dataclass, field from dataclasses import dataclass, field
from ..utils.blocklist_history import ( from ..utils.blocklist_history import (
committed_lines, committed_lines,
cumulative_blocklist, cumulative_blocklist,
current_file_lines, current_file_lines,
write_blocklist_report, write_blocklist_report,
) )
from ..utils.constants import ( from ..utils.constants import (
COPYPARTY_LOG_PATH, COPYPARTY_LOG_PATH,
COPYPARTY_REPORT_DIR, COPYPARTY_REPORT_DIR,
DEFAULT_ERROR_RATIO, DEFAULT_ERROR_RATIO,
DEFAULT_MIN_DENIED, DEFAULT_MIN_DENIED,
DEFAULT_MIN_ERRORS, DEFAULT_MIN_ERRORS,
DEFAULT_TOP, DEFAULT_TOP,
NPM_LOG_PATH, NPM_LOG_PATH,
NPM_REPORT_DIR, NPM_REPORT_DIR,
REPORT_DIR, REPORT_DIR,
REPORT_RETENTION_DAYS, REPORT_RETENTION_DAYS,
REPORT_RETENTION_MAX_FILES, REPORT_RETENTION_MAX_FILES,
SUBNET_THRESHOLD, SUBNET_THRESHOLD,
) )
from ..firewall import MikrotikFirewall, write_blocklist_files from ..firewall import MikrotikFirewall, write_blocklist_files
from ..utils.retention import prune_report_dirs from ..utils.retention import prune_report_dirs
from ..copyparty import CopypartyAnalyzer from ..copyparty import CopypartyAnalyzer
from ..npm import NpmAnalyzer from ..npm import NpmAnalyzer
from ..utils.ip_utils import is_whitelisted from ..utils.ip_utils import is_whitelisted
from ..mikrotik import split_cidr_versions from ..mikrotik import split_cidr_versions
from ..utils.whois import consolidate_by_subnet from ..utils.whois import consolidate_by_subnet
def _sync_blocklist_to_db(connected: bool, desired_v4: list[str], def _sync_blocklist_to_db(connected: bool, desired_v4: list[str],
desired_v6: list[str], added_by: str | None = None) -> None: desired_v6: list[str],
"""Best-effort grow-only sync of block targets into the SQLite ledger. added_by: str | None = None) -> dict | None:
"""Best-effort grow-only sync of block targets into the SQLite ledger.
Uses ``record_blocklist`` (add-only) so the ledger is the authoritative
cumulative record: once an IP is ever blocked it stays active even after it Uses ``record_blocklist`` (add-only) so the ledger is the authoritative
leaves the scan window, matching the router's cumulative model. Returns the cumulative record: once an IP is ever blocked it stays active even after it
added delta or None if the DB is unavailable. leaves the scan window, matching the router's cumulative model. Returns the
""" added delta or None if the DB is unavailable.
try: """
from ..db import get_connection try:
from ..db import repositories as repo from ..db import get_connection
conn = get_connection() from ..db import repositories as repo
return repo.record_blocklist( conn = get_connection()
conn, list(desired_v4) + list(desired_v6), added_by=added_by) result = repo.record_blocklist(
except Exception as exc: # never break the scan for a DB issue conn, list(desired_v4) + list(desired_v6), added_by=added_by)
print(f"WARN: could not sync blocklist to db ({exc})") # Retention policy: expire stale single-host entries after 30 days
return None # while keeping CIDR ranges/subnets intact.
pruned = repo.prune_old_single_ip_blocks(conn, older_than_days=30)
result["pruned_single_ips"] = pruned
def _db_complete_blocklist(desired_v4: list[str], desired_v6: list[str]): return result
"""Complete cumulative blocklist from the DB ledger (B001). except Exception as exc: # never break the scan for a DB issue
print(f"WARN: could not sync blocklist to db ({exc})")
Returns ``(complete_v4, complete_v6)`` as **sets** (same contract as return None
``cumulative_blocklist``) derived from the active blocklist ledger plus
this scan's newly desired targets (in case the DB write for this scan has
not run yet). Falls back to the git-history reconstruction when the DB is def _db_complete_blocklist(desired_v4: list[str], desired_v6: list[str]):
unavailable, so nothing regresses. """Complete cumulative blocklist from the DB ledger (B001).
"""
try: Returns ``(complete_v4, complete_v6)`` as **sets** (same contract as
from ..db import get_connection ``cumulative_blocklist``) derived from the active blocklist ledger plus
from ..db import repositories as repo this scan's newly desired targets (in case the DB write for this scan has
conn = get_connection() not run yet). Falls back to the git-history reconstruction when the DB is
active = [e["cidr"] for e in repo.active_blocklist(conn)] unavailable, so nothing regresses.
except Exception: """
active = [] try:
union = set(active) | set(desired_v4) | set(desired_v6) from ..db import get_connection
v4, v6 = split_cidr_versions(union) from ..db import repositories as repo
return set(v4), set(v6) conn = get_connection()
active = [e["cidr"] for e in repo.active_blocklist(conn)]
except Exception:
@dataclass active = []
class ScanParams: union = set(active) | set(desired_v4) | set(desired_v6)
"""Configuration for a consolidated scan. v4, v6 = split_cidr_versions(union)
return set(v4), set(v6)
Shared by the CLI entry point and the web interface so both run the
exact same pipeline.
""" @dataclass
class ScanParams:
copyparty_log_path: str = COPYPARTY_LOG_PATH """Configuration for a consolidated scan.
copyparty_report_dir: str = COPYPARTY_REPORT_DIR
min_denied: int = DEFAULT_MIN_DENIED Shared by the CLI entry point and the web interface so both run the
npm_log_path: str = NPM_LOG_PATH exact same pipeline.
npm_report_dir: str = NPM_REPORT_DIR """
min_errors: int = DEFAULT_MIN_ERRORS
error_ratio: float = DEFAULT_ERROR_RATIO copyparty_log_path: str = COPYPARTY_LOG_PATH
report_dir: str = REPORT_DIR copyparty_report_dir: str = COPYPARTY_REPORT_DIR
top: int = DEFAULT_TOP min_denied: int = DEFAULT_MIN_DENIED
quiet: bool = False npm_log_path: str = NPM_LOG_PATH
full: bool = False npm_report_dir: str = NPM_REPORT_DIR
update_firewall: bool = False min_errors: int = DEFAULT_MIN_ERRORS
dry_run: bool = False error_ratio: float = DEFAULT_ERROR_RATIO
report_dir: str = REPORT_DIR
top: int = DEFAULT_TOP
@dataclass quiet: bool = False
class ScanResult: full: bool = False
"""Structured outcome of a consolidated scan.""" update_firewall: bool = False
dry_run: bool = False
ok: bool
cp_exists: bool = False
npm_exists: bool = False @dataclass
cp_candidates: int = 0 class ScanResult:
npm_candidates: int = 0 """Structured outcome of a consolidated scan."""
blocked_ips: int = 0
blocked_targets: int = 0 ok: bool
blocked_ipv4: list[str] = field(default_factory=list) cp_exists: bool = False
blocked_ipv6: list[str] = field(default_factory=list) npm_exists: bool = False
blocklist_v4_path: str | None = None cp_candidates: int = 0
blocklist_v6_path: str | None = None npm_candidates: int = 0
firewall_connected: bool = False blocked_ips: int = 0
firewall_changes: dict | None = None blocked_targets: int = 0
blocklist_report_path: str | None = None blocked_ipv4: list[str] = field(default_factory=list)
errors: list[str] = field(default_factory=list) blocked_ipv6: list[str] = field(default_factory=list)
blocklist_v4_path: str | None = None
blocklist_v6_path: str | None = None
def run_scan(params: ScanParams) -> ScanResult: firewall_connected: bool = False
"""Run the full consolidated scan pipeline and return structured results. firewall_changes: dict | None = None
blocklist_report_path: str | None = None
Steps: connect to MikroTik (optional) -> snapshot state -> run copyparty errors: list[str] = field(default_factory=list)
+ NPM analyzers -> consolidate candidates -> write blocklists -> compare
and apply firewall changes (optional).
""" def run_scan(params: ScanParams) -> ScanResult:
errors: list[str] = [] """Run the full consolidated scan pipeline and return structured results.
# --- Step 0: open the SQLite store for this consolidated run ------------- Steps: connect to MikroTik (optional) -> snapshot state -> run copyparty
db_scan_id: int | None = None + NPM analyzers -> consolidate candidates -> write blocklists -> compare
try: and apply firewall changes (optional).
from ..db import get_connection """
from ..db import repositories as repo errors: list[str] = []
from ..utils.whitelists import sync_whitelists_to_db
conn = get_connection() # --- Step 0: open the SQLite store for this consolidated run -------------
sync_whitelists_to_db() db_scan_id: int | None = None
params_map = { try:
"full": params.full, "dry_run": params.dry_run, from ..db import get_connection
"update_firewall": params.update_firewall, from ..db import repositories as repo
"quiet": params.quiet, "top": params.top, from ..utils.whitelists import sync_whitelists_to_db
} conn = get_connection()
db_scan_id = repo.start_scan(conn, "ids", params=params_map, sync_whitelists_to_db()
full=params.full, dry_run=params.dry_run) params_map = {
except Exception as exc: "full": params.full, "dry_run": params.dry_run,
print(f"WARN: sqlite store unavailable ({exc})") "update_firewall": params.update_firewall,
"quiet": params.quiet, "top": params.top,
# --- Step 1-3: connect to MikroTik RouterOS and capture current state --- }
# If the router is unreachable we fall back to scan-only mode: still db_scan_id = repo.start_scan(conn, "ids", params=params_map,
# generate reports and blocklists, but skip the firewall sync. full=params.full, dry_run=params.dry_run)
firewall: MikrotikFirewall | None = None except Exception as exc:
if params.update_firewall: print(f"WARN: sqlite store unavailable ({exc})")
firewall = MikrotikFirewall()
try: # --- Step 1-3: connect to MikroTik RouterOS and capture current state ---
firewall.connect() # If the router is unreachable we fall back to scan-only mode: still
# Read current firewall rules + address lists and snapshot them to # generate reports and blocklists, but skip the firewall sync.
# reports/mikrotik/data/ and the git-diffable data/mikrotik/. firewall: MikrotikFirewall | None = None
firewall.save_state_report(params.report_dir) if params.update_firewall:
except Exception as e: firewall = MikrotikFirewall()
message = f"could not read MikroTik router ({firewall.router_ip}): {e}" try:
errors.append(message) firewall.connect()
print(f"\nWARNING: {message}") # Read current firewall rules + address lists and snapshot them to
firewall.disconnect() # reports/mikrotik/data/ and the git-diffable data/mikrotik/.
firewall = None firewall.save_state_report(params.report_dir)
except Exception as e:
# --- Step 4: run copyparty + NPM analyzers -> blocklist candidates ------ message = f"could not read MikroTik router ({firewall.router_ip}): {e}"
cp_analyzer = CopypartyAnalyzer( errors.append(message)
log_path=params.copyparty_log_path, print(f"\nWARNING: {message}")
report_dir=params.copyparty_report_dir, firewall.disconnect()
top=params.top, firewall = None
quiet=params.quiet,
full=params.full, # --- Step 4: run copyparty + NPM analyzers -> blocklist candidates ------
min_denied=params.min_denied, cp_analyzer = CopypartyAnalyzer(
) log_path=params.copyparty_log_path,
report_dir=params.copyparty_report_dir,
npm_analyzer = NpmAnalyzer( top=params.top,
log_path=params.npm_log_path, quiet=params.quiet,
report_dir=params.npm_report_dir, full=params.full,
top=params.top, min_denied=params.min_denied,
quiet=params.quiet, )
full=params.full,
min_errors=params.min_errors, npm_analyzer = NpmAnalyzer(
error_ratio=params.error_ratio, log_path=params.npm_log_path,
) report_dir=params.npm_report_dir,
top=params.top,
print("=== Running copyparty scan ===") quiet=params.quiet,
cp_exists = True full=params.full,
try: min_errors=params.min_errors,
cp_analyzer.run(db_scan_id=db_scan_id) error_ratio=params.error_ratio,
except Exception as e: )
errors.append(f"copyparty scan failed: {e}")
print(f"Error running copyparty scan: {e}") print("=== Running copyparty scan ===")
cp_exists = False cp_exists = True
try:
print("\n=== Running NPM scan ===") cp_analyzer.run(db_scan_id=db_scan_id)
npm_exists = True except Exception as e:
try: errors.append(f"copyparty scan failed: {e}")
npm_analyzer.run(db_scan_id=db_scan_id) print(f"Error running copyparty scan: {e}")
except Exception as e: cp_exists = False
errors.append(f"NPM scan failed: {e}")
print(f"Error running NPM scan: {e}") print("\n=== Running NPM scan ===")
npm_exists = False npm_exists = True
try:
if not cp_exists and not npm_exists: npm_analyzer.run(db_scan_id=db_scan_id)
message = "Both scans failed. Unable to generate consolidated blocklist." except Exception as e:
errors.append(message) errors.append(f"NPM scan failed: {e}")
print(f"\n{message}") print(f"Error running NPM scan: {e}")
if firewall is not None: npm_exists = False
firewall.disconnect()
if db_scan_id is not None: if not cp_exists and not npm_exists:
try: message = "Both scans failed. Unable to generate consolidated blocklist."
from ..db import get_connection errors.append(message)
from ..db import repositories as repo print(f"\n{message}")
repo.finish_scan(get_connection(), db_scan_id, status="error", if firewall is not None:
error="; ".join(errors)) firewall.disconnect()
except Exception: if db_scan_id is not None:
pass try:
return ScanResult(ok=False, errors=errors) from ..db import get_connection
from ..db import repositories as repo
# Consolidate block candidates repo.finish_scan(get_connection(), db_scan_id, status="error",
consolidated_candidates: set[str] = set() error="; ".join(errors))
if cp_exists: except Exception:
consolidated_candidates.update(cp_analyzer.block_candidates()) pass
if npm_exists: return ScanResult(ok=False, errors=errors)
consolidated_candidates.update(npm_analyzer.block_candidates())
# Consolidate block candidates
# Exclude whitelisted IPs consolidated_candidates: set[str] = set()
final_blocked_ips = { if cp_exists:
ip for ip in consolidated_candidates if not is_whitelisted(ip)} consolidated_candidates.update(cp_analyzer.block_candidates())
if npm_exists:
# Group into subnets via WHOIS if multiple malicious IPs in same subnet consolidated_candidates.update(npm_analyzer.block_candidates())
print("\nGrouping blocked IPs into subnets using WHOIS...")
final_blocked_targets = consolidate_by_subnet( # Exclude whitelisted IPs
final_blocked_ips, threshold=SUBNET_THRESHOLD) final_blocked_ips = {
blocked_ipv4, blocked_ipv6 = split_cidr_versions(final_blocked_targets) ip for ip in consolidated_candidates if not is_whitelisted(ip)}
cp_candidates = len(cp_analyzer.block_candidates()) if cp_exists else 0 # Group into subnets via WHOIS if multiple malicious IPs in same subnet
npm_candidates = len(npm_analyzer.block_candidates()) if npm_exists else 0 print("\nGrouping blocked IPs into subnets using WHOIS...")
final_blocked_targets = consolidate_by_subnet(
print("\n=== Consolidated Summary ===") final_blocked_ips, threshold=SUBNET_THRESHOLD)
print(f"copyparty block candidates: {cp_candidates}") blocked_ipv4, blocked_ipv6 = split_cidr_versions(final_blocked_targets)
print(f"NPM block candidates: {npm_candidates}")
print(f"Total unified blocked IPs: {len(final_blocked_ips)}") cp_candidates = len(cp_analyzer.block_candidates()) if cp_exists else 0
print(f"Total target blocks/subnets: {len(final_blocked_targets)}") npm_candidates = len(npm_analyzer.block_candidates()) if npm_exists else 0
# --- Step 6: write consolidated blocklists to reports/app/data ---------- print("\n=== Consolidated Summary ===")
# Capture the previous blocklist state BEFORE the files are overwritten so print(f"copyparty block candidates: {cp_candidates}")
# the change report can compute the complete-set delta. print(f"NPM block candidates: {npm_candidates}")
app_data_dir = os.path.join(params.report_dir, "app", "data") print(f"Total unified blocked IPs: {len(final_blocked_ips)}")
blocklist_v4_path = os.path.join(app_data_dir, "blocklist_ipv4.txt") print(f"Total target blocks/subnets: {len(final_blocked_targets)}")
blocklist_v6_path = os.path.join(app_data_dir, "blocklist_ipv6.txt")
prev_v4 = current_file_lines(blocklist_v4_path) # --- Step 6: write consolidated blocklists to reports/app/data ----------
prev_v6 = current_file_lines(blocklist_v6_path) # Capture the previous blocklist state BEFORE the files are overwritten so
history_v4 = committed_lines(blocklist_v4_path) # the change report can compute the complete-set delta.
history_v6 = committed_lines(blocklist_v6_path) app_data_dir = os.path.join(params.report_dir, "app", "data")
blocklist_v4_path = os.path.join(app_data_dir, "blocklist_ipv4.txt")
write_blocklist_files(blocked_ipv4, blocked_ipv6, params.report_dir) blocklist_v6_path = os.path.join(app_data_dir, "blocklist_ipv6.txt")
print(f"blocklist files:\n {blocklist_v4_path}\n {blocklist_v6_path}") prev_v4 = current_file_lines(blocklist_v4_path)
prev_v6 = current_file_lines(blocklist_v6_path)
# --- Steps 5+7: compare current state with candidates and apply --------- history_v4 = committed_lines(blocklist_v4_path)
firewall_changes: dict | None = None history_v6 = committed_lines(blocklist_v6_path)
blocklist_report_path: str | None = None
if firewall is not None: write_blocklist_files(blocked_ipv4, blocked_ipv6, params.report_dir)
try: print(f"blocklist files:\n {blocklist_v4_path}\n {blocklist_v6_path}")
firewall_changes = firewall.sync_blocklist(
blocked_ipv4, blocked_ipv6, dry_run=params.dry_run, # --- Steps 5+7: compare current state with candidates and apply ---------
report_dir=params.report_dir) firewall_changes: dict | None = None
finally: blocklist_report_path: str | None = None
firewall.disconnect() if firewall is not None:
action = "Would" if params.dry_run else "" try:
print(f"{action} add {len(firewall_changes['added_ipv4'])} IPv4 and " firewall_changes = firewall.sync_blocklist(
f"{len(firewall_changes['added_ipv6'])} IPv6 addresses; " blocked_ipv4, blocked_ipv6, dry_run=params.dry_run,
f"remove {len(firewall_changes['removed_ipv4'])} IPv4 and " report_dir=params.report_dir)
f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.") finally:
firewall.disconnect()
# Timestamped change report — only when the router was reachable. action = "Would" if params.dry_run else ""
# Source of truth for the cumulative set: the SQLite |ledger| first print(f"{action} add {len(firewall_changes['added_ipv4'])} IPv4 and "
# (B001), falling back to the git-history reconstruction only if the f"{len(firewall_changes['added_ipv6'])} IPv6 addresses; "
# DB is unavailable. f"remove {len(firewall_changes['removed_ipv4'])} IPv4 and "
complete_v4, complete_v6 = _db_complete_blocklist( f"{len(firewall_changes['removed_ipv6'])} IPv6 addresses.")
blocked_ipv4, blocked_ipv6)
if not complete_v4 and not complete_v6: # Timestamped change report — only when the router was reachable.
complete_v4, complete_v6 = cumulative_blocklist( # Source of truth for the cumulative set: the SQLite |ledger| first
blocklist_v4_path, blocklist_v6_path, # (B001), falling back to the git-history reconstruction only if the
blocked_ipv4, blocked_ipv6) # DB is unavailable.
blocklist_report_path = write_blocklist_report( complete_v4, complete_v6 = _db_complete_blocklist(
params.report_dir, blocked_ipv4, blocked_ipv6)
history_v4=history_v4, history_v6=history_v6, if not complete_v4 and not complete_v6:
prev_v4=prev_v4, prev_v6=prev_v6, complete_v4, complete_v6 = cumulative_blocklist(
complete_v4=complete_v4, complete_v6=complete_v6, blocklist_v4_path, blocklist_v6_path,
firewall_changes=firewall_changes, blocked_ipv4, blocked_ipv6)
dry_run=params.dry_run, blocklist_report_path = write_blocklist_report(
errors=errors) params.report_dir,
print(f"blocklist report: {blocklist_report_path}") history_v4=history_v4, history_v6=history_v6,
prev_v4=prev_v4, prev_v6=prev_v6,
# --- Step 8: prune old reports (retention) ------------------------------ complete_v4=complete_v4, complete_v6=complete_v6,
# Each report subdirectory keeps at most REPORT_RETENTION_MAX_FILES files, firewall_changes=firewall_changes,
# all newer than REPORT_RETENTION_DAYS days. The blocklist files in dry_run=params.dry_run,
# ``app/data`` are always protected. ``--dry-run`` only previews. errors=errors)
pruned = prune_report_dirs( print(f"blocklist report: {blocklist_report_path}")
params.report_dir, params.copyparty_report_dir, params.npm_report_dir,
dry_run=params.dry_run) # --- Step 8: prune old reports (retention) ------------------------------
total_pruned = sum(len(paths) for paths in pruned.values()) # Each report subdirectory keeps at most REPORT_RETENTION_MAX_FILES files,
action = "Would prune" if params.dry_run else "Pruned" # all newer than REPORT_RETENTION_DAYS days. The blocklist files in
print(f"{action} {total_pruned} report file(s) " # ``app/data`` are always protected. ``--dry-run`` only previews.
f"(keep {REPORT_RETENTION_DAYS}d / " pruned = prune_report_dirs(
f"max {REPORT_RETENTION_MAX_FILES} per dir)") params.report_dir, params.copyparty_report_dir, params.npm_report_dir,
dry_run=params.dry_run)
# --- Step 9: persist consolidated results to SQLite --------------------- total_pruned = sum(len(paths) for paths in pruned.values())
# Sync the blocklist ledger and close out the scan record regardless of action = "Would prune" if params.dry_run else "Pruned"
# whether the router sync was attempted. print(f"{action} {total_pruned} report file(s) "
_sync_blocklist_to_db( f"(keep {REPORT_RETENTION_DAYS}d / "
firewall is not None, blocked_ipv4, blocked_ipv6, f"max {REPORT_RETENTION_MAX_FILES} per dir)")
added_by=("scan" if db_scan_id is None else f"scan:{db_scan_id}"))
if db_scan_id is not None: # --- Step 9: persist consolidated results to SQLite ---------------------
try: # Sync the blocklist ledger and close out the scan record regardless of
from ..db import get_connection # whether the router sync was attempted.
from ..db import repositories as repo _sync_blocklist_to_db(
conn = get_connection() firewall is not None, blocked_ipv4, blocked_ipv6,
result_payload = { added_by=("scan" if db_scan_id is None else f"scan:{db_scan_id}"))
"ok": True, if db_scan_id is not None:
"cp_candidates": cp_candidates, try:
"npm_candidates": npm_candidates, from ..db import get_connection
"blocked_ips": len(final_blocked_ips), from ..db import repositories as repo
"blocked_targets": len(final_blocked_targets), conn = get_connection()
"firewall_connected": firewall is not None, result_payload = {
} "ok": True,
repo.finish_scan( "cp_candidates": cp_candidates,
conn, db_scan_id, status="done", "npm_candidates": npm_candidates,
result=result_payload, log_files=0, "blocked_ips": len(final_blocked_ips),
request_count=(cp_analyzer and len(cp_analyzer.requests) or 0) "blocked_targets": len(final_blocked_targets),
+ (npm_analyzer and len(npm_analyzer.requests) or 0), "firewall_connected": firewall is not None,
error="; ".join(errors) or None, }
) repo.finish_scan(
except Exception as exc: conn, db_scan_id, status="done",
print(f"WARN: could not finalize scan in db ({exc})") result=result_payload, log_files=0,
request_count=(cp_analyzer and len(cp_analyzer.requests) or 0)
return ScanResult( + (npm_analyzer and len(npm_analyzer.requests) or 0),
ok=True, error="; ".join(errors) or None,
cp_exists=cp_exists, )
npm_exists=npm_exists, except Exception as exc:
cp_candidates=cp_candidates, print(f"WARN: could not finalize scan in db ({exc})")
npm_candidates=npm_candidates,
blocked_ips=len(final_blocked_ips), return ScanResult(
blocked_targets=len(final_blocked_targets), ok=True,
blocked_ipv4=blocked_ipv4, cp_exists=cp_exists,
blocked_ipv6=blocked_ipv6, npm_exists=npm_exists,
blocklist_v4_path=blocklist_v4_path, cp_candidates=cp_candidates,
blocklist_v6_path=blocklist_v6_path, npm_candidates=npm_candidates,
firewall_connected=firewall is not None, blocked_ips=len(final_blocked_ips),
firewall_changes=firewall_changes, blocked_targets=len(final_blocked_targets),
blocklist_report_path=blocklist_report_path, blocked_ipv4=blocked_ipv4,
errors=errors, blocked_ipv6=blocked_ipv6,
) blocklist_v4_path=blocklist_v4_path,
blocklist_v6_path=blocklist_v6_path,
firewall_connected=firewall is not None,
def _build_parser() -> argparse.ArgumentParser: firewall_changes=firewall_changes,
parser = argparse.ArgumentParser( blocklist_report_path=blocklist_report_path,
description="Consolidated IDS Scanner: run copyparty + NPM analyzers and generate unified blocklist." errors=errors,
) )
# copyparty arguments
parser.add_argument("--copyparty-log-path", default=COPYPARTY_LOG_PATH, def _build_parser() -> argparse.ArgumentParser:
help=f"copyparty log directory (default: {COPYPARTY_LOG_PATH})") parser = argparse.ArgumentParser(
parser.add_argument("--copyparty-report-dir", default=COPYPARTY_REPORT_DIR, description="Consolidated IDS Scanner: run copyparty + NPM analyzers and generate unified blocklist."
help=f"copyparty report directory (default: {COPYPARTY_REPORT_DIR})") )
parser.add_argument("--min-denied", type=int, default=DEFAULT_MIN_DENIED,
help=f"minimum copyparty denied hits before flagging (default: {DEFAULT_MIN_DENIED})") # copyparty arguments
parser.add_argument("--copyparty-log-path", default=COPYPARTY_LOG_PATH,
# NPM arguments help=f"copyparty log directory (default: {COPYPARTY_LOG_PATH})")
parser.add_argument("--npm-log-path", default=NPM_LOG_PATH, parser.add_argument("--copyparty-report-dir", default=COPYPARTY_REPORT_DIR,
help=f"NPM log directory (default: {NPM_LOG_PATH})") help=f"copyparty report directory (default: {COPYPARTY_REPORT_DIR})")
parser.add_argument("--npm-report-dir", default=NPM_REPORT_DIR, parser.add_argument("--min-denied", type=int, default=DEFAULT_MIN_DENIED,
help=f"NPM report directory (default: {NPM_REPORT_DIR})") help=f"minimum copyparty denied hits before flagging (default: {DEFAULT_MIN_DENIED})")
parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
help=f"minimum NPM error responses to flag (default: {DEFAULT_MIN_ERRORS})") # NPM arguments
parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO, parser.add_argument("--npm-log-path", default=NPM_LOG_PATH,
help=f"minimum NPM error ratio to flag (default: {DEFAULT_ERROR_RATIO})") help=f"NPM log directory (default: {NPM_LOG_PATH})")
parser.add_argument("--npm-report-dir", default=NPM_REPORT_DIR,
# Shared arguments help=f"NPM report directory (default: {NPM_REPORT_DIR})")
parser.add_argument("--report-dir", default=REPORT_DIR, parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
help=f"consolidated reports directory (default: {REPORT_DIR})") help=f"minimum NPM error responses to flag (default: {DEFAULT_MIN_ERRORS})")
parser.add_argument("--top", type=int, default=DEFAULT_TOP, parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
help=f"rows in the top-IP tables (default: {DEFAULT_TOP})") help=f"minimum NPM error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
parser.add_argument("--quiet", action="store_true",
help="write files, suppress console reports") # Shared arguments
parser.add_argument("--full", action="store_true", parser.add_argument("--report-dir", default=REPORT_DIR,
help="rescan all log files from the start (ignore incremental state)") help=f"consolidated reports directory (default: {REPORT_DIR})")
parser.add_argument("--update-firewall", action="store_true", parser.add_argument("--top", type=int, default=DEFAULT_TOP,
help="synchronize the MikroTik Blocked list after scanning") help=f"rows in the top-IP tables (default: {DEFAULT_TOP})")
parser.add_argument("--dry-run", action="store_true", parser.add_argument("--quiet", action="store_true",
help="preview firewall changes without applying them") help="write files, suppress console reports")
return parser parser.add_argument("--full", action="store_true",
help="rescan all log files from the start (ignore incremental state)")
parser.add_argument("--update-firewall", action="store_true",
def main(argv: list[str] | None = None) -> None: help="synchronize the MikroTik Blocked list after scanning")
args = _build_parser().parse_args(argv) parser.add_argument("--dry-run", action="store_true",
params = ScanParams( help="preview firewall changes without applying them")
copyparty_log_path=args.copyparty_log_path, return parser
copyparty_report_dir=args.copyparty_report_dir,
min_denied=args.min_denied,
npm_log_path=args.npm_log_path, def main(argv: list[str] | None = None) -> None:
npm_report_dir=args.npm_report_dir, args = _build_parser().parse_args(argv)
min_errors=args.min_errors, params = ScanParams(
error_ratio=args.error_ratio, copyparty_log_path=args.copyparty_log_path,
report_dir=args.report_dir, copyparty_report_dir=args.copyparty_report_dir,
top=args.top, min_denied=args.min_denied,
quiet=args.quiet, npm_log_path=args.npm_log_path,
full=args.full, npm_report_dir=args.npm_report_dir,
update_firewall=args.update_firewall, min_errors=args.min_errors,
dry_run=args.dry_run, error_ratio=args.error_ratio,
) report_dir=args.report_dir,
run_scan(params) top=args.top,
quiet=args.quiet,
full=args.full,
if __name__ == "__main__": update_firewall=args.update_firewall,
main() dry_run=args.dry_run,
)
run_scan(params)
if __name__ == "__main__":
main()
tests/test_repositories.py
+35 -1
View File
@@ -13,6 +13,7 @@ import os
import sqlite3 import sqlite3
import tempfile import tempfile
import unittest import unittest
from datetime import datetime, timedelta, timezone
from app.db import connect from app.db import connect
from app.db import repositories as repo from app.db import repositories as repo
@@ -110,7 +111,8 @@ class BlocklistTest(unittest.TestCase):
added_by="scan:1") added_by="scan:1")
self.assertEqual(r1["active"], 2) self.assertEqual(r1["active"], 2)
# a later scan only sees 5.5.5.5; 6.6.6.6 stays (cumulative). # a later scan only sees 5.5.5.5; 6.6.6.6 stays (cumulative).
r2 = repo.record_blocklist(self.conn, ["5.5.5.5/32"], added_by="scan:2") r2 = repo.record_blocklist(
self.conn, ["5.5.5.5/32"], added_by="scan:2")
active = [e["cidr"] for e in repo.active_blocklist(self.conn)] active = [e["cidr"] for e in repo.active_blocklist(self.conn)]
self.assertIn("6.6.6.6/32", active) self.assertIn("6.6.6.6/32", active)
self.assertEqual(r2["active"], 2) self.assertEqual(r2["active"], 2)
@@ -122,6 +124,38 @@ class BlocklistTest(unittest.TestCase):
[e["cidr"] for e in repo.active_blocklist(self.conn)]) [e["cidr"] for e in repo.active_blocklist(self.conn)])
self.assertFalse(repo.unblock_entry(self.conn, "7.7.7.7/32")) self.assertFalse(repo.unblock_entry(self.conn, "7.7.7.7/32"))
def test_prune_old_single_ip_blocks_keeps_ranges(self):
old = (datetime.now(timezone.utc) - timedelta(days=31)).isoformat(
timespec="seconds")
with repo.transaction(self.conn):
repo._block_entry_inner(self.conn, "1.2.3.4/32", last_seen=old)
repo._block_entry_inner(
self.conn, "2001:db8::1/128", last_seen=old)
repo._block_entry_inner(self.conn, "1.2.3.0/24", last_seen=old)
repo._block_entry_inner(self.conn, "2001:db8::/64", last_seen=old)
removed = repo.prune_old_single_ip_blocks(
self.conn, older_than_days=30)
self.assertCountEqual(removed, ["1.2.3.4/32", "2001:db8::1/128"])
active = {e["cidr"] for e in repo.active_blocklist(self.conn)}
self.assertIn("1.2.3.0/24", active)
self.assertIn("2001:db8::/64", active)
self.assertNotIn("1.2.3.4/32", active)
self.assertNotIn("2001:db8::1/128", active)
def test_prune_old_single_ip_blocks_respects_recency(self):
fresh = (datetime.now(timezone.utc) - timedelta(days=5)).isoformat(
timespec="seconds")
with repo.transaction(self.conn):
repo._block_entry_inner(self.conn, "9.9.9.9/32", last_seen=fresh)
removed = repo.prune_old_single_ip_blocks(
self.conn, older_than_days=30)
self.assertEqual(removed, [])
self.assertIn("9.9.9.9/32",
{e["cidr"] for e in repo.active_blocklist(self.conn)})
class WhitelistTest(unittest.TestCase): class WhitelistTest(unittest.TestCase):
def setUp(self): def setUp(self):