zwitschi
9f8ab0f61c
- Implemented a new utility for querying WHOIS information and consolidating IP addresses by subnet. - Added caching mechanism for WHOIS responses to improve performance. - Introduced functions for loading and saving the WHOIS cache, querying subnets, and consolidating IPs based on a threshold. feat: create Flask web UI for intrusion detection system - Developed a Flask web application to manage the intrusion detection system. - Implemented routes for scanning, reports, blocklist management, and firewall configuration. - Added helper functions for reading files and handling requests. feat: define routes for web UI - Registered routes for various web pages and API endpoints in the Flask application. - Implemented functionality for scanning, reporting, blocklist management, and firewall state retrieval. feat: implement background scan runner for web UI - Created a background scan runner to handle scan jobs in a separate thread. - Managed an in-memory job store for tracking scan jobs and their statuses. - Added persistence for scan jobs to maintain state across application restarts. chore: update start script for Flask application - Enhanced the PowerShell start script to check for the existence of the virtual environment. - Added comments and structured the script for better readability and usability.
108 lines
3.8 KiB
Python
108 lines
3.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Report retention for generated files under ``/reports/``.
|
|
|
|
Every subdirectory keeps at most ``REPORT_RETENTION_MAX_FILES`` files, all
|
|
newer than ``REPORT_RETENTION_DAYS`` days. Older timestamped reports (e.g.
|
|
``reports/copyparty/*.txt``) are pruned automatically after each consolidated
|
|
scan so the tree does not grow without bound.
|
|
|
|
The git-tracked blocklist source of truth
|
|
(``<report_dir>/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``) is
|
|
always protected and never pruned.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import time
|
|
from typing import Iterable
|
|
|
|
from .constants import (
|
|
REPORT_RETENTION_DAYS,
|
|
REPORT_RETENTION_MAX_FILES,
|
|
)
|
|
|
|
#: default protected files (never pruned): the cumulative blocklist
|
|
BLOCKLIST_FILES = ("blocklist_ipv4.txt", "blocklist_ipv6.txt")
|
|
|
|
|
|
def _age_days(mtime: float) -> float:
|
|
return (time.time() - mtime) / 86400.0
|
|
|
|
|
|
def prune_report_tree(root: str,
|
|
*,
|
|
max_age_days: int = REPORT_RETENTION_DAYS,
|
|
max_files: int = REPORT_RETENTION_MAX_FILES,
|
|
dry_run: bool = False,
|
|
protected: Iterable[str] = ()) -> dict[str, list[str]]:
|
|
"""Prune a report tree so each directory keeps ≤ ``max_files`` recent files.
|
|
|
|
A file is kept only if it is both among the ``max_files`` most recently
|
|
modified files in its directory AND newer than ``max_age_days`` days.
|
|
Protected paths are never touched. Returns ``{dir: [deleted paths]}``.
|
|
"""
|
|
protected_abs = {os.path.abspath(p) for p in protected}
|
|
removed: dict[str, list[str]] = {}
|
|
|
|
for dirpath, _dirnames, filenames in os.walk(root):
|
|
files: list[tuple[str, float]] = []
|
|
for name in filenames:
|
|
path = os.path.join(dirpath, name)
|
|
if os.path.abspath(path) in protected_abs:
|
|
continue # protected files are neither kept-counted nor pruned
|
|
try:
|
|
mtime = os.path.getmtime(path)
|
|
except OSError:
|
|
continue
|
|
files.append((path, mtime))
|
|
|
|
# Newest first; only these (within the age window) survive.
|
|
files.sort(key=lambda t: t[1], reverse=True)
|
|
newest = {path for path, _ in files[:max_files]}
|
|
cutoff_days = max_age_days
|
|
|
|
deleted: list[str] = []
|
|
for path, mtime in files:
|
|
if path in newest and _age_days(mtime) <= cutoff_days:
|
|
continue # keep: recent and within the file cap
|
|
deleted.append(path)
|
|
if not dry_run:
|
|
try:
|
|
os.remove(path)
|
|
except OSError:
|
|
# e.g. file in use on Windows — skip and leave it.
|
|
continue
|
|
if deleted:
|
|
removed[dirpath] = deleted
|
|
|
|
return removed
|
|
|
|
|
|
def prune_report_dirs(*roots: str,
|
|
dry_run: bool = False,
|
|
max_age_days: int = REPORT_RETENTION_DAYS,
|
|
max_files: int = REPORT_RETENTION_MAX_FILES,
|
|
) -> dict[str, list[str]]:
|
|
"""Prune each of the given report roots (deduplicated by real path).
|
|
|
|
The blocklist source-of-truth files are always protected.
|
|
Returns ``{dir: [deleted paths]}`` across all roots.
|
|
"""
|
|
protected = [
|
|
os.path.join(root, "app", "data", name)
|
|
for root in roots
|
|
for name in BLOCKLIST_FILES
|
|
]
|
|
seen: set[str] = set()
|
|
summary: dict[str, list[str]] = {}
|
|
for root in roots:
|
|
real = os.path.realpath(root)
|
|
if real in seen or not os.path.isdir(root):
|
|
continue
|
|
seen.add(real)
|
|
summary.update(prune_report_tree(
|
|
root, max_age_days=max_age_days, max_files=max_files,
|
|
dry_run=dry_run, protected=protected))
|
|
return summary
|