feat: add WHOIS subnet querying and IP consolidation utility
- Implemented a new utility for querying WHOIS information and consolidating IP addresses by subnet. - Added caching mechanism for WHOIS responses to improve performance. - Introduced functions for loading and saving the WHOIS cache, querying subnets, and consolidating IPs based on a threshold. feat: create Flask web UI for intrusion detection system - Developed a Flask web application to manage the intrusion detection system. - Implemented routes for scanning, reports, blocklist management, and firewall configuration. - Added helper functions for reading files and handling requests. feat: define routes for web UI - Registered routes for various web pages and API endpoints in the Flask application. - Implemented functionality for scanning, reporting, blocklist management, and firewall state retrieval. feat: implement background scan runner for web UI - Created a background scan runner to handle scan jobs in a separate thread. - Managed an in-memory job store for tracking scan jobs and their statuses. - Added persistence for scan jobs to maintain state across application restarts. chore: update start script for Flask application - Enhanced the PowerShell start script to check for the existence of the virtual environment. - Added comments and structured the script for better readability and usability.
This commit is contained in:
1 parent
41230ce04c
commit
9f8ab0f61c
102 files changed
+1102652
-268
No files matched your search
@@ -0,0 +1,107 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Report retention for generated files under ``/reports/``.
|
||||
|
||||
Every subdirectory keeps at most ``REPORT_RETENTION_MAX_FILES`` files, all
|
||||
newer than ``REPORT_RETENTION_DAYS`` days. Older timestamped reports (e.g.
|
||||
``reports/copyparty/*.txt``) are pruned automatically after each consolidated
|
||||
scan so the tree does not grow without bound.
|
||||
|
||||
The git-tracked blocklist source of truth
|
||||
(``<report_dir>/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``) is
|
||||
always protected and never pruned.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
from typing import Iterable
|
||||
|
||||
from .constants import (
|
||||
REPORT_RETENTION_DAYS,
|
||||
REPORT_RETENTION_MAX_FILES,
|
||||
)
|
||||
|
||||
#: default protected files (never pruned): the cumulative blocklist
|
||||
BLOCKLIST_FILES = ("blocklist_ipv4.txt", "blocklist_ipv6.txt")
|
||||
|
||||
|
||||
def _age_days(mtime: float) -> float:
|
||||
return (time.time() - mtime) / 86400.0
|
||||
|
||||
|
||||
def prune_report_tree(root: str,
|
||||
*,
|
||||
max_age_days: int = REPORT_RETENTION_DAYS,
|
||||
max_files: int = REPORT_RETENTION_MAX_FILES,
|
||||
dry_run: bool = False,
|
||||
protected: Iterable[str] = ()) -> dict[str, list[str]]:
|
||||
"""Prune a report tree so each directory keeps ≤ ``max_files`` recent files.
|
||||
|
||||
A file is kept only if it is both among the ``max_files`` most recently
|
||||
modified files in its directory AND newer than ``max_age_days`` days.
|
||||
Protected paths are never touched. Returns ``{dir: [deleted paths]}``.
|
||||
"""
|
||||
protected_abs = {os.path.abspath(p) for p in protected}
|
||||
removed: dict[str, list[str]] = {}
|
||||
|
||||
for dirpath, _dirnames, filenames in os.walk(root):
|
||||
files: list[tuple[str, float]] = []
|
||||
for name in filenames:
|
||||
path = os.path.join(dirpath, name)
|
||||
if os.path.abspath(path) in protected_abs:
|
||||
continue # protected files are neither kept-counted nor pruned
|
||||
try:
|
||||
mtime = os.path.getmtime(path)
|
||||
except OSError:
|
||||
continue
|
||||
files.append((path, mtime))
|
||||
|
||||
# Newest first; only these (within the age window) survive.
|
||||
files.sort(key=lambda t: t[1], reverse=True)
|
||||
newest = {path for path, _ in files[:max_files]}
|
||||
cutoff_days = max_age_days
|
||||
|
||||
deleted: list[str] = []
|
||||
for path, mtime in files:
|
||||
if path in newest and _age_days(mtime) <= cutoff_days:
|
||||
continue # keep: recent and within the file cap
|
||||
deleted.append(path)
|
||||
if not dry_run:
|
||||
try:
|
||||
os.remove(path)
|
||||
except OSError:
|
||||
# e.g. file in use on Windows — skip and leave it.
|
||||
continue
|
||||
if deleted:
|
||||
removed[dirpath] = deleted
|
||||
|
||||
return removed
|
||||
|
||||
|
||||
def prune_report_dirs(*roots: str,
|
||||
dry_run: bool = False,
|
||||
max_age_days: int = REPORT_RETENTION_DAYS,
|
||||
max_files: int = REPORT_RETENTION_MAX_FILES,
|
||||
) -> dict[str, list[str]]:
|
||||
"""Prune each of the given report roots (deduplicated by real path).
|
||||
|
||||
The blocklist source-of-truth files are always protected.
|
||||
Returns ``{dir: [deleted paths]}`` across all roots.
|
||||
"""
|
||||
protected = [
|
||||
os.path.join(root, "app", "data", name)
|
||||
for root in roots
|
||||
for name in BLOCKLIST_FILES
|
||||
]
|
||||
seen: set[str] = set()
|
||||
summary: dict[str, list[str]] = {}
|
||||
for root in roots:
|
||||
real = os.path.realpath(root)
|
||||
if real in seen or not os.path.isdir(root):
|
||||
continue
|
||||
seen.add(real)
|
||||
summary.update(prune_report_tree(
|
||||
root, max_age_days=max_age_days, max_files=max_files,
|
||||
dry_run=dry_run, protected=protected))
|
||||
return summary
|
||||
Reference in new issue
Block a user