zwitschi
6036fa8b1e
- Added schema definition and migration logic in app/db/schema.py. - Created tables for log sources, requests, IP stats, scans, blocklist, whitelist, and firewall rules. - Introduced functions to apply migrations and ensure database integrity. feat: Enhance scan functionality with database integration - Added _sync_blocklist_to_db function to synchronize blocklist with SQLite. - Updated run_scan function to handle database connections and scan records. - Implemented error handling for database operations during scans. feat: Introduce log analysis page and API endpoints - Created analysis.html template for displaying log analysis results. - Added routes for fetching top IPs, IP request timelines, and cross-source IPs. - Integrated analysis data into the dashboard and status endpoints. fix: Ensure whitelists are synchronized with the database - Implemented sync_whitelists_to_db function to persist in-memory whitelists. - Updated constants.py to define the path for the SQLite database. refactor: Modularize analysis helpers for better maintainability - Created _analysis_helpers function to encapsulate SQLite query logic. - Updated routes.py to utilize new analysis helper functions for API responses.
352 lines
14 KiB
Python
352 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Analysis pipeline for the intrusion-detection system.
|
|
|
|
`Analyzer` is the log-source-agnostic base class. Subclasses implement
|
|
`parse_line()` and `flag()`.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
from collections import Counter
|
|
from datetime import datetime
|
|
from typing import Any, Sequence
|
|
|
|
from .utils.files import ensure_dirs, find_logs, read_lines, read_lines_from, validate_dir, write_csv, write_json, write_lines
|
|
from .flags import by_total, flag_malicious_patterns
|
|
from .utils.ip_utils import is_whitelisted, split_versions
|
|
from .mikrotik import write_mikrotik
|
|
from .utils.models import IPStats, Request, aggregate
|
|
from .utils.reporting import Column, columns_header, columns_row, render_kv, render_table, timestamp_name
|
|
from .utils.constants import DEFAULT_TOP, DEFAULT_TOP_PATHS
|
|
from .scan import scan_state
|
|
from .utils.whitelists import refresh_whitelists
|
|
|
|
|
|
class Analyzer:
|
|
"""Log-source-agnostic analysis pipeline.
|
|
|
|
Subclasses must set the class attributes below and implement
|
|
`parse_line()` and `flag()`.
|
|
"""
|
|
|
|
name = "ids" # short id, used in file names / tags
|
|
title = "IDS log analysis report"
|
|
log_pattern: re.Pattern | None = None
|
|
log_suffixes: tuple[str, ...] = ()
|
|
exclude_prefixes: tuple[str, ...] = ()
|
|
ip_columns: Sequence[Column] = ()
|
|
request_columns: Sequence[Column] = ()
|
|
paths_file = "error_paths.csv"
|
|
paths_label = "error paths"
|
|
flag_files = ("flagged_ips.txt", "flagged_ipv6.txt")
|
|
flag_label = "Flagged IPs"
|
|
flag_note = ""
|
|
|
|
def __init__(self, log_path: str, report_dir: str, top: int = DEFAULT_TOP,
|
|
quiet: bool = False, write_rsc: bool = True,
|
|
full: bool = False):
|
|
self.log_path = log_path
|
|
self.report_dir = report_dir
|
|
self.data_dir = os.path.join(report_dir, "data")
|
|
self.top = top
|
|
self.quiet = quiet
|
|
self.write_rsc = write_rsc
|
|
self.full = full
|
|
self.files: list[str] = []
|
|
self.requests: list[Request] = []
|
|
self.stats: dict[str, IPStats] = {}
|
|
self.flagged: dict[str, IPStats] = {}
|
|
self.per_file: dict[str, int] = {}
|
|
|
|
# --- sqlite persistence ------------------------------------------------
|
|
|
|
def persist_to_db(self, scan_id: int | None = None) -> None:
|
|
"""Ingest this analyzer's requests + per-IP stats into SQLite.
|
|
|
|
Lazily imported to avoid cycles. When ``scan_id`` is None a standalone
|
|
scan record is created for this analyzer so individual
|
|
(non-consolidated) runs are still tracked. Never raises: a DB failure
|
|
logs a warning but does not break the scan.
|
|
"""
|
|
if not self.requests and not self.stats:
|
|
return
|
|
try:
|
|
from .db import get_connection
|
|
from .db import repositories as repo
|
|
from .utils.ip_utils import ip_version as _ip_ver
|
|
conn = get_connection()
|
|
if scan_id is None:
|
|
scan_id = repo.start_scan(conn, self.name,
|
|
params={"full": self.full})
|
|
source = self.name
|
|
rows: list[dict] = []
|
|
for r in self.requests:
|
|
rows.append({
|
|
"source": source, "scan_id": scan_id,
|
|
"log_file": ((r.extra or {}).get("log_file")
|
|
or (self.files[0] if self.files else None)),
|
|
"ts": r.ts, "ip": r.ip,
|
|
"version": _ip_ver(r.ip) or 0,
|
|
"method": r.method, "path": r.path,
|
|
"status": r.status,
|
|
"upstream_status": r.upstream_status,
|
|
"host": r.host, "user": r.user, "ua": r.ua,
|
|
"referer": r.referer, "denied": r.denied,
|
|
"banned": r.banned, "is_error": r.is_error,
|
|
"extra": r.extra,
|
|
})
|
|
repo.insert_requests(conn, rows)
|
|
repo.insert_ip_stats(conn, scan_id,
|
|
list(self.stats.values()),
|
|
set(self.block_candidates()))
|
|
repo.set_scan_source_counts(
|
|
conn, scan_id, source, len(self.requests), len(self.files))
|
|
repo.finish_scan(conn, scan_id, status="done",
|
|
log_files=len(self.files),
|
|
request_count=len(self.requests))
|
|
except Exception as exc: # never break the scan because of DB issues
|
|
if not self.quiet:
|
|
print(f"WARN: sqlite ingest skipped ({exc})")
|
|
# --- hooks -------------------------------------------------------------
|
|
|
|
def parse_line(self, line: str) -> Request | None:
|
|
raise NotImplementedError
|
|
|
|
def attr_source(self, req: Request, log_file: str) -> None:
|
|
"""Record the originating log file onto a parsed request."""
|
|
if req.extra is None:
|
|
req.extra = {}
|
|
req.extra["log_file"] = req.extra.get("log_file", log_file)
|
|
|
|
def flag(self, stats: dict[str, IPStats]) -> dict[str, IPStats]:
|
|
raise NotImplementedError
|
|
|
|
def flag_comment(self, s: IPStats) -> str:
|
|
return (f"errors={s.errors} total={s.total} ratio={s.error_ratio} "
|
|
f"first={s.first} last={s.last}")
|
|
|
|
def block_candidates(self) -> set[str]:
|
|
malicious = flag_malicious_patterns(self.stats)
|
|
return set(self.flagged) | set(malicious)
|
|
|
|
def extra_summary_rows(self) -> list[tuple[str, Any]]:
|
|
return []
|
|
|
|
def write_extra(self, data_dir: str) -> list[str]:
|
|
return []
|
|
|
|
# --- pipeline ----------------------------------------------------------
|
|
|
|
def discover(self) -> list[str]:
|
|
self.files = find_logs(self.log_path, self.log_pattern,
|
|
self.log_suffixes, self.exclude_prefixes)
|
|
if not self.files:
|
|
raise FileNotFoundError(f"no log files found in: {self.log_path}")
|
|
return self.files
|
|
|
|
def load(self) -> list[Request]:
|
|
"""Read log entries, resuming from the last processed offset.
|
|
|
|
Non-gzip logs are read incrementally (only new bytes since the last
|
|
scan) unless ``full`` is set; gzip archives are always fully read
|
|
because they cannot be seeked.
|
|
"""
|
|
state: dict[str, int] = {}
|
|
if not self.full:
|
|
state = scan_state.load_state()
|
|
# Prune entries whose log file no longer exists (safe across
|
|
# analyzers: only removes dead keys, keeps live ones).
|
|
state = {k: v for k, v in state.items() if os.path.exists(k)}
|
|
for name in self.files:
|
|
full_path = os.path.join(self.log_path, name)
|
|
count = 0
|
|
if self.full or full_path.endswith(".gz"):
|
|
for line in read_lines(full_path):
|
|
req = self.parse_line(line)
|
|
if req is not None:
|
|
self.attr_source(req, name)
|
|
self.requests.append(req)
|
|
count += 1
|
|
else:
|
|
try:
|
|
size = os.path.getsize(full_path)
|
|
except OSError:
|
|
size = 0
|
|
recorded = state.get(full_path, 0)
|
|
# Reset to the start if the file shrank or was truncated.
|
|
start = recorded if 0 < recorded <= size else 0
|
|
last_resume = start
|
|
for line, resume in read_lines_from(full_path, start):
|
|
req = self.parse_line(line)
|
|
if req is not None:
|
|
self.attr_source(req, name)
|
|
self.requests.append(req)
|
|
count += 1
|
|
last_resume = resume
|
|
state[full_path] = last_resume
|
|
self.per_file[name] = count
|
|
if not self.full:
|
|
scan_state.save_state(state)
|
|
return self.requests
|
|
|
|
def analyze(self) -> dict[str, IPStats]:
|
|
self.stats = aggregate(self.requests)
|
|
self.flagged = self.flag(self.stats)
|
|
return self.stats
|
|
|
|
# --- derived data ------------------------------------------------------
|
|
|
|
def error_paths(self) -> Counter:
|
|
return Counter(r.path for r in self.requests if r.is_error and r.path)
|
|
|
|
def flagged_split(self) -> tuple[list[IPStats], list[IPStats]]:
|
|
v4 = [s for s in self.flagged.values() if s.version == 4]
|
|
v6 = [s for s in self.flagged.values() if s.version == 6]
|
|
def key(s): return (-s.errors, s.ip)
|
|
return sorted(v4, key=key), sorted(v6, key=key)
|
|
|
|
def summary_rows(self) -> list[tuple[str, Any]]:
|
|
v4, v6 = split_versions(self.stats)
|
|
rows = [
|
|
("log path", self.log_path),
|
|
("log files", len(self.files)),
|
|
("total requests", len(self.requests)),
|
|
("unique IPs",
|
|
f"{len(self.stats)} (IPv4 {len(v4)} / IPv6 {len(v6)})"),
|
|
("error requests", sum(1 for r in self.requests if r.is_error)),
|
|
(f"unique {self.paths_label}", len(self.error_paths())),
|
|
]
|
|
rows += self.extra_summary_rows()
|
|
rows += [
|
|
(self.flag_label.lower(), len(self.flagged)),
|
|
("whitelisted IPs seen",
|
|
sum(1 for ip in self.stats if is_whitelisted(ip))),
|
|
("block candidates", len(self.block_candidates())),
|
|
]
|
|
return rows
|
|
|
|
# --- reporting ---------------------------------------------------------
|
|
|
|
def text_report(self) -> str:
|
|
out = [self.title,
|
|
f"generated: {datetime.now().isoformat(timespec='seconds')}",
|
|
"",
|
|
render_kv(self.summary_rows()), ""]
|
|
|
|
out.append(f"== top {self.top} IPs ==")
|
|
rows = [columns_row(self.ip_columns, s)
|
|
for s in by_total(self.stats)[:self.top]]
|
|
out.append(render_table(columns_header(self.ip_columns), rows))
|
|
if len(self.stats) > self.top:
|
|
out.append(f"... {len(self.stats) - self.top} more IPs omitted")
|
|
out.append("")
|
|
|
|
out.append(f"== top {DEFAULT_TOP_PATHS} {self.paths_label} ==")
|
|
for path, n in self.error_paths().most_common(DEFAULT_TOP_PATHS):
|
|
out.append(f" {n:>5} {path}")
|
|
out.append("")
|
|
|
|
v4, v6 = self.flagged_split()
|
|
out.append(f"== {self.flag_label} ({len(self.flagged)}) ==")
|
|
if self.flag_note:
|
|
out.append(self.flag_note)
|
|
for label, group in (("IPv4", v4), ("IPv6", v6)):
|
|
if not group:
|
|
continue
|
|
out.append(f"-- {label} ({len(group)}) --")
|
|
rows = [columns_row(self.ip_columns, s) for s in group]
|
|
out.append(render_table(columns_header(self.ip_columns), rows))
|
|
out.append("")
|
|
return "\n".join(out)
|
|
|
|
# --- writing -----------------------------------------------------------
|
|
|
|
def write_data(self) -> list[str]:
|
|
ensure_dirs(self.report_dir, self.data_dir)
|
|
written: list[str] = []
|
|
d = self.data_dir
|
|
header = columns_header(self.ip_columns)
|
|
ordered = by_total(self.stats)
|
|
|
|
for version, fname in ((4, "ips_ipv4.csv"), (6, "ips_ipv6.csv")):
|
|
rows = [columns_row(self.ip_columns, s)
|
|
for s in ordered if s.version == version]
|
|
written.append(write_csv(os.path.join(d, fname), header, rows))
|
|
|
|
written.append(write_json(
|
|
os.path.join(d, "ips.json"),
|
|
{s.ip: s.to_dict() for s in ordered}))
|
|
|
|
written.append(write_csv(
|
|
os.path.join(d, self.paths_file), ["path", "count"],
|
|
self.error_paths().most_common()))
|
|
|
|
v4, v6 = self.flagged_split()
|
|
for group, fname in zip((v4, v6), self.flag_files):
|
|
written.append(write_lines(
|
|
os.path.join(d, fname),
|
|
[f"{s.ip} # {self.flag_comment(s)}" for s in group]))
|
|
|
|
blocked_v4, blocked_v6 = split_versions(
|
|
ip for ip in self.block_candidates() if not is_whitelisted(ip))
|
|
written.append(write_lines(
|
|
os.path.join(d, "blocked_ipv4.txt"), blocked_v4))
|
|
written.append(write_lines(
|
|
os.path.join(d, "blocked_ipv6.txt"), blocked_v6))
|
|
|
|
written.append(write_csv(
|
|
os.path.join(d, "requests.csv"),
|
|
columns_header(self.request_columns),
|
|
(columns_row(self.request_columns, r) for r in self.requests)))
|
|
|
|
written += self.write_extra(d)
|
|
return written
|
|
|
|
def write_report(self) -> str:
|
|
ensure_dirs(self.report_dir)
|
|
path = os.path.join(self.report_dir, f"{timestamp_name()}.txt")
|
|
with open(path, "w", encoding="utf-8") as f:
|
|
f.write(self.text_report())
|
|
return path
|
|
|
|
def write_rsc_file(self) -> tuple[str, int, int]:
|
|
ensure_dirs(self.data_dir)
|
|
return write_mikrotik(
|
|
os.path.join(self.data_dir, "mikrotik-block.rsc"),
|
|
f"{self.name}-ids", self.block_candidates())
|
|
|
|
# --- entrypoint --------------------------------------------------------
|
|
|
|
def run(self, db_scan_id: int | None = None) -> dict[str, Any]:
|
|
validate_dir(self.log_path)
|
|
ensure_dirs(self.report_dir, self.data_dir)
|
|
refresh_whitelists()
|
|
self.discover()
|
|
self.load()
|
|
self.analyze()
|
|
|
|
if not self.quiet:
|
|
print(self.text_report())
|
|
|
|
# Ingest into the SQLite store (creates its own scan record when no
|
|
# consolidated ``db_scan_id`` is supplied).
|
|
self.persist_to_db(scan_id=db_scan_id)
|
|
|
|
self.write_data()
|
|
report_path = self.write_report()
|
|
print(f"data files: {self.data_dir}")
|
|
print(f"summary report: {report_path}")
|
|
|
|
result: dict[str, Any] = {"report": report_path,
|
|
"data_dir": self.data_dir}
|
|
if self.write_rsc:
|
|
rsc, n_blocked, n_whitelisted = self.write_rsc_file()
|
|
print(f"mikrotik script: {rsc}")
|
|
print(f" {n_blocked} IPs blocked, "
|
|
f"{n_whitelisted} CIDR ranges whitelisted")
|
|
result.update(rsc=rsc, blocked=n_blocked,
|
|
whitelisted=n_whitelisted)
|
|
return result
|