168 lines
5.9 KiB
Python
168 lines
5.9 KiB
Python
#!/usr/bin/env python3
|
|
"""Nginx Proxy Manager access-log analyzer.
|
|
|
|
Parses NPM proxy access logs (plain + gzipped), aggregates per-IP
|
|
behaviour and flags scanner IPs: clients with many error responses and a
|
|
high error ratio. Flagged IPs become the MikroTik block list.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import re
|
|
from collections import Counter
|
|
|
|
from analyzer import Analyzer
|
|
from files import write_csv
|
|
from flags import flag_scanners
|
|
from models import Request
|
|
from utils import arg_parser
|
|
|
|
|
|
DEFAULT_LOG_PATH = os.environ.get("NPM_LOG_PATH", r"Y:\npm\logs")
|
|
DEFAULT_REPORT_DIR = os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), "reports", "npm")
|
|
|
|
# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path"
|
|
# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer"
|
|
LINE_RE = re.compile(
|
|
r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+" # [datetime]
|
|
r"-\s+"
|
|
r"(\d+|-)\s+" # upstream
|
|
r"(\d+)\s+" # status
|
|
r"-\s+"
|
|
r"(\S+)\s+" # method
|
|
r"(\S+)\s+" # scheme
|
|
r"(\S+)\s+" # host
|
|
r'"([^"]*)"\s+' # "path"
|
|
r"\[Client\s+(\S+)\]\s+"
|
|
r"\[Length\s+(\d+|-)\]\s+"
|
|
r"\[Gzip\s+(\S+)\]\s+"
|
|
r"\[Sent-to\s+(\S+)\]\s+"
|
|
r'"([^"]*)"\s+' # "UA"
|
|
r'"([^"]*)"' # "Referer"
|
|
)
|
|
|
|
|
|
class NpmAnalyzer(Analyzer):
|
|
name = "npm"
|
|
title = "Nginx Proxy Manager log analysis report"
|
|
log_suffixes = (".log", ".gz")
|
|
exclude_prefixes = ("letsencrypt",)
|
|
paths_file = "error_paths.csv"
|
|
paths_label = "error paths"
|
|
flag_files = ("scanner_ips.txt", "scanner_ipv6.txt")
|
|
flag_label = "Scanner IPs"
|
|
flag_note = "high error rate -- probable scanners/attackers"
|
|
|
|
ip_columns = (
|
|
("ip", lambda s: s.ip),
|
|
("total", lambda s: s.total),
|
|
("4xx", lambda s: s.c4xx),
|
|
("5xx", lambda s: s.c5xx),
|
|
("ratio", lambda s: s.error_ratio),
|
|
("first", lambda s: s.first),
|
|
("last", lambda s: s.last),
|
|
("methods", lambda s: s.methods_str),
|
|
("whitelist", lambda s: s.whitelist),
|
|
)
|
|
|
|
request_columns = (
|
|
("datetime", lambda r: r.ts),
|
|
("ip", lambda r: r.ip),
|
|
("method", lambda r: r.method),
|
|
("scheme", lambda r: r.extra.get("scheme")),
|
|
("host", lambda r: r.host),
|
|
("path", lambda r: r.path),
|
|
("status", lambda r: r.status),
|
|
("upstream_status", lambda r: r.upstream_status),
|
|
("length", lambda r: r.extra.get("length")),
|
|
("gzip", lambda r: r.extra.get("gzip")),
|
|
("sent_to", lambda r: r.extra.get("sent_to")),
|
|
("referer", lambda r: r.referer),
|
|
("ua", lambda r: r.ua),
|
|
)
|
|
|
|
def __init__(self, *args, min_errors: int = 5,
|
|
error_ratio: float = 0.5, **kwargs):
|
|
super().__init__(*args, **kwargs)
|
|
self.min_errors = min_errors
|
|
self.min_ratio = error_ratio
|
|
|
|
# --- parsing -----------------------------------------------------------
|
|
|
|
def parse_line(self, line: str) -> Request | None:
|
|
if not line:
|
|
return None
|
|
m = LINE_RE.match(line)
|
|
if not m:
|
|
return None
|
|
(ts, upstream, status, method, scheme, host, path, ip,
|
|
length, gz, sent_to, ua, referer) = m.groups()
|
|
return Request(
|
|
ts=ts,
|
|
ip=ip,
|
|
method=method,
|
|
path=path,
|
|
status=int(status),
|
|
upstream_status=int(upstream) if upstream.isdigit() else None,
|
|
host=host,
|
|
ua=ua,
|
|
referer=referer,
|
|
extra={"scheme": scheme, "length": length,
|
|
"gzip": gz, "sent_to": sent_to},
|
|
)
|
|
|
|
# --- flagging ----------------------------------------------------------
|
|
|
|
def flag(self, stats):
|
|
return flag_scanners(stats, min_errors=self.min_errors,
|
|
min_ratio=self.min_ratio)
|
|
|
|
# --- extras ------------------------------------------------------------
|
|
|
|
def extra_summary_rows(self):
|
|
c4 = sum(1 for r in self.requests if r.is_client_error)
|
|
c5 = sum(1 for r in self.requests if r.is_server_error)
|
|
return [
|
|
("4xx / 5xx", f"{c4} / {c5}"),
|
|
("hosts seen", len({r.host for r in self.requests if r.host})),
|
|
("scanner threshold",
|
|
f">= {self.min_errors} errors and ratio >= {self.min_ratio}"),
|
|
]
|
|
|
|
def write_extra(self, data_dir: str) -> list[str]:
|
|
uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-")
|
|
hosts = Counter(r.host for r in self.requests if r.host)
|
|
return [
|
|
write_csv(os.path.join(data_dir, "ua_summary.csv"),
|
|
["user_agent", "count"], uas.most_common()),
|
|
write_csv(os.path.join(data_dir, "host_summary.csv"),
|
|
["host", "count"], hosts.most_common()),
|
|
]
|
|
|
|
|
|
def main() -> None:
|
|
parser = arg_parser(
|
|
"analyze NPM proxy logs, flag scanner IPs, export MikroTik rules",
|
|
DEFAULT_LOG_PATH, DEFAULT_REPORT_DIR)
|
|
parser.add_argument("--min-errors", type=int, default=5,
|
|
help="minimum error responses to flag (default: 5)")
|
|
parser.add_argument("--error-ratio", type=float, default=0.5,
|
|
help="minimum error ratio to flag (default: 0.5)")
|
|
args = parser.parse_args()
|
|
|
|
NpmAnalyzer(
|
|
log_path=args.log_path,
|
|
report_dir=args.report_dir,
|
|
top=args.top,
|
|
quiet=args.quiet,
|
|
write_rsc=not args.no_rsc,
|
|
min_errors=args.min_errors,
|
|
error_ratio=args.error_ratio,
|
|
).run()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|