intrusion-detection/app/npm.py

#!/usr/bin/env python3
"""Nginx Proxy Manager access-log analyzer.

Parses NPM proxy access logs (plain + gzipped), aggregates per-IP
behaviour and flags scanner IPs: clients with many error responses and a
high error ratio. Flagged IPs become the MikroTik block list.
"""

from __future__ import annotations

import os
import re
from collections import Counter

from .analyzer import Analyzer
from .utils.constants import (
    DEFAULT_ERROR_RATIO,
    DEFAULT_MIN_ERRORS,
    NPM_LOG_PATH,
    NPM_REPORT_DIR,
)
from .utils.files import write_csv
from .flags import flag_scanners
from .utils.models import Request
from .utils.utils import arg_parser

# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path"
# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer"
LINE_RE = re.compile(
    r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+"  # [datetime]
    r"-\s+"
    r"(\d+|-)\s+"                                                  # upstream
    r"(\d+)\s+"                                                    # status
    r"-\s+"
    r"(\S+)\s+"                                                    # method
    r"(\S+)\s+"                                                    # scheme
    r"(\S+)\s+"                                                    # host
    r'"([^"]*)"\s+'                                                # "path"
    r"\[Client\s+(\S+)\]\s+"
    r"\[Length\s+(\d+|-)\]\s+"
    r"\[Gzip\s+(\S+)\]\s+"
    r"\[Sent-to\s+(\S+)\]\s+"
    r'"([^"]*)"\s+'                                                # "UA"
    r'"([^"]*)"'                                                   # "Referer"
)


class NpmAnalyzer(Analyzer):
    name = "npm"
    title = "Nginx Proxy Manager log analysis report"
    log_suffixes = (".log", ".gz")
    exclude_prefixes = ("letsencrypt",)
    paths_file = "error_paths.csv"
    paths_label = "error paths"
    flag_files = ("scanner_ips.txt", "scanner_ipv6.txt")
    flag_label = "Scanner IPs"
    flag_note = "high error rate -- probable scanners/attackers"

    ip_columns = (
        ("ip", lambda s: s.ip),
        ("total", lambda s: s.total),
        ("4xx", lambda s: s.c4xx),
        ("5xx", lambda s: s.c5xx),
        ("ratio", lambda s: s.error_ratio),
        ("first", lambda s: s.first),
        ("last", lambda s: s.last),
        ("methods", lambda s: s.methods_str),
        ("whitelist", lambda s: s.whitelist),
    )

    request_columns = (
        ("datetime", lambda r: r.ts),
        ("ip", lambda r: r.ip),
        ("method", lambda r: r.method),
        ("scheme", lambda r: r.extra.get("scheme")),
        ("host", lambda r: r.host),
        ("path", lambda r: r.path),
        ("status", lambda r: r.status),
        ("upstream_status", lambda r: r.upstream_status),
        ("length", lambda r: r.extra.get("length")),
        ("gzip", lambda r: r.extra.get("gzip")),
        ("sent_to", lambda r: r.extra.get("sent_to")),
        ("referer", lambda r: r.referer),
        ("ua", lambda r: r.ua),
    )

    def __init__(self, *args, min_errors: int = DEFAULT_MIN_ERRORS,
                 error_ratio: float = DEFAULT_ERROR_RATIO, **kwargs):
        super().__init__(*args, **kwargs)
        self.min_errors = min_errors
        self.min_ratio = error_ratio

    # --- parsing -----------------------------------------------------------

    def parse_line(self, line: str) -> Request | None:
        if not line:
            return None
        m = LINE_RE.match(line)
        if not m:
            return None
        (ts, upstream, status, method, scheme, host, path, ip,
         length, gz, sent_to, ua, referer) = m.groups()
        return Request(
            ts=ts,
            ip=ip,
            method=method,
            path=path,
            status=int(status),
            upstream_status=int(upstream) if upstream.isdigit() else None,
            host=host,
            ua=ua,
            referer=referer,
            extra={"scheme": scheme, "length": length,
                   "gzip": gz, "sent_to": sent_to},
        )

    # --- flagging ----------------------------------------------------------

    def flag(self, stats):
        return flag_scanners(stats, min_errors=self.min_errors,
                             min_ratio=self.min_ratio)

    # --- extras ------------------------------------------------------------

    def extra_summary_rows(self):
        c4 = sum(1 for r in self.requests if r.is_client_error)
        c5 = sum(1 for r in self.requests if r.is_server_error)
        return [
            ("4xx / 5xx", f"{c4} / {c5}"),
            ("hosts seen", len({r.host for r in self.requests if r.host})),
            ("scanner threshold",
             f">= {self.min_errors} errors and ratio >= {self.min_ratio}"),
        ]

    def write_extra(self, data_dir: str) -> list[str]:
        uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-")
        hosts = Counter(r.host for r in self.requests if r.host)
        return [
            write_csv(os.path.join(data_dir, "ua_summary.csv"),
                      ["user_agent", "count"], uas.most_common()),
            write_csv(os.path.join(data_dir, "host_summary.csv"),
                      ["host", "count"], hosts.most_common()),
        ]


def main() -> None:
    parser = arg_parser(
        "analyze NPM proxy logs, flag scanner IPs, export MikroTik rules",
        NPM_LOG_PATH, NPM_REPORT_DIR)
    parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
                        help=f"minimum error responses to flag (default: {DEFAULT_MIN_ERRORS})")
    parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
                        help=f"minimum error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
    parser.add_argument("--full", action="store_true",
                        help="rescan all logs from the start (ignore incremental state)")
    args = parser.parse_args()

    NpmAnalyzer(
        log_path=args.log_path,
        report_dir=args.report_dir,
        top=args.top,
        quiet=args.quiet,
        full=args.full,
        min_errors=args.min_errors,
        error_ratio=args.error_ratio,
    ).run()


if __name__ == "__main__":
    main()