Refactor web_scan.py for improved readability and maintainability; add cron job script for npm log backups; update start.ps1 for consistent formatting.
This commit is contained in:
zwitschi committed
1 parent
0d1cd21468
commit
c2eba88ac8
113 files changed
+772606
-772579
No files matched your search
+170
-170
@@ -1,170 +1,170 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Nginx Proxy Manager access-log analyzer.
|
||||
|
||||
Parses NPM proxy access logs (plain + gzipped), aggregates per-IP
|
||||
behaviour and flags scanner IPs: clients with many error responses and a
|
||||
high error ratio. Flagged IPs become the MikroTik block list.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
from .analyzer import Analyzer
|
||||
from .utils.constants import (
|
||||
DEFAULT_ERROR_RATIO,
|
||||
DEFAULT_MIN_ERRORS,
|
||||
NPM_LOG_PATH,
|
||||
NPM_REPORT_DIR,
|
||||
)
|
||||
from .utils.files import write_csv
|
||||
from .flags import flag_scanners
|
||||
from .utils.models import Request
|
||||
from .utils.utils import arg_parser
|
||||
|
||||
# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path"
|
||||
# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer"
|
||||
LINE_RE = re.compile(
|
||||
r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+" # [datetime]
|
||||
r"-\s+"
|
||||
r"(\d+|-)\s+" # upstream
|
||||
r"(\d+)\s+" # status
|
||||
r"-\s+"
|
||||
r"(\S+)\s+" # method
|
||||
r"(\S+)\s+" # scheme
|
||||
r"(\S+)\s+" # host
|
||||
r'"([^"]*)"\s+' # "path"
|
||||
r"\[Client\s+(\S+)\]\s+"
|
||||
r"\[Length\s+(\d+|-)\]\s+"
|
||||
r"\[Gzip\s+(\S+)\]\s+"
|
||||
r"\[Sent-to\s+(\S+)\]\s+"
|
||||
r'"([^"]*)"\s+' # "UA"
|
||||
r'"([^"]*)"' # "Referer"
|
||||
)
|
||||
|
||||
|
||||
class NpmAnalyzer(Analyzer):
|
||||
name = "npm"
|
||||
title = "Nginx Proxy Manager log analysis report"
|
||||
log_suffixes = (".log", ".gz")
|
||||
exclude_prefixes = ("letsencrypt",)
|
||||
paths_file = "error_paths.csv"
|
||||
paths_label = "error paths"
|
||||
flag_files = ("scanner_ips.txt", "scanner_ipv6.txt")
|
||||
flag_label = "Scanner IPs"
|
||||
flag_note = "high error rate -- probable scanners/attackers"
|
||||
|
||||
ip_columns = (
|
||||
("ip", lambda s: s.ip),
|
||||
("total", lambda s: s.total),
|
||||
("4xx", lambda s: s.c4xx),
|
||||
("5xx", lambda s: s.c5xx),
|
||||
("ratio", lambda s: s.error_ratio),
|
||||
("first", lambda s: s.first),
|
||||
("last", lambda s: s.last),
|
||||
("methods", lambda s: s.methods_str),
|
||||
("whitelist", lambda s: s.whitelist),
|
||||
)
|
||||
|
||||
request_columns = (
|
||||
("datetime", lambda r: r.ts),
|
||||
("ip", lambda r: r.ip),
|
||||
("method", lambda r: r.method),
|
||||
("scheme", lambda r: r.extra.get("scheme")),
|
||||
("host", lambda r: r.host),
|
||||
("path", lambda r: r.path),
|
||||
("status", lambda r: r.status),
|
||||
("upstream_status", lambda r: r.upstream_status),
|
||||
("length", lambda r: r.extra.get("length")),
|
||||
("gzip", lambda r: r.extra.get("gzip")),
|
||||
("sent_to", lambda r: r.extra.get("sent_to")),
|
||||
("referer", lambda r: r.referer),
|
||||
("ua", lambda r: r.ua),
|
||||
)
|
||||
|
||||
def __init__(self, *args, min_errors: int = DEFAULT_MIN_ERRORS,
|
||||
error_ratio: float = DEFAULT_ERROR_RATIO, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.min_errors = min_errors
|
||||
self.min_ratio = error_ratio
|
||||
|
||||
# --- parsing -----------------------------------------------------------
|
||||
|
||||
def parse_line(self, line: str) -> Request | None:
|
||||
if not line:
|
||||
return None
|
||||
m = LINE_RE.match(line)
|
||||
if not m:
|
||||
return None
|
||||
(ts, upstream, status, method, scheme, host, path, ip,
|
||||
length, gz, sent_to, ua, referer) = m.groups()
|
||||
return Request(
|
||||
ts=ts,
|
||||
ip=ip,
|
||||
method=method,
|
||||
path=path,
|
||||
status=int(status),
|
||||
upstream_status=int(upstream) if upstream.isdigit() else None,
|
||||
host=host,
|
||||
ua=ua,
|
||||
referer=referer,
|
||||
extra={"scheme": scheme, "length": length,
|
||||
"gzip": gz, "sent_to": sent_to},
|
||||
)
|
||||
|
||||
# --- flagging ----------------------------------------------------------
|
||||
|
||||
def flag(self, stats):
|
||||
return flag_scanners(stats, min_errors=self.min_errors,
|
||||
min_ratio=self.min_ratio)
|
||||
|
||||
# --- extras ------------------------------------------------------------
|
||||
|
||||
def extra_summary_rows(self):
|
||||
c4 = sum(1 for r in self.requests if r.is_client_error)
|
||||
c5 = sum(1 for r in self.requests if r.is_server_error)
|
||||
return [
|
||||
("4xx / 5xx", f"{c4} / {c5}"),
|
||||
("hosts seen", len({r.host for r in self.requests if r.host})),
|
||||
("scanner threshold",
|
||||
f">= {self.min_errors} errors and ratio >= {self.min_ratio}"),
|
||||
]
|
||||
|
||||
def write_extra(self, data_dir: str) -> list[str]:
|
||||
uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-")
|
||||
hosts = Counter(r.host for r in self.requests if r.host)
|
||||
return [
|
||||
write_csv(os.path.join(data_dir, "ua_summary.csv"),
|
||||
["user_agent", "count"], uas.most_common()),
|
||||
write_csv(os.path.join(data_dir, "host_summary.csv"),
|
||||
["host", "count"], hosts.most_common()),
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = arg_parser(
|
||||
"analyze NPM proxy logs, flag scanner IPs, export MikroTik rules",
|
||||
NPM_LOG_PATH, NPM_REPORT_DIR)
|
||||
parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
|
||||
help=f"minimum error responses to flag (default: {DEFAULT_MIN_ERRORS})")
|
||||
parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
|
||||
help=f"minimum error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
|
||||
parser.add_argument("--full", action="store_true",
|
||||
help="rescan all logs from the start (ignore incremental state)")
|
||||
args = parser.parse_args()
|
||||
|
||||
NpmAnalyzer(
|
||||
log_path=args.log_path,
|
||||
report_dir=args.report_dir,
|
||||
top=args.top,
|
||||
quiet=args.quiet,
|
||||
full=args.full,
|
||||
min_errors=args.min_errors,
|
||||
error_ratio=args.error_ratio,
|
||||
).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
#!/usr/bin/env python3
|
||||
"""Nginx Proxy Manager access-log analyzer.
|
||||
|
||||
Parses NPM proxy access logs (plain + gzipped), aggregates per-IP
|
||||
behaviour and flags scanner IPs: clients with many error responses and a
|
||||
high error ratio. Flagged IPs become the MikroTik block list.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
from collections import Counter
|
||||
|
||||
from .analyzer import Analyzer
|
||||
from .utils.constants import (
|
||||
DEFAULT_ERROR_RATIO,
|
||||
DEFAULT_MIN_ERRORS,
|
||||
NPM_LOG_PATH,
|
||||
NPM_REPORT_DIR,
|
||||
)
|
||||
from .utils.files import write_csv
|
||||
from .flags import flag_scanners
|
||||
from .utils.models import Request
|
||||
from .utils.utils import arg_parser
|
||||
|
||||
# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path"
|
||||
# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer"
|
||||
LINE_RE = re.compile(
|
||||
r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+" # [datetime]
|
||||
r"-\s+"
|
||||
r"(\d+|-)\s+" # upstream
|
||||
r"(\d+)\s+" # status
|
||||
r"-\s+"
|
||||
r"(\S+)\s+" # method
|
||||
r"(\S+)\s+" # scheme
|
||||
r"(\S+)\s+" # host
|
||||
r'"([^"]*)"\s+' # "path"
|
||||
r"\[Client\s+(\S+)\]\s+"
|
||||
r"\[Length\s+(\d+|-)\]\s+"
|
||||
r"\[Gzip\s+(\S+)\]\s+"
|
||||
r"\[Sent-to\s+(\S+)\]\s+"
|
||||
r'"([^"]*)"\s+' # "UA"
|
||||
r'"([^"]*)"' # "Referer"
|
||||
)
|
||||
|
||||
|
||||
class NpmAnalyzer(Analyzer):
|
||||
name = "npm"
|
||||
title = "Nginx Proxy Manager log analysis report"
|
||||
log_suffixes = (".log", ".gz")
|
||||
exclude_prefixes = ("letsencrypt",)
|
||||
paths_file = "error_paths.csv"
|
||||
paths_label = "error paths"
|
||||
flag_files = ("scanner_ips.txt", "scanner_ipv6.txt")
|
||||
flag_label = "Scanner IPs"
|
||||
flag_note = "high error rate -- probable scanners/attackers"
|
||||
|
||||
ip_columns = (
|
||||
("ip", lambda s: s.ip),
|
||||
("total", lambda s: s.total),
|
||||
("4xx", lambda s: s.c4xx),
|
||||
("5xx", lambda s: s.c5xx),
|
||||
("ratio", lambda s: s.error_ratio),
|
||||
("first", lambda s: s.first),
|
||||
("last", lambda s: s.last),
|
||||
("methods", lambda s: s.methods_str),
|
||||
("whitelist", lambda s: s.whitelist),
|
||||
)
|
||||
|
||||
request_columns = (
|
||||
("datetime", lambda r: r.ts),
|
||||
("ip", lambda r: r.ip),
|
||||
("method", lambda r: r.method),
|
||||
("scheme", lambda r: r.extra.get("scheme")),
|
||||
("host", lambda r: r.host),
|
||||
("path", lambda r: r.path),
|
||||
("status", lambda r: r.status),
|
||||
("upstream_status", lambda r: r.upstream_status),
|
||||
("length", lambda r: r.extra.get("length")),
|
||||
("gzip", lambda r: r.extra.get("gzip")),
|
||||
("sent_to", lambda r: r.extra.get("sent_to")),
|
||||
("referer", lambda r: r.referer),
|
||||
("ua", lambda r: r.ua),
|
||||
)
|
||||
|
||||
def __init__(self, *args, min_errors: int = DEFAULT_MIN_ERRORS,
|
||||
error_ratio: float = DEFAULT_ERROR_RATIO, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.min_errors = min_errors
|
||||
self.min_ratio = error_ratio
|
||||
|
||||
# --- parsing -----------------------------------------------------------
|
||||
|
||||
def parse_line(self, line: str) -> Request | None:
|
||||
if not line:
|
||||
return None
|
||||
m = LINE_RE.match(line)
|
||||
if not m:
|
||||
return None
|
||||
(ts, upstream, status, method, scheme, host, path, ip,
|
||||
length, gz, sent_to, ua, referer) = m.groups()
|
||||
return Request(
|
||||
ts=ts,
|
||||
ip=ip,
|
||||
method=method,
|
||||
path=path,
|
||||
status=int(status),
|
||||
upstream_status=int(upstream) if upstream.isdigit() else None,
|
||||
host=host,
|
||||
ua=ua,
|
||||
referer=referer,
|
||||
extra={"scheme": scheme, "length": length,
|
||||
"gzip": gz, "sent_to": sent_to},
|
||||
)
|
||||
|
||||
# --- flagging ----------------------------------------------------------
|
||||
|
||||
def flag(self, stats):
|
||||
return flag_scanners(stats, min_errors=self.min_errors,
|
||||
min_ratio=self.min_ratio)
|
||||
|
||||
# --- extras ------------------------------------------------------------
|
||||
|
||||
def extra_summary_rows(self):
|
||||
c4 = sum(1 for r in self.requests if r.is_client_error)
|
||||
c5 = sum(1 for r in self.requests if r.is_server_error)
|
||||
return [
|
||||
("4xx / 5xx", f"{c4} / {c5}"),
|
||||
("hosts seen", len({r.host for r in self.requests if r.host})),
|
||||
("scanner threshold",
|
||||
f">= {self.min_errors} errors and ratio >= {self.min_ratio}"),
|
||||
]
|
||||
|
||||
def write_extra(self, data_dir: str) -> list[str]:
|
||||
uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-")
|
||||
hosts = Counter(r.host for r in self.requests if r.host)
|
||||
return [
|
||||
write_csv(os.path.join(data_dir, "ua_summary.csv"),
|
||||
["user_agent", "count"], uas.most_common()),
|
||||
write_csv(os.path.join(data_dir, "host_summary.csv"),
|
||||
["host", "count"], hosts.most_common()),
|
||||
]
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = arg_parser(
|
||||
"analyze NPM proxy logs, flag scanner IPs, export MikroTik rules",
|
||||
NPM_LOG_PATH, NPM_REPORT_DIR)
|
||||
parser.add_argument("--min-errors", type=int, default=DEFAULT_MIN_ERRORS,
|
||||
help=f"minimum error responses to flag (default: {DEFAULT_MIN_ERRORS})")
|
||||
parser.add_argument("--error-ratio", type=float, default=DEFAULT_ERROR_RATIO,
|
||||
help=f"minimum error ratio to flag (default: {DEFAULT_ERROR_RATIO})")
|
||||
parser.add_argument("--full", action="store_true",
|
||||
help="rescan all logs from the start (ignore incremental state)")
|
||||
args = parser.parse_args()
|
||||
|
||||
NpmAnalyzer(
|
||||
log_path=args.log_path,
|
||||
report_dir=args.report_dir,
|
||||
top=args.top,
|
||||
quiet=args.quiet,
|
||||
full=args.full,
|
||||
min_errors=args.min_errors,
|
||||
error_ratio=args.error_ratio,
|
||||
).run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user