From 15019de3d8423441afd45261e2dc973497cbae32 Mon Sep 17 00:00:00 2001 From: zwitschi Date: Mon, 3 Aug 2026 16:34:45 +0200 Subject: [PATCH] Initial commit: intrusion detection system for copyparty and NPM proxy logs with MikroTik blocklist export --- .gitignore | 14 + README.md | 246 ++++++++++ bad_patterns.py | 190 ++++++++ copyparty.py | 175 +++++++ data/README.md | 10 + data/cloudflare/.gitkeep | 1 + data/cloudflare/ips-v4.txt | 15 + data/cloudflare/ips-v6.txt | 7 + data/openfactory/.gitkeep | 1 + data/openfactory/geofeed.csv | 46 ++ data/paloalto/.gitkeep | 1 + data/paloalto/scanner-ips.txt | 12 + npm.py | 163 +++++++ reports/mikrotik-block.rsc | 602 +++++++++++++++++++++++ reports/whois_cache.json | 867 +++++++++++++++++++++++++++++++++ scan.py | 141 ++++++ utils.py | 886 ++++++++++++++++++++++++++++++++++ whitelists.py | 284 +++++++++++ 18 files changed, 3661 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 bad_patterns.py create mode 100644 copyparty.py create mode 100644 data/README.md create mode 100644 data/cloudflare/.gitkeep create mode 100644 data/cloudflare/ips-v4.txt create mode 100644 data/cloudflare/ips-v6.txt create mode 100644 data/openfactory/.gitkeep create mode 100644 data/openfactory/geofeed.csv create mode 100644 data/paloalto/.gitkeep create mode 100644 data/paloalto/scanner-ips.txt create mode 100644 npm.py create mode 100644 reports/mikrotik-block.rsc create mode 100644 reports/whois_cache.json create mode 100644 scan.py create mode 100644 utils.py create mode 100644 whitelists.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..7aa6b66 --- /dev/null +++ b/.gitignore @@ -0,0 +1,14 @@ +# instructions for agents +.github/instructions/ + +# environment variables +.env + +# python cache +__pycache__/ + +# reports data +reports/copyparty/data/ +reports/npm/data/ +reports/copyparty/**/*.txt +reports/npm/**/*.txt \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..4ece0da --- /dev/null +++ b/README.md @@ -0,0 +1,246 @@ +# allucanget-ids + +Local-development intrusion detection / log analysis for two log sources: + +| Script | Source | Detection rule | +| --- | --- | --- | +| `copyparty.py` | [copyparty](https://github.com/9001/copyparty) server logs | denied-only IPs + copyparty auto-bans | +| `npm.py` | [Nginx Proxy Manager](https://nginxproxymanager.com) proxy logs | high error-rate IPs (scanners) | +| `scan.py` | both of the above | consolidated scan + unified MikroTik export | +| `whitelists.py` | whitelist module | provider CIDR loading, refresh, validation | +| `bad_patterns.py` | pattern detection | malicious URL/file regex classification | +| `utils.py` | shared core (no CLI) | aggregation, reports, rsc, pipeline | + +Both per-source scripts parse logs, aggregate per-IP behaviour, split +IPv4/IPv6, skip whitelisted provider ranges, detect malicious path patterns, +group flagged IPs into WHOIS subnets, and emit a ready-to-import MikroTik +RouterOS firewall script. + +`scan.py` runs both analyzers, merges their blocklists, and writes a single +consolidated `.rsc`. + +> Status: **local dev**. Runs offline against log files on disk. It does not +> talk to the router, does not run as a service, and blocks nothing by itself. +> You review the generated `.rsc` and import it manually. + +## Architecture + +``` +allucanget-ids/ +├── scan.py consolidated scanner entry point +├── copyparty.py copyparty log adapter (~150 lines) +├── npm.py NPM proxy log adapter (~150 lines) +├── whitelists.py whitelist constants, download, CIDR extraction +├── bad_patterns.py malicious URL/file regex patterns +├── utils.py shared core: aggregation, reporting, MikroTik export +├── data/ whitelist source files (cached locally) +│ ├── openfactory/ +│ │ └── geofeed.csv +│ ├── cloudflare/ +│ │ ├── ips-v4.txt +│ │ └── ips-v6.txt +│ └── paloalto/ +│ └── scanner-ips.txt +├── reports/ generated reports (per-run + per-source) +│ ├── copyparty/ +│ ├── npm/ +│ └── mikrotik-block.rsc consolidated .rsc (scan.py only) +└── README.md +``` + +`copyparty.py` and `npm.py` are thin adapters. Each supplies a log matcher, +a line parser, a flagging rule, and column definitions. Everything else lives in +`utils.py`: + +- `Request` — normalised request record +- `IPStats` + `aggregate()` — per-IP counters, malicious-hit tracking +- `flag_scanners()` / `flag_denied_only()` / `flag_malicious_patterns()` — detection rules +- `whitelist_of()` / `is_whitelisted()` — whitelist lookups (backed by `whitelists.py`) +- `is_malicious()` / `classify_path()` — pattern classification (backed by `bad_patterns.py`) +- `query_whois_subnet()` / `consolidate_by_subnet()` — WHOIS subnet grouping +- IP version helpers, log discovery, transparent `.gz`, ANSI stripping +- CSV/JSON/table writers, `mikrotik_script()` — RouterOS export +- `Analyzer` — pipeline: discover → load → analyze → report → write + +Console output and saved reports are the **same string** (`Analyzer.text_report()`). + +## Requirements + +- Python 3.10+ +- stdlib only, no dependencies + +## Usage + +```powershell +# individual scanners +python copyparty.py +python npm.py + +# consolidated (both sources, unified blocklist) +python scan.py +``` + +All scripts accept CLI overrides: + +| Flag | Default | Meaning | +| --- | --- | --- | +| `--log-path` | see below | log directory | +| `--report-dir` | `reports/` | output directory | +| `--top N` | `30` | rows in the top-IP table | +| `--quiet` | off | write files, print only paths | +| `--no-rsc` | off | skip MikroTik export | +| `--min-errors N` | `5` (`npm.py`) | error count threshold | +| `--error-ratio F` | `0.5` (`npm.py`) | error ratio threshold | +| `--min-denied N` | `1` (`copyparty.py`) | denied-hit threshold | + +`scan.py` accepts all flags from both scanners plus `--copyparty-log-path`, +`--copyparty-report-dir`, `--npm-log-path`, `--npm-report-dir`, and +`--report-dir` for the consolidated output. + +Default log paths (overridable by `.env`): + +| Script | Env var | Default | +| --- | --- | --- | +| `copyparty.py` | `COPYPARTY_LOG_PATH` | `P:\copyparty` | +| `npm.py` | `NPM_LOG_PATH` | `Y:\npm\logs` | + +Examples: + +```powershell +python npm.py --log-path D:\logs\npm --min-errors 10 --error-ratio 0.8 +python copyparty.py --top 100 --no-rsc +python scan.py --quiet --no-rsc +``` + +## Output + +Each script writes to its own report directory. `scan.py` also writes a +consolidated `.rsc`: + +```text +reports/ +├── copyparty/ +│ ├── data/ +│ │ ├── ips_ipv4.csv per-IP: total, denied, banned, users +│ │ ├── ips_ipv6.csv +│ │ ├── ips.json full per-IP detail w/ malicious hits +│ │ ├── denied_paths.csv denied paths, hit counts +│ │ ├── denied_ips.txt flagged IPv4 + metadata +│ │ ├── denied_ipv6.txt +│ │ ├── banned_ips.txt copyparty auto-banned IPv4 +│ │ ├── banned_ipv6.txt +│ │ ├── blocked_ipv4.txt final block set (non-whitelisted) +│ │ ├── blocked_ipv6.txt +│ │ ├── authed_users.json authenticated user counts +│ │ ├── requests.csv every parsed request +│ │ └── mikrotik-block.rsc +│ └── 2026-08-02-12-00-00.txt timestamped report +├── npm/ +│ ├── data/ +│ │ ├── ips_ipv4.csv per-IP: total, 4xx, 5xx, ratio, methods +│ │ ├── ips_ipv6.csv +│ │ ├── ips.json +│ │ ├── error_paths.csv +│ │ ├── scanner_ips.txt flagged IPv4 + metadata +│ │ ├── scanner_ipv6.txt +│ │ ├── blocked_ipv4.txt +│ │ ├── blocked_ipv6.txt +│ │ ├── ua_summary.csv user agent breakdown +│ │ ├── host_summary.csv virtual host breakdown +│ │ ├── requests.csv +│ │ └── mikrotik-block.rsc +│ └── 2026-08-02-12-00-00.txt +└── mikrotik-block.rsc consolidated (scan.py only) +``` + +## Detection + +### Three detection layers + +Blocklist candidates come from three independent sources, merged at scan time: + +1. **Error-rate flagging** (`npm.py`) — IPs with many 4xx/5xx responses and a + high error ratio: + + $$\text{errors} \geq 5 \qquad \text{and} \qquad \frac{\text{errors}}{\text{total}} \geq 0.5$$ + +2. **Denied-only flagging** (`copyparty.py`) — IPs that hit denied paths and + never authenticated: + + $$\text{block} = \text{denied-only} \cup \text{banned} \setminus \text{whitelist}$$ + +3. **Malicious-pattern flagging** (both analyzers) — IPs whose request paths + match known probe/attack signatures (`.env`, `.git/config`, WordPress + probes, webshell filenames, etc.) at severity ≥ 2. + +### Pattern categories (`bad_patterns.py`) + +| Pattern | Severity | Matches | +| --- | --- | --- | +| `env_file` | 3 (critical) | `.env`, `.env.production`, `.aws/credentials`, `config.json` | +| `vcs_probe` | 3 (critical) | `.git/config`, `.svn/` | +| `sensitive_file` | 3 (critical) | `id_rsa`, `id_dsa`, `.pem`, `.key`, `.ppk` | +| `webshell_probe` | 3 (critical) | Known webshell filenames (xstelth, adminfuns, wp_filemanager, etc.) | +| `wp_probe` | 2 (high) | `wp-admin`, `wp-login`, `wp-content`, `wp-includes` | +| `generic_php_probe` | 1 (moderate) | Single/double-char/numeric `.php` files | +| `well_known_probe` | 1 (moderate) | `.well-known/` paths | + +Flagging metadata fields: + +- `errors` / `denied` — hostile hit count +- `total` — all requests from that IP +- `ratio` — error ratio (npm only) +- `first` / `last` — first and last timestamp seen +- `malicious_hits` — count of malicious-pattern request paths + +## Whitelisted ranges + +Whitelists live in `whitelists.py` and `data/`. Loading priority per provider: + +1. **Local files** in `data//` (offline-safe, committed to repo) +2. **Remote download** (fetches latest, saves to local file on success) +3. **Built-in fallback** (hardcoded constants, always available) + +Each layer overrides the previous. Known-good ranges are emitted as +`accept` rules placed **above** the `drop` rules in the MikroTik script. + +### Subnet consolidation + +Blocked IPs are grouped by WHOIS subnet (via `whois.radb.net`). If ≥ 3 +malicious IPs share a subnet, the entire subnet is blocked instead of +individual `/32` or `/128` entries. Results are cached to +`reports/whois_cache.json`. + +## Log format reference + +### copyparty + +```log +15:55:47.466 34.105.30.251 37140 LOG: GET //2020/wp-includes/wlwmanifest.xml @* +15:55:47.466 34.105.30.251 37140 LOG: @* has no access to '/2020/wp-includes/wlwmanifest.xml' +15:55:47.466 34.105.30.251 37140 CRIT: client banned: 403s +15:55:47.695 34.105.30.251 37144 BTW: banned for 86399 sec +16:00:34.385 hsrv LOG: ok bye +``` + +### Nginx Proxy Manager + +```log +[02/Aug/2026:05:01:31 +0200] - 401 401 - GET https example.tld "/wp-content/plugins/x.php" [Client 72.146.20.230] [Length 2127] [Gzip -] [Sent-to 192.168.88.12] "-" "-" +[02/Aug/2026:09:01:11 +0200] - 200 200 - GET https example.tld "/" [Client 2a00:6340:2008:ec80::1] [Length 1651] [Gzip 2.97] [Sent-to 192.168.88.12] "Mozilla/5.0 ..." "-" +``` + +Fields: `[datetime]`, upstream status, proxy status, method, scheme, host, +`"path"`, `[Client IP]`, `[Length N]`, `[Gzip X]`, `[Sent-to IP]`, `"UA"`, +`"Referer"`. + +## Extending + +- **new whitelist provider** → add URLs, files, and fallback CIDRs to + `whitelists.py` +- **new detection rule** → add a `BadPattern` to `bad_patterns.py` or a + `flag_*()` function to `utils.py`, return it from the analyzer's `flag()` +- **new log source** → subclass `utils.Analyzer`, implement `parse_line()` + + `flag()`, declare `ip_columns` / `request_columns` +- **new output file** → override `write_extra(data_dir)` and return the paths + written diff --git a/bad_patterns.py b/bad_patterns.py new file mode 100644 index 0000000..35e4864 --- /dev/null +++ b/bad_patterns.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 +"""Malicious-pattern detection rules for the intrusion-detection system. + +Defines compiled regex patterns and classifiers that identify +probing/scanning URLs (e.g. env files, wp-admin, webshells) and +suspicious file names. Imported by analyzers to flag IPs hitting +these patterns as blocklist candidates. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from typing import ClassVar, Pattern + +# --------------------------------------------------------------------------- +# Pattern categories +# --------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class BadPattern: + """A single detection rule. + + Attributes: + name: Short identifier (e.g. "env_file", "wp_admin"). + label: Human-readable description for reports. + regex: Compiled regex matched against the request path. + severity: 1 = moderate, 2 = high, 3 = critical (auto-block). + """ + + name: str + label: str + regex: Pattern[str] + severity: int = 2 + + def matches(self, path: str) -> bool: + return bool(self.regex.search(path)) + + +# --------------------------------------------------------------------------- +# Compiled pattern bank +# --------------------------------------------------------------------------- + +# -- env / config file probes ------------------------------------------------- + +ENV_FILE = BadPattern( + "env_file", + "env/config file probe (.env, .aws/credentials, config.json)", + re.compile( + r"/\.(?:env|config)(?:\.\w+)*|" + r"/\.aws/credentials|" + r"/config\.json", + re.IGNORECASE, + ), + severity=3, +) + +# -- version control probes --------------------------------------------------- + +VCS_PROBE = BadPattern( + "vcs_probe", + "version-control probe (.git, .svn)", + re.compile( + r"/\.(?:git|svn)(?:/|$)", + re.IGNORECASE, + ), + severity=3, +) + +# -- WordPress / CMS probes --------------------------------------------------- + +WP_PROBE = BadPattern( + "wp_probe", + "WordPress/CMS probe (wp-admin, wp-login, wp-content, wp-includes)", + re.compile( + r"/wp-(?:admin|login|content|includes|load|cron|xmlrpc)(?:/|\.php)", + re.IGNORECASE, + ), + severity=2, +) + +# -- well-known / common probe paths ------------------------------------------ + +WELL_KNOWN_PROBE = BadPattern( + "well_known_probe", + ".well-known probe", + re.compile( + r"/\.well-known/", + re.IGNORECASE, + ), + severity=1, +) + +# -- webshell / backdoor probes ----------------------------------------------- + +WEBSHELL_PROBE = BadPattern( + "webshell_probe", + "webshell/backdoor probe (known shell filenames, phpinfo, file managers)", + re.compile( + r"/(?:" + r"phpinfo|info|shell|cmd|upload|" + r"wp_filemanager|filemanager|" + r"xstelth|adminfuns|hellopress|" + r"fpwch|crgio|coffexium|" + r"ccou|biufile|" + r"ms-edit|classwithtostring|" + r"this_is_a_new_hello_world|" + r"chosen|ops|simple|inputs" + r")\.php", + re.IGNORECASE, + ), + severity=3, +) + +# -- suspicious generic PHP file requests ------------------------------------- + +GENERIC_PHP_PROBE = BadPattern( + "generic_php_probe", + "generic/random PHP file probe (single-char, numeric names)", + re.compile( + r"/(?:" + r"[a-z]\.php|" # single letter: a.php, k.php, x.php + r"\d{1,3}\.php|" # numeric: 1.php, 222.php, 155.php + r"[a-z]{2}\.php|" # two-letter: aa.php, dx.php + r"a\d\.php|" # a1.php + r"\w+\.php" # generic: anything.php + r")", + re.IGNORECASE, + ), + severity=1, +) + +# -- sensitive file probes ---------------------------------------------------- + +SENSITIVE_FILE = BadPattern( + "sensitive_file", + "sensitive file probe (id_rsa, id_dsa, .pem, .key)", + re.compile( + r"/(?:id_rsa|id_dsa|id_ecdsa|id_ed25519|.*\.(?:pem|key|ppk|pfx|p12))", + re.IGNORECASE, + ), + severity=3, +) + +# -- suspicious user-agent (optional, path-based only for now) ----------------- + + +# --------------------------------------------------------------------------- +# All active patterns (ordered by severity, highest first) +# --------------------------------------------------------------------------- + +ALL_PATTERNS: list[BadPattern] = [ + ENV_FILE, + VCS_PROBE, + SENSITIVE_FILE, + WEBSHELL_PROBE, + WP_PROBE, + GENERIC_PHP_PROBE, + WELL_KNOWN_PROBE, +] + +# Quick lookup by name +PATTERNS_BY_NAME: dict[str, BadPattern] = {p.name: p for p in ALL_PATTERNS} + + +# --------------------------------------------------------------------------- +# Classification API +# --------------------------------------------------------------------------- + + +def classify_path(path: str) -> list[BadPattern]: + """Return all patterns that match a given request path.""" + return [p for p in ALL_PATTERNS if p.matches(path)] + + +def is_malicious(path: str) -> bool: + """Return True if any pattern matches the path.""" + return any(p.matches(path) for p in ALL_PATTERNS) + + +def malicious_severity(path: str) -> int: + """Return the highest severity among matching patterns, or 0 if none.""" + matched = classify_path(path) + return max((p.severity for p in matched), default=0) + + +def pattern_names(path: str) -> list[str]: + """Return names of all patterns matching the path.""" + return [p.name for p in classify_path(path)] diff --git a/copyparty.py b/copyparty.py new file mode 100644 index 0000000..e2b35f4 --- /dev/null +++ b/copyparty.py @@ -0,0 +1,175 @@ +#!/usr/bin/env python3 +"""copyparty access-log analyzer. + +Parses copyparty server logs, aggregates per-IP behaviour and flags +"denied-only" clients: IPs that never authenticated and only ever hit +`@* has no access to '...'`. Those plus copyparty's own auto-bans become +the MikroTik block list. +""" + +from __future__ import annotations + +import os +import re + +import utils +from utils import Request + +DEFAULT_LOG_PATH = os.environ.get("COPYPARTY_LOG_PATH", r"P:\copyparty") +DEFAULT_REPORT_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "reports", "copyparty") + +# "HH:MM:SS.mmm LEVEL: message" +# System lines ("hsrv", "root", ...) have no port and are skipped. +LINE_RE = re.compile( + r"^(\d{2}:\d{2}:\d{2}\.\d{3})\s+" # timestamp + r"(\S+)\s+" # ip + r"(\d+)\s+" # port + r"(LOG|WARN|CRIT|BTW|INFO|ERR|DEBUG):\s" # level + r"(.*)$" # message +) + +METHOD_RE = re.compile( + r"^(GET|HEAD|POST|PUT|PATCH|DELETE|OPTIONS|PROPFIND|PROPPATCH|MKCOL|" + r"MOVE|COPY|LOCK|UNLOCK)\s+(\S+)(?:\s+@(\S+))?") +DENY_RE = re.compile(r"^@(\S+) has no access to '([^']+)'") +CLIENT_BAN_RE = re.compile(r"client banned") +BAN_SECS_RE = re.compile(r"banned for (\d+) sec") + + +class CopypartyAnalyzer(utils.Analyzer): + name = "copyparty" + title = "copyparty log analysis report" + log_pattern = re.compile(r"\d{4}-\d{2}-\d{2}\.txt(\.\d+)?$") + paths_file = "denied_paths.csv" + paths_label = "denied paths" + flag_files = ("denied_ips.txt", "denied_ipv6.txt") + flag_label = "Denied-only IPs" + flag_note = "never authenticated -- ban candidates" + + ip_columns = ( + ("ip", lambda s: s.ip), + ("total", lambda s: s.total), + ("denied", lambda s: s.denied), + ("banned", lambda s: s.banned), + ("first", lambda s: s.first), + ("last", lambda s: s.last), + ("users", lambda s: s.users_str), + ("whitelist", lambda s: s.whitelist), + ) + + request_columns = ( + ("time", lambda r: r.ts), + ("ip", lambda r: r.ip), + ("port", lambda r: r.extra.get("port")), + ("level", lambda r: r.extra.get("level")), + ("method", lambda r: r.method), + ("path", lambda r: r.path), + ("user", lambda r: r.user), + ("denied", lambda r: r.denied), + ("banned", lambda r: r.banned), + ("ban_secs", lambda r: r.extra.get("ban_secs")), + ) + + def __init__(self, *args, min_denied: int = 1, **kwargs): + super().__init__(*args, **kwargs) + self.min_denied = min_denied + + # --- parsing ----------------------------------------------------------- + + def parse_line(self, line: str) -> Request | None: + if not line: + return None + m = LINE_RE.match(line) + if not m: + return None + ts, ip, port, level, msg = m.groups() + req = Request(ts=ts, ip=ip, user=None, + extra={"port": int(port), "level": level}) + + mm = METHOD_RE.match(msg) + if mm: + req.method, req.path, user = mm.groups() + req.user = user or "*" + return req + + dm = DENY_RE.match(msg) + if dm: + req.user, req.path = dm.groups() + req.denied = True + return req + + if CLIENT_BAN_RE.search(msg): + req.banned = True + bs = BAN_SECS_RE.search(msg) + if bs: + req.extra["ban_secs"] = int(bs.group(1)) + return req + + bs = BAN_SECS_RE.search(msg) + if bs: + req.banned = True + req.extra["ban_secs"] = int(bs.group(1)) + return req + + # --- flagging ---------------------------------------------------------- + + def flag(self, stats): + return utils.flag_denied_only(stats, min_denied=self.min_denied) + + def flag_comment(self, s): + return (f"denied={s.denied} total={s.total} " + f"first={s.first} last={s.last}") + + def block_candidates(self) -> set[str]: + banned = {ip for ip, s in self.stats.items() if s.banned} + return set(self.flagged) | banned + + # --- extras ------------------------------------------------------------ + + def banned_ips(self) -> dict[str, utils.IPStats]: + return {ip: s for ip, s in self.stats.items() if s.banned} + + def authed_users(self) -> dict[str, int]: + counts: dict[str, int] = {} + for r in self.requests: + if r.user and r.user != "*": + counts[r.user] = counts.get(r.user, 0) + 1 + return dict(sorted(counts.items(), key=lambda kv: -kv[1])) + + def extra_summary_rows(self): + return [("denied requests", + sum(1 for r in self.requests if r.denied)), + ("banned IPs", len(self.banned_ips())), + ("authenticated users", self.authed_users() or "{}")] + + def write_extra(self, data_dir: str) -> list[str]: + v4, v6 = utils.split_versions(self.banned_ips()) + return [ + utils.write_lines(os.path.join(data_dir, "banned_ips.txt"), v4), + utils.write_lines(os.path.join(data_dir, "banned_ipv6.txt"), v6), + utils.write_json(os.path.join(data_dir, "authed_users.json"), + self.authed_users()), + ] + + +def main() -> None: + parser = utils.arg_parser( + "analyze copyparty logs, flag denied-only IPs, export MikroTik rules", + DEFAULT_LOG_PATH, DEFAULT_REPORT_DIR) + parser.add_argument("--min-denied", type=int, default=1, + help="minimum denied hits before flagging (default: 1)") + args = parser.parse_args() + + CopypartyAnalyzer( + log_path=args.log_path, + report_dir=args.report_dir, + top=args.top, + quiet=args.quiet, + write_rsc=not args.no_rsc, + min_denied=args.min_denied, + ).run() + + +if __name__ == "__main__": + main() diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000..56a1108 --- /dev/null +++ b/data/README.md @@ -0,0 +1,10 @@ +# Data directory for allucanget-ids + +This directory contains data files used by the `allucanget-ids` Intrusion Detection System application, including whitelists, blocklists, and other configuration files. The data files are organized into subdirectories for easier management and maintenance. + +## Data files + +- [Openfactory Geofeed](./openfactory/geofeed.csv): Contains Openfactory IP addresses and associated geolocation data from `https://openfactory.net/geofeed.csv`. Used for identifying and whitelisting traffic from known Openfactory sources. +- [Cloudflare IPv4 list](./cloudflare/ips-v4.txt): Contains a list of Cloudflare IPv4 addresses from `https://www.cloudflare.com/ips-v4`, used for whitelisting traffic from Cloudflare services. +- [Cloudflare IPv6 list](./cloudflare/ips-v6.txt): Contains a list of Cloudflare IPv6 addresses from `https://www.cloudflare.com/ips-v6`, used for whitelisting traffic from Cloudflare services. +- [PaloAlto Scanner IP list](./paloalto/scanner-ips.txt): List of known PaloAlto scanner IP addresses from `https://docs-cortex.paloaltonetworks.com/r/1/Cortex-Xpanse/Scanning-activity`, used for whitelisting traffic from PaloAlto scanners. diff --git a/data/cloudflare/.gitkeep b/data/cloudflare/.gitkeep new file mode 100644 index 0000000..1cf3322 --- /dev/null +++ b/data/cloudflare/.gitkeep @@ -0,0 +1 @@ +$ = cloudflare diff --git a/data/cloudflare/ips-v4.txt b/data/cloudflare/ips-v4.txt new file mode 100644 index 0000000..626248f --- /dev/null +++ b/data/cloudflare/ips-v4.txt @@ -0,0 +1,15 @@ +173.245.48.0/20 +103.21.244.0/22 +103.22.200.0/22 +103.31.4.0/22 +141.101.64.0/18 +108.162.192.0/18 +190.93.240.0/20 +188.114.96.0/20 +197.234.240.0/22 +198.41.128.0/17 +162.158.0.0/15 +104.16.0.0/13 +104.24.0.0/14 +172.64.0.0/13 +131.0.72.0/22 \ No newline at end of file diff --git a/data/cloudflare/ips-v6.txt b/data/cloudflare/ips-v6.txt new file mode 100644 index 0000000..4584f6a --- /dev/null +++ b/data/cloudflare/ips-v6.txt @@ -0,0 +1,7 @@ +2400:cb00::/32 +2606:4700::/32 +2803:f800::/32 +2405:b500::/32 +2405:8100::/32 +2a06:98c0::/29 +2c0f:f248::/32 \ No newline at end of file diff --git a/data/openfactory/.gitkeep b/data/openfactory/.gitkeep new file mode 100644 index 0000000..25531d6 --- /dev/null +++ b/data/openfactory/.gitkeep @@ -0,0 +1 @@ +$ = openfactory diff --git a/data/openfactory/geofeed.csv b/data/openfactory/geofeed.csv new file mode 100644 index 0000000..9fffd5d --- /dev/null +++ b/data/openfactory/geofeed.csv @@ -0,0 +1,46 @@ +45.11.248.0/23,DE,DE-HE,Frankfurt Am Main, +45.11.250.0/24,NL,NL-NH,Amsterdam, +45.11.251.0/24,AT,AT-9,Vienna, +2a0e:8f04::/32,DE,DE-HE,Frankfurt Am Main, +2a0e:8f05::/32,GB,GB-LND,London, +2a0e:8f06::/32,NL,NL-NH,Amsterdam, +2a0e:8f07::/32,AT,AT-9,Vienna, +5.226.144.0/21,CH,CH-TG,Frauenfeld, +5.226.144.192/26,CH,CH-ZH,Winterthur, +5.226.145.0/26,CH,CH-ZH,Wangen-Bruettisellen, +5.226.145.64/26,LI,LI-07,Schaan, +5.226.145.128/25,CH,CH-ZH,Kloten, +5.226.148.0/29,CH,CH-ZH,Zurich, +5.226.148.8/29,CH,CH-ZH,Kloten, +5.226.148.16/28,CH,CH-ZH,Zurich, +5.226.148.32/27,CH,CH-ZH,Kloten, +5.226.148.64/26,CH,CH-ZH,Zurich, +5.226.148.96/27,CH,CH-ZH,Zurich, +5.226.148.128/25,CH,CH-ZH,Zurich, +5.226.149.0/26,CH,CH-ZH,Zurich, +5.226.149.64/28,CH,CH-LU,Lutzern, +5.226.149.128/26,CH,CH-ZH,Glattfelden, +5.226.149.240/30,CH,CH-SZ,Wollerau, +5.226.150.0/27,CH,CH-ZH,Glattfelden, +5.226.150.32/27,CH,CH-BE,Biel, +5.226.150.152/29,LI,LI-07,Schaan, +5.226.150.160/28,CH,CH-ZH,Winterthur, +5.226.150.224/27,CH,CH-BE,Bern, +5.226.151.0/26,NL,NL-NH,Amsterdam, +5.226.151.64/27,CH,CH-ZH,Ruemlang, +185.86.228.0/23,CH,CH-ZH,Zurich, +195.138.203.0/24,CH,CH-ZH,Zurich, +2a00:6340::/32,CH,CH-TG,Frauenfeld, +2a00:6340:2080::/48,NL,NL-NH,Amsterdam, +2001:67c:2648::/48,CH,CH-ZH,Winterthur, + +# Customer geofeed details on request +# Having issues with your geolocation? send us a ticket! + +45.11.248.94/32,DE,DE-NW,Dortmund, +2a00:6340:2044:200::/56,DE,DE-NW,Dortmund, +2a00:6340:2044:600::/56,DE,DE-BW,Stuttgart, +45.11.248.96/32,DE,DE-BW,Stuttgart, +45.11.248.37/32,DE,DE-BE,Berlin, +2a00:6340:2044:800::/56,DE,DE-BE,Berlin, +5.226.146.49/32,CH,CH-ZH,Winterthur,8405,47.472502,8.779930 diff --git a/data/paloalto/.gitkeep b/data/paloalto/.gitkeep new file mode 100644 index 0000000..f7cb8c1 --- /dev/null +++ b/data/paloalto/.gitkeep @@ -0,0 +1 @@ +$ = paloalto diff --git a/data/paloalto/scanner-ips.txt b/data/paloalto/scanner-ips.txt new file mode 100644 index 0000000..86e3e6b --- /dev/null +++ b/data/paloalto/scanner-ips.txt @@ -0,0 +1,12 @@ +35.203.210.0/23 +144.86.173.0/24 +147.185.132.0/23 +162.216.149.0/24 +162.216.150.0/24 +172.105.147.0/24 +198.235.24.0/24 +205.210.31.0/24 +216.25.88.0/21 +2604:a940:300:5b6:0:0:0:0/64 +2604:a940:301:225:0:0:0:0/64 +2604:a940:302:118:0:0:0:0/64 \ No newline at end of file diff --git a/npm.py b/npm.py new file mode 100644 index 0000000..b4c548c --- /dev/null +++ b/npm.py @@ -0,0 +1,163 @@ +#!/usr/bin/env python3 +"""Nginx Proxy Manager access-log analyzer. + +Parses NPM proxy access logs (plain + gzipped), aggregates per-IP +behaviour and flags scanner IPs: clients with many error responses and a +high error ratio. Flagged IPs become the MikroTik block list. +""" + +from __future__ import annotations + +import os +import re +from collections import Counter + +import utils +from utils import Request + +DEFAULT_LOG_PATH = os.environ.get("NPM_LOG_PATH", r"Y:\npm\logs") +DEFAULT_REPORT_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "reports", "npm") + +# [02/Aug/2026:05:01:31 +0200] - 401 401 - GET https host "/path" +# [Client IP] [Length N] [Gzip X] [Sent-to IP] "UA" "Referer" +LINE_RE = re.compile( + r"^\[(\d{2}/\w{3}/\d{4}:\d{2}:\d{2}:\d{2}\s+[+-]\d{4})\]\s+" # [datetime] + r"-\s+" + r"(\d+|-)\s+" # upstream + r"(\d+)\s+" # status + r"-\s+" + r"(\S+)\s+" # method + r"(\S+)\s+" # scheme + r"(\S+)\s+" # host + r'"([^"]*)"\s+' # "path" + r"\[Client\s+(\S+)\]\s+" + r"\[Length\s+(\d+|-)\]\s+" + r"\[Gzip\s+(\S+)\]\s+" + r"\[Sent-to\s+(\S+)\]\s+" + r'"([^"]*)"\s+' # "UA" + r'"([^"]*)"' # "Referer" +) + + +class NpmAnalyzer(utils.Analyzer): + name = "npm" + title = "Nginx Proxy Manager log analysis report" + log_suffixes = (".log", ".gz") + exclude_prefixes = ("letsencrypt",) + paths_file = "error_paths.csv" + paths_label = "error paths" + flag_files = ("scanner_ips.txt", "scanner_ipv6.txt") + flag_label = "Scanner IPs" + flag_note = "high error rate -- probable scanners/attackers" + + ip_columns = ( + ("ip", lambda s: s.ip), + ("total", lambda s: s.total), + ("4xx", lambda s: s.c4xx), + ("5xx", lambda s: s.c5xx), + ("ratio", lambda s: s.error_ratio), + ("first", lambda s: s.first), + ("last", lambda s: s.last), + ("methods", lambda s: s.methods_str), + ("whitelist", lambda s: s.whitelist), + ) + + request_columns = ( + ("datetime", lambda r: r.ts), + ("ip", lambda r: r.ip), + ("method", lambda r: r.method), + ("scheme", lambda r: r.extra.get("scheme")), + ("host", lambda r: r.host), + ("path", lambda r: r.path), + ("status", lambda r: r.status), + ("upstream_status", lambda r: r.upstream_status), + ("length", lambda r: r.extra.get("length")), + ("gzip", lambda r: r.extra.get("gzip")), + ("sent_to", lambda r: r.extra.get("sent_to")), + ("referer", lambda r: r.referer), + ("ua", lambda r: r.ua), + ) + + def __init__(self, *args, min_errors: int = 5, + error_ratio: float = 0.5, **kwargs): + super().__init__(*args, **kwargs) + self.min_errors = min_errors + self.min_ratio = error_ratio + + # --- parsing ----------------------------------------------------------- + + def parse_line(self, line: str) -> Request | None: + if not line: + return None + m = LINE_RE.match(line) + if not m: + return None + (ts, upstream, status, method, scheme, host, path, ip, + length, gz, sent_to, ua, referer) = m.groups() + return Request( + ts=ts, + ip=ip, + method=method, + path=path, + status=int(status), + upstream_status=int(upstream) if upstream.isdigit() else None, + host=host, + ua=ua, + referer=referer, + extra={"scheme": scheme, "length": length, + "gzip": gz, "sent_to": sent_to}, + ) + + # --- flagging ---------------------------------------------------------- + + def flag(self, stats): + return utils.flag_scanners(stats, min_errors=self.min_errors, + min_ratio=self.min_ratio) + + # --- extras ------------------------------------------------------------ + + def extra_summary_rows(self): + c4 = sum(1 for r in self.requests if r.is_client_error) + c5 = sum(1 for r in self.requests if r.is_server_error) + return [ + ("4xx / 5xx", f"{c4} / {c5}"), + ("hosts seen", len({r.host for r in self.requests if r.host})), + ("scanner threshold", + f">= {self.min_errors} errors and ratio >= {self.min_ratio}"), + ] + + def write_extra(self, data_dir: str) -> list[str]: + uas = Counter(r.ua for r in self.requests if r.ua and r.ua != "-") + hosts = Counter(r.host for r in self.requests if r.host) + return [ + utils.write_csv(os.path.join(data_dir, "ua_summary.csv"), + ["user_agent", "count"], uas.most_common()), + utils.write_csv(os.path.join(data_dir, "host_summary.csv"), + ["host", "count"], hosts.most_common()), + ] + + +def main() -> None: + parser = utils.arg_parser( + "analyze NPM proxy logs, flag scanner IPs, export MikroTik rules", + DEFAULT_LOG_PATH, DEFAULT_REPORT_DIR) + parser.add_argument("--min-errors", type=int, default=5, + help="minimum error responses to flag (default: 5)") + parser.add_argument("--error-ratio", type=float, default=0.5, + help="minimum error ratio to flag (default: 0.5)") + args = parser.parse_args() + + NpmAnalyzer( + log_path=args.log_path, + report_dir=args.report_dir, + top=args.top, + quiet=args.quiet, + write_rsc=not args.no_rsc, + min_errors=args.min_errors, + error_ratio=args.error_ratio, + ).run() + + +if __name__ == "__main__": + main() diff --git a/reports/mikrotik-block.rsc b/reports/mikrotik-block.rsc new file mode 100644 index 0000000..e08c2c1 --- /dev/null +++ b/reports/mikrotik-block.rsc @@ -0,0 +1,602 @@ +# ids firewall export - generated 2026-08-03T16:15:01 + +# Openfactory IPv4 ranges (whitelisted) +/ip firewall address-list +add list=Openfactory address=5.226.144.0/21 + +# Cloudflare IPv4 ranges (whitelisted) +/ip firewall address-list +add list=Cloudflare address=173.245.48.0/20 +add list=Cloudflare address=103.21.244.0/22 +add list=Cloudflare address=103.22.200.0/22 +add list=Cloudflare address=103.31.4.0/22 +add list=Cloudflare address=141.101.64.0/18 +add list=Cloudflare address=108.162.192.0/18 +add list=Cloudflare address=190.93.240.0/20 +add list=Cloudflare address=188.114.96.0/20 +add list=Cloudflare address=197.234.240.0/22 +add list=Cloudflare address=198.41.128.0/17 +add list=Cloudflare address=162.158.0.0/15 +add list=Cloudflare address=104.16.0.0/13 +add list=Cloudflare address=104.24.0.0/14 +add list=Cloudflare address=172.64.0.0/13 +add list=Cloudflare address=131.0.72.0/22 + +# Cloudflare IPv6 ranges (whitelisted) +/ipv6 firewall address-list +add list=Cloudflare address=2400:cb00::/32 +add list=Cloudflare address=2606:4700::/32 +add list=Cloudflare address=2803:f800::/32 +add list=Cloudflare address=2405:b500::/32 +add list=Cloudflare address=2405:8100::/32 +add list=Cloudflare address=2a06:98c0::/29 +add list=Cloudflare address=2c0f:f248::/32 + +# PaloAlto IPv4 ranges (whitelisted) +/ip firewall address-list +add list=PaloAlto address=35.203.210.0/23 +add list=PaloAlto address=144.86.173.0/24 +add list=PaloAlto address=147.185.132.0/23 +add list=PaloAlto address=162.216.149.0/24 +add list=PaloAlto address=162.216.150.0/24 +add list=PaloAlto address=172.105.147.0/24 +add list=PaloAlto address=198.235.24.0/24 +add list=PaloAlto address=205.210.31.0/24 +add list=PaloAlto address=216.25.88.0/21 + +# PaloAlto IPv6 ranges (whitelisted) +/ipv6 firewall address-list +add list=PaloAlto address=2604:a940:300:5b6::/64 +add list=PaloAlto address=2604:a940:301:225::/64 +add list=PaloAlto address=2604:a940:302:118::/64 + +# Blocked IPv4 (508, non-whitelisted) +/ip firewall address-list +add list=Blocked address=2.57.122.173/32 +add list=Blocked address=2.58.56.101/32 +add list=Blocked address=3.27.206.50/32 +add list=Blocked address=3.77.92.222/32 +add list=Blocked address=3.79.41.12/32 +add list=Blocked address=3.101.54.4/32 +add list=Blocked address=3.107.107.18/32 +add list=Blocked address=3.237.185.144/32 +add list=Blocked address=3.250.52.216/32 +add list=Blocked address=4.176.0.0/12 +add list=Blocked address=4.192.0.0/12 +add list=Blocked address=4.208.0.0/12 +add list=Blocked address=4.224.0.0/12 +add list=Blocked address=4.246.70.112/32 +add list=Blocked address=5.34.213.142/32 +add list=Blocked address=5.161.62.209/32 +add list=Blocked address=5.161.225.116/32 +add list=Blocked address=5.253.247.16/32 +add list=Blocked address=5.255.119.161/32 +add list=Blocked address=8.229.1.130/32 +add list=Blocked address=8.229.130.50/32 +add list=Blocked address=8.231.78.19/32 +add list=Blocked address=8.234.157.73/32 +add list=Blocked address=8.234.207.247/32 +add list=Blocked address=8.235.114.162/32 +add list=Blocked address=9.205.16.190/32 +add list=Blocked address=13.57.15.41/32 +add list=Blocked address=13.83.161.80/32 +add list=Blocked address=13.126.86.18/32 +add list=Blocked address=13.221.106.198/32 +add list=Blocked address=15.206.146.240/32 +add list=Blocked address=16.146.99.254/32 +add list=Blocked address=18.142.90.109/32 +add list=Blocked address=18.191.56.164/32 +add list=Blocked address=18.227.73.7/32 +add list=Blocked address=18.236.136.230/32 +add list=Blocked address=20.0.0.0/11 +add list=Blocked address=20.46.121.148/32 +add list=Blocked address=20.46.252.52/32 +add list=Blocked address=20.48.0.0/12 +add list=Blocked address=20.64.0.0/10 +add list=Blocked address=20.150.0.0/15 +add list=Blocked address=20.160.0.0/12 +add list=Blocked address=20.184.150.240/32 +add list=Blocked address=20.192.0.0/10 +add list=Blocked address=20.215.189.213/32 +add list=Blocked address=23.94.155.57/32 +add list=Blocked address=23.95.226.104/32 +add list=Blocked address=23.102.123.231/32 +add list=Blocked address=23.161.169.62/32 +add list=Blocked address=23.161.169.127/32 +add list=Blocked address=23.178.112.0/24 +add list=Blocked address=23.180.120.134/32 +add list=Blocked address=23.180.120.147/32 +add list=Blocked address=23.234.98.171/32 +add list=Blocked address=24.144.85.55/32 +add list=Blocked address=31.56.58.0/24 +add list=Blocked address=31.58.51.130/32 +add list=Blocked address=31.58.51.184/32 +add list=Blocked address=31.59.129.193/32 +add list=Blocked address=31.184.236.8/32 +add list=Blocked address=34.7.160.235/32 +add list=Blocked address=34.7.193.250/32 +add list=Blocked address=34.11.108.163/32 +add list=Blocked address=34.11.127.22/32 +add list=Blocked address=34.17.159.73/32 +add list=Blocked address=34.18.38.137/32 +add list=Blocked address=34.21.144.169/32 +add list=Blocked address=34.24.155.203/32 +add list=Blocked address=34.26.191.118/32 +add list=Blocked address=34.31.88.250/32 +add list=Blocked address=34.40.240.124/32 +add list=Blocked address=34.42.48.100/32 +add list=Blocked address=34.47.183.174/32 +add list=Blocked address=34.47.194.183/32 +add list=Blocked address=34.55.200.45/32 +add list=Blocked address=34.59.187.134/32 +add list=Blocked address=34.64.173.205/32 +add list=Blocked address=34.65.14.246/32 +add list=Blocked address=34.65.18.71/32 +add list=Blocked address=34.65.138.195/32 +add list=Blocked address=34.69.107.154/32 +add list=Blocked address=34.73.128.17/32 +add list=Blocked address=34.75.4.161/32 +add list=Blocked address=34.82.10.34/32 +add list=Blocked address=34.86.126.198/32 +add list=Blocked address=34.86.217.208/32 +add list=Blocked address=34.90.87.32/32 +add list=Blocked address=34.90.88.37/32 +add list=Blocked address=34.91.243.226/32 +add list=Blocked address=34.94.172.72/32 +add list=Blocked address=34.95.239.96/32 +add list=Blocked address=34.105.30.251/32 +add list=Blocked address=34.116.255.65/32 +add list=Blocked address=34.118.184.2/32 +add list=Blocked address=34.131.251.163/32 +add list=Blocked address=34.139.141.56/32 +add list=Blocked address=34.145.62.28/32 +add list=Blocked address=34.146.12.12/32 +add list=Blocked address=34.148.8.63/32 +add list=Blocked address=34.154.122.183/32 +add list=Blocked address=34.158.83.192/32 +add list=Blocked address=34.171.75.208/32 +add list=Blocked address=34.175.82.8/32 +add list=Blocked address=34.181.134.189/32 +add list=Blocked address=34.186.83.134/32 +add list=Blocked address=34.187.20.70/32 +add list=Blocked address=34.187.82.183/32 +add list=Blocked address=34.187.200.119/32 +add list=Blocked address=34.242.226.103/32 +add list=Blocked address=34.244.177.56/32 +add list=Blocked address=35.95.56.230/32 +add list=Blocked address=35.157.251.190/32 +add list=Blocked address=35.182.17.8/32 +add list=Blocked address=35.184.157.121/32 +add list=Blocked address=35.187.9.30/32 +add list=Blocked address=35.192.232.168/32 +add list=Blocked address=35.196.22.146/32 +add list=Blocked address=35.198.180.224/32 +add list=Blocked address=35.199.111.12/32 +add list=Blocked address=35.199.115.51/32 +add list=Blocked address=35.199.183.190/32 +add list=Blocked address=35.201.159.247/32 +add list=Blocked address=35.204.88.252/32 +add list=Blocked address=35.204.141.252/32 +add list=Blocked address=35.230.52.27/32 +add list=Blocked address=35.231.41.225/32 +add list=Blocked address=35.231.100.82/32 +add list=Blocked address=35.237.138.104/32 +add list=Blocked address=35.237.178.50/32 +add list=Blocked address=35.243.173.211/32 +add list=Blocked address=35.243.223.147/32 +add list=Blocked address=35.255.40.65/32 +add list=Blocked address=36.106.166.92/32 +add list=Blocked address=36.106.166.223/32 +add list=Blocked address=37.140.223.6/32 +add list=Blocked address=37.140.223.58/32 +add list=Blocked address=38.248.95.248/32 +add list=Blocked address=41.143.154.1/32 +add list=Blocked address=43.98.201.177/32 +add list=Blocked address=43.228.157.102/32 +add list=Blocked address=43.228.157.191/32 +add list=Blocked address=44.224.0.0/11 +add list=Blocked address=45.8.196.200/32 +add list=Blocked address=45.45.237.69/32 +add list=Blocked address=45.45.237.214/32 +add list=Blocked address=45.61.150.67/32 +add list=Blocked address=45.61.187.30/32 +add list=Blocked address=45.67.85.190/32 +add list=Blocked address=45.88.138.44/32 +add list=Blocked address=45.130.203.154/32 +add list=Blocked address=45.130.203.177/32 +add list=Blocked address=45.131.193.139/32 +add list=Blocked address=45.131.195.0/24 +add list=Blocked address=45.134.20.116/32 +add list=Blocked address=45.134.20.128/32 +add list=Blocked address=45.138.16.114/32 +add list=Blocked address=45.139.104.172/32 +add list=Blocked address=45.146.54.80/32 +add list=Blocked address=45.146.54.137/32 +add list=Blocked address=45.146.55.11/32 +add list=Blocked address=45.146.55.48/32 +add list=Blocked address=45.148.10.0/24 +add list=Blocked address=45.153.34.16/32 +add list=Blocked address=45.153.34.43/32 +add list=Blocked address=45.153.34.182/32 +add list=Blocked address=45.154.98.0/24 +add list=Blocked address=45.154.138.181/32 +add list=Blocked address=45.154.138.232/32 +add list=Blocked address=46.23.108.234/32 +add list=Blocked address=46.232.235.0/24 +add list=Blocked address=47.85.160.47/32 +add list=Blocked address=51.79.208.246/32 +add list=Blocked address=51.103.133.211/32 +add list=Blocked address=51.103.152.96/32 +add list=Blocked address=51.116.233.22/32 +add list=Blocked address=51.120.0.0/16 +add list=Blocked address=52.139.36.144/32 +add list=Blocked address=52.159.244.176/32 +add list=Blocked address=52.160.0.0/11 +add list=Blocked address=52.224.0.0/11 +add list=Blocked address=54.82.69.43/32 +add list=Blocked address=54.163.186.39/32 +add list=Blocked address=54.169.103.174/32 +add list=Blocked address=54.179.129.22/32 +add list=Blocked address=54.206.147.115/32 +add list=Blocked address=54.210.219.163/32 +add list=Blocked address=54.216.227.171/32 +add list=Blocked address=58.212.237.231/32 +add list=Blocked address=62.60.130.128/32 +add list=Blocked address=62.60.130.227/32 +add list=Blocked address=62.93.179.146/32 +add list=Blocked address=64.23.218.208/32 +add list=Blocked address=64.89.161.82/32 +add list=Blocked address=64.89.161.85/32 +add list=Blocked address=64.225.63.129/32 +add list=Blocked address=64.225.75.246/32 +add list=Blocked address=64.227.32.66/32 +add list=Blocked address=65.0.122.218/32 +add list=Blocked address=66.187.5.228/32 +add list=Blocked address=66.187.6.102/32 +add list=Blocked address=66.228.38.221/32 +add list=Blocked address=66.249.74.165/32 +add list=Blocked address=68.183.9.16/32 +add list=Blocked address=68.183.180.73/32 +add list=Blocked address=68.220.0.0/15 +add list=Blocked address=69.30.85.194/32 +add list=Blocked address=69.164.220.85/32 +add list=Blocked address=69.164.220.232/32 +add list=Blocked address=72.146.20.230/32 +add list=Blocked address=73.116.194.16/32 +add list=Blocked address=74.7.0.0/16 +add list=Blocked address=74.208.40.151/32 +add list=Blocked address=74.248.0.0/15 +add list=Blocked address=74.248.33.172/32 +add list=Blocked address=77.90.185.230/32 +add list=Blocked address=80.91.86.169/32 +add list=Blocked address=81.171.72.93/32 +add list=Blocked address=81.171.72.135/32 +add list=Blocked address=82.39.206.154/32 +add list=Blocked address=82.102.18.0/24 +add list=Blocked address=82.102.18.190/32 +add list=Blocked address=82.165.82.154/32 +add list=Blocked address=84.17.60.251/32 +add list=Blocked address=85.11.167.225/32 +add list=Blocked address=85.204.70.0/24 +add list=Blocked address=87.121.79.42/32 +add list=Blocked address=88.184.210.4/32 +add list=Blocked address=91.92.41.115/32 +add list=Blocked address=91.92.42.86/32 +add list=Blocked address=91.92.47.101/32 +add list=Blocked address=91.92.47.153/32 +add list=Blocked address=91.92.241.196/32 +add list=Blocked address=91.92.243.245/32 +add list=Blocked address=91.124.88.29/32 +add list=Blocked address=91.124.88.36/32 +add list=Blocked address=91.134.35.95/32 +add list=Blocked address=91.148.244.131/32 +add list=Blocked address=91.148.245.81/32 +add list=Blocked address=91.192.10.86/32 +add list=Blocked address=91.193.232.0/24 +add list=Blocked address=91.224.92.159/32 +add list=Blocked address=91.224.92.190/32 +add list=Blocked address=91.230.225.0/24 +add list=Blocked address=91.239.79.146/32 +add list=Blocked address=92.118.39.29/32 +add list=Blocked address=92.118.39.126/32 +add list=Blocked address=92.119.121.196/32 +add list=Blocked address=93.123.109.0/24 +add list=Blocked address=93.152.221.13/32 +add list=Blocked address=94.154.43.0/24 +add list=Blocked address=95.85.238.18/32 +add list=Blocked address=98.159.36.147/32 +add list=Blocked address=98.159.37.147/32 +add list=Blocked address=102.37.110.74/32 +add list=Blocked address=103.76.88.36/32 +add list=Blocked address=103.76.88.37/32 +add list=Blocked address=103.153.183.126/32 +add list=Blocked address=103.168.66.237/32 +add list=Blocked address=103.177.150.66/32 +add list=Blocked address=103.215.75.19/32 +add list=Blocked address=103.215.75.66/32 +add list=Blocked address=103.221.220.62/32 +add list=Blocked address=104.28.215.219/32 +add list=Blocked address=104.28.215.220/32 +add list=Blocked address=104.28.222.131/32 +add list=Blocked address=104.28.240.0/24 +add list=Blocked address=104.28.247.219/32 +add list=Blocked address=104.28.247.220/32 +add list=Blocked address=104.28.254.131/32 +add list=Blocked address=104.28.254.134/32 +add list=Blocked address=104.168.93.75/32 +add list=Blocked address=104.196.62.35/32 +add list=Blocked address=104.199.182.104/32 +add list=Blocked address=104.208.94.19/32 +add list=Blocked address=104.234.19.67/32 +add list=Blocked address=104.238.222.26/32 +add list=Blocked address=104.248.5.228/32 +add list=Blocked address=104.248.151.168/32 +add list=Blocked address=104.251.181.106/32 +add list=Blocked address=106.63.0.0/17 +add list=Blocked address=107.170.9.103/32 +add list=Blocked address=108.130.204.104/32 +add list=Blocked address=110.176.0.0/13 +add list=Blocked address=111.113.88.210/32 +add list=Blocked address=111.113.89.107/32 +add list=Blocked address=112.94.252.15/32 +add list=Blocked address=113.57.186.101/32 +add list=Blocked address=117.14.153.168/32 +add list=Blocked address=118.81.196.142/32 +add list=Blocked address=118.81.207.202/32 +add list=Blocked address=118.194.253.208/32 +add list=Blocked address=118.212.120.78/32 +add list=Blocked address=123.158.61.179/32 +add list=Blocked address=123.160.175.254/32 +add list=Blocked address=123.178.210.228/32 +add list=Blocked address=124.88.113.10/32 +add list=Blocked address=124.117.194.69/32 +add list=Blocked address=128.90.128.13/32 +add list=Blocked address=128.90.141.10/32 +add list=Blocked address=128.199.182.55/32 +add list=Blocked address=128.199.182.152/32 +add list=Blocked address=129.212.237.196/32 +add list=Blocked address=130.12.180.39/32 +add list=Blocked address=130.12.180.126/32 +add list=Blocked address=130.12.182.12/32 +add list=Blocked address=130.12.182.254/32 +add list=Blocked address=130.131.231.171/32 +add list=Blocked address=132.196.11.13/32 +add list=Blocked address=132.196.83.33/32 +add list=Blocked address=134.122.94.138/32 +add list=Blocked address=134.199.148.147/32 +add list=Blocked address=134.209.25.199/32 +add list=Blocked address=134.209.162.128/32 +add list=Blocked address=134.209.185.72/32 +add list=Blocked address=135.119.63.61/32 +add list=Blocked address=135.119.74.228/32 +add list=Blocked address=135.225.93.215/32 +add list=Blocked address=135.232.177.160/32 +add list=Blocked address=136.64.51.224/32 +add list=Blocked address=136.66.128.85/32 +add list=Blocked address=136.66.153.34/32 +add list=Blocked address=136.70.149.147/32 +add list=Blocked address=136.107.208.33/32 +add list=Blocked address=136.108.11.195/32 +add list=Blocked address=136.109.87.37/32 +add list=Blocked address=136.110.80.8/32 +add list=Blocked address=136.115.17.125/32 +add list=Blocked address=136.116.215.248/32 +add list=Blocked address=136.144.19.51/32 +add list=Blocked address=136.144.33.102/32 +add list=Blocked address=136.144.35.201/32 +add list=Blocked address=136.144.43.127/32 +add list=Blocked address=138.124.103.234/32 +add list=Blocked address=138.197.34.12/32 +add list=Blocked address=138.197.104.211/32 +add list=Blocked address=138.197.191.87/32 +add list=Blocked address=139.28.219.70/32 +add list=Blocked address=139.59.118.17/32 +add list=Blocked address=139.59.231.238/32 +add list=Blocked address=140.248.75.106/32 +add list=Blocked address=141.98.11.171/32 +add list=Blocked address=142.93.62.168/32 +add list=Blocked address=142.93.155.36/32 +add list=Blocked address=142.111.152.16/32 +add list=Blocked address=142.111.152.57/32 +add list=Blocked address=143.20.64.236/32 +add list=Blocked address=143.110.178.165/32 +add list=Blocked address=143.110.217.244/32 +add list=Blocked address=143.198.6.225/32 +add list=Blocked address=143.198.25.99/32 +add list=Blocked address=143.244.56.0/23 +add list=Blocked address=143.244.57.123/32 +add list=Blocked address=143.244.168.161/32 +add list=Blocked address=144.76.19.27/32 +add list=Blocked address=144.126.207.57/32 +add list=Blocked address=144.172.93.117/32 +add list=Blocked address=144.172.97.164/32 +add list=Blocked address=144.172.100.174/32 +add list=Blocked address=144.172.101.168/32 +add list=Blocked address=145.132.103.19/32 +add list=Blocked address=146.70.40.68/32 +add list=Blocked address=146.70.194.0/24 +add list=Blocked address=146.190.63.48/32 +add list=Blocked address=146.190.63.248/32 +add list=Blocked address=146.190.83.166/32 +add list=Blocked address=147.93.168.91/32 +add list=Blocked address=147.93.169.103/32 +add list=Blocked address=147.93.171.188/32 +add list=Blocked address=147.182.149.75/32 +add list=Blocked address=148.251.182.79/32 +add list=Blocked address=149.56.150.98/32 +add list=Blocked address=150.255.94.250/32 +add list=Blocked address=152.39.198.115/32 +add list=Blocked address=152.42.137.175/32 +add list=Blocked address=152.53.242.93/32 +add list=Blocked address=153.75.89.249/32 +add list=Blocked address=155.2.215.98/32 +add list=Blocked address=157.143.3.35/32 +add list=Blocked address=157.245.105.107/32 +add list=Blocked address=157.245.113.227/32 +add list=Blocked address=157.245.197.36/32 +add list=Blocked address=157.245.204.205/32 +add list=Blocked address=158.101.144.198/32 +add list=Blocked address=158.158.0.0/16 +add list=Blocked address=159.65.144.72/32 +add list=Blocked address=159.89.12.166/32 +add list=Blocked address=159.89.46.165/32 +add list=Blocked address=159.89.97.19/32 +add list=Blocked address=159.89.117.109/32 +add list=Blocked address=159.89.123.252/32 +add list=Blocked address=159.89.174.87/32 +add list=Blocked address=159.203.13.62/32 +add list=Blocked address=159.223.132.86/32 +add list=Blocked address=160.179.166.124/32 +add list=Blocked address=160.179.166.222/32 +add list=Blocked address=161.118.251.181/32 +add list=Blocked address=162.141.167.14/32 +add list=Blocked address=162.243.79.174/32 +add list=Blocked address=163.7.11.71/32 +add list=Blocked address=163.7.13.205/32 +add list=Blocked address=164.90.169.163/32 +add list=Blocked address=165.22.24.78/32 +add list=Blocked address=165.22.147.152/32 +add list=Blocked address=165.22.235.3/32 +add list=Blocked address=165.245.185.158/32 +add list=Blocked address=165.245.190.205/32 +add list=Blocked address=167.71.0.47/32 +add list=Blocked address=167.71.39.61/32 +add list=Blocked address=167.86.82.167/32 +add list=Blocked address=167.99.181.249/32 +add list=Blocked address=167.99.182.39/32 +add list=Blocked address=167.99.210.137/32 +add list=Blocked address=167.172.31.131/32 +add list=Blocked address=167.172.53.28/32 +add list=Blocked address=167.172.192.26/32 +add list=Blocked address=168.62.180.116/32 +add list=Blocked address=168.144.83.150/32 +add list=Blocked address=168.144.100.227/32 +add list=Blocked address=168.144.106.212/32 +add list=Blocked address=168.144.167.238/32 +add list=Blocked address=168.144.174.42/32 +add list=Blocked address=170.64.144.116/32 +add list=Blocked address=171.104.81.195/32 +add list=Blocked address=172.86.122.106/32 +add list=Blocked address=172.98.33.251/32 +add list=Blocked address=172.170.254.20/32 +add list=Blocked address=172.171.198.214/32 +add list=Blocked address=172.196.32.64/32 +add list=Blocked address=172.200.0.0/13 +add list=Blocked address=172.208.0.0/13 +add list=Blocked address=172.213.216.126/32 +add list=Blocked address=172.232.175.132/32 +add list=Blocked address=172.245.102.64/32 +add list=Blocked address=173.239.240.222/32 +add list=Blocked address=173.249.34.197/32 +add list=Blocked address=176.227.240.17/32 +add list=Blocked address=176.227.240.61/32 +add list=Blocked address=178.16.54.137/32 +add list=Blocked address=178.62.196.238/32 +add list=Blocked address=178.128.176.132/32 +add list=Blocked address=179.43.134.114/32 +add list=Blocked address=180.95.231.78/32 +add list=Blocked address=182.119.230.145/32 +add list=Blocked address=185.93.89.147/32 +add list=Blocked address=185.93.89.167/32 +add list=Blocked address=185.151.21.124/32 +add list=Blocked address=185.177.72.0/24 +add list=Blocked address=185.177.72.29/32 +add list=Blocked address=185.192.71.0/24 +add list=Blocked address=185.213.175.168/32 +add list=Blocked address=185.223.152.0/24 +add list=Blocked address=185.238.231.167/32 +add list=Blocked address=185.251.19.30/32 +add list=Blocked address=188.166.245.173/32 +add list=Blocked address=188.213.202.0/24 +add list=Blocked address=191.232.0.0/13 +add list=Blocked address=192.34.61.236/32 +add list=Blocked address=192.133.76.0/23 +add list=Blocked address=192.133.77.15/32 +add list=Blocked address=192.253.248.7/32 +add list=Blocked address=192.253.248.142/32 +add list=Blocked address=193.19.109.0/24 +add list=Blocked address=193.36.224.57/32 +add list=Blocked address=193.36.225.0/24 +add list=Blocked address=193.37.32.0/24 +add list=Blocked address=193.37.33.0/24 +add list=Blocked address=193.47.62.168/32 +add list=Blocked address=193.143.1.112/32 +add list=Blocked address=193.148.56.148/32 +add list=Blocked address=193.148.57.33/32 +add list=Blocked address=193.149.176.55/32 +add list=Blocked address=194.5.82.0/24 +add list=Blocked address=194.32.120.151/32 +add list=Blocked address=194.156.79.68/32 +add list=Blocked address=194.233.91.21/32 +add list=Blocked address=195.178.110.0/24 +add list=Blocked address=195.178.110.211/32 +add list=Blocked address=195.178.110.223/32 +add list=Blocked address=198.98.52.229/32 +add list=Blocked address=199.16.157.0/24 +add list=Blocked address=200.162.144.0/20 +add list=Blocked address=201.78.128.89/32 +add list=Blocked address=203.159.90.217/32 +add list=Blocked address=206.81.12.187/32 +add list=Blocked address=206.81.24.227/32 +add list=Blocked address=206.189.19.19/32 +add list=Blocked address=206.189.95.232/32 +add list=Blocked address=206.189.233.36/32 +add list=Blocked address=207.154.197.113/32 +add list=Blocked address=207.241.173.229/32 +add list=Blocked address=208.84.102.195/32 +add list=Blocked address=209.87.169.142/32 +add list=Blocked address=209.99.184.0/21 +add list=Blocked address=209.126.2.173/32 +add list=Blocked address=211.175.22.56/32 +add list=Blocked address=213.35.113.47/32 +add list=Blocked address=213.111.158.220/32 +add list=Blocked address=216.24.210.46/32 +add list=Blocked address=216.24.212.0/24 +add list=Blocked address=216.26.233.39/32 +add list=Blocked address=216.73.163.0/24 +add list=Blocked address=216.73.216.0/22 +add list=Blocked address=216.73.217.176/32 +add list=Blocked address=216.81.248.20/32 +add list=Blocked address=217.217.252.197/32 +add list=Blocked address=220.202.112.100/32 +add list=Blocked address=223.199.187.171/32 + +# Blocked IPv6 (23, non-whitelisted) +/ipv6 firewall address-list +add list=Blocked address=2001:ba0:24c:a500::1/128 +add list=Blocked address=2001:4ca0:108:42::24/128 +add list=Blocked address=2400:8901::2000:62ff:fe9a:e1c2/128 +add list=Blocked address=2406:da18:525:0:a23e:e8f6:3ea1:4c2c/128 +add list=Blocked address=2600:3000::/32 +add list=Blocked address=2600:3c09::2000:5cff:fe99:c802/128 +add list=Blocked address=2600:3c15::2000:53ff:fe31:8c0b/128 +add list=Blocked address=2600:3c15::2000:9cff:fef0:8ad6/128 +add list=Blocked address=2602:fa59::/40 +add list=Blocked address=2a00:6340:2008:ec80:64cc:9116:beb2:1931/128 +add list=Blocked address=2a00:6340:2008:ec80:be24:11ff:fe3b:4058/128 +add list=Blocked address=2a00:6340:2008:ec80:be24:11ff:fe82:2775/128 +add list=Blocked address=2a01:2a8:a0dd:1::1012/128 +add list=Blocked address=2a01:4f8:c014:7f9e::1/128 +add list=Blocked address=2a04:4e40:4400:0:537:cdd2:7250:95db/128 +add list=Blocked address=2a04:4e40:e000::/48 +add list=Blocked address=2a04:c300::/29 +add list=Blocked address=2a05:4140:3::37cd/128 +add list=Blocked address=2a06:a880:5:7fb3::1/128 +add list=Blocked address=2a09:bac1:76c0:460::5e:6a/128 +add list=Blocked address=2a09:bac5::/32 +add list=Blocked address=2a0a:4cc0:2000:8528:c4e8:3ff:febb:3ee9/128 +add list=Blocked address=2a13:adc0::2ed:89ff:fe58:19c5/128 + +/ip firewall filter +add chain=input action=accept src-address-list=Openfactory comment="ids openfactory whitelist (v4)" +add chain=input action=accept src-address-list=Cloudflare comment="ids cloudflare whitelist (v4)" +add chain=input action=accept src-address-list=PaloAlto comment="ids paloalto whitelist (v4)" +add chain=input action=drop src-address-list=Blocked comment="ids blocked (v4)" +add chain=forward action=drop src-address-list=Blocked comment="ids blocked (v4)" + +/ipv6 firewall filter +add chain=input action=accept src-address-list=Openfactory comment="ids openfactory whitelist (v6)" +add chain=input action=accept src-address-list=Cloudflare comment="ids cloudflare whitelist (v6)" +add chain=input action=accept src-address-list=PaloAlto comment="ids paloalto whitelist (v6)" +add chain=input action=drop src-address-list=Blocked comment="ids blocked (v6)" +add chain=forward action=drop src-address-list=Blocked comment="ids blocked (v6)" diff --git a/reports/whois_cache.json b/reports/whois_cache.json new file mode 100644 index 0000000..d0d1d2b --- /dev/null +++ b/reports/whois_cache.json @@ -0,0 +1,867 @@ +{ + "185.177.72.31": "185.177.72.0/24", + "195.178.110.155": "195.178.110.0/24", + "66.249.74.165": "66.249.64.0/20", + "64.225.75.246": "64.225.64.0/20", + "193.19.109.151": "193.19.109.0/24", + "92.118.39.126": "92.118.39.0/24", + "54.179.129.22": "54.179.128.0/18", + "20.220.227.237": "20.192.0.0/10", + "199.16.157.180": "199.16.157.0/24", + "74.249.245.134": "74.248.0.0/15", + "136.107.208.33": "136.107.128.0/17", + "104.28.222.131": "104.28.222.0/24", + "45.148.10.62": "45.148.10.0/24", + "136.70.149.147": "136.70.128.0/17", + "158.158.32.229": "158.158.0.0/16", + "216.73.216.244": "216.73.216.0/22", + "211.175.22.56": "211.175.0.0/16", + "8.229.130.50": "8.229.0.0/16", + "107.170.9.103": "107.170.0.0/17", + "192.34.61.236": "192.34.56.0/21", + "194.5.82.61": "194.5.82.0/24", + "194.5.82.104": "194.5.82.0/24", + "104.28.247.220": "104.28.247.0/24", + "64.23.218.208": "64.23.192.0/19", + "157.245.204.205": "157.245.192.0/20", + "54.169.103.174": "54.169.0.0/17", + "20.199.183.73": "20.192.0.0/10", + "52.238.192.78": "52.224.0.0/11", + "199.16.157.182": "199.16.157.0/24", + "20.226.90.242": "20.192.0.0/10", + "168.144.106.212": "168.144.96.0/20", + "34.175.82.8": "34.175.0.0/17", + "20.170.16.216": "20.160.0.0/12", + "65.0.122.218": "65.0.0.0/14", + "51.116.233.22": "51.116.0.0/16", + "31.58.51.184": "31.58.51.0/24", + "191.235.102.147": "191.232.0.0/13", + "20.9.44.210": "20.0.0.0/11", + "123.160.175.254": "123.160.0.0/14", + "195.178.110.135": "195.178.110.0/24", + "45.61.187.30": "45.61.184.0/22", + "8.231.78.19": "8.231.64.0/18", + "20.226.114.112": "20.192.0.0/10", + "146.190.63.48": "146.190.32.0/19", + "216.73.217.54": "216.73.216.0/22", + "3.107.107.18": "3.104.0.0/14", + "69.164.220.85": "69.164.208.0/20", + "191.235.105.17": "191.232.0.0/13", + "139.28.219.70": "139.28.219.0/24", + "185.177.72.49": "185.177.72.0/24", + "20.206.67.15": "20.192.0.0/10", + "35.237.178.50": "35.237.0.0/16", + "158.158.36.150": "158.158.0.0/16", + "136.64.51.224": "136.64.0.0/15", + "94.154.43.186": "94.154.43.0/24", + "3.101.54.4": "3.101.0.0/16", + "23.234.98.171": "23.234.64.0/18", + "45.67.85.190": "45.67.85.0/24", + "199.16.157.181": "199.16.157.0/24", + "93.123.109.165": "93.123.109.0/24", + "20.79.29.209": "20.64.0.0/10", + "162.141.167.14": "162.141.167.0/24", + "163.7.11.71": "163.7.11.0/24", + "66.187.6.102": "66.187.6.0/24", + "45.153.34.16": "45.153.34.0/24", + "74.7.227.43": "74.7.0.0/16", + "188.166.245.173": "188.166.240.0/20", + "72.146.20.230": "72.144.0.0/14", + "216.73.217.145": "216.73.216.0/22", + "158.158.76.106": "158.158.0.0/16", + "44.248.34.34": "44.224.0.0/11", + "4.194.24.143": "4.192.0.0/12", + "124.88.113.10": "124.88.0.0/16", + "172.213.244.85": "172.208.0.0/13", + "170.64.144.116": "170.64.128.0/18", + "143.244.57.120": "143.244.56.0/23", + "135.119.63.61": "135.119.0.0/16", + "45.154.98.52": "45.154.98.0/24", + "112.94.252.15": "112.94.128.0/17", + "2600:3c09::2000:5cff:fe99:c802": "2600:3c09::/32", + "35.243.173.211": "35.243.160.0/20", + "206.189.233.36": "206.189.224.0/20", + "45.148.10.95": "45.148.10.0/24", + "168.62.180.116": "168.62.0.0/15", + "23.180.120.134": "23.180.120.0/24", + "31.59.129.193": "31.59.129.0/24", + "203.159.90.217": "203.159.90.0/24", + "194.5.82.153": "194.5.82.0/24", + "143.244.57.121": "143.244.56.0/23", + "216.73.217.121": "216.73.216.0/22", + "91.134.35.95": "91.134.0.0/16", + "135.225.93.215": "135.224.0.0/15", + "103.221.220.62": "103.221.220.0/22", + "20.116.62.111": "20.64.0.0/10", + "2.57.122.173": "2.57.122.0/24", + "20.203.221.142": "20.192.0.0/10", + "2a04:c300:400::99": "2a04:c300::/29", + "45.148.10.67": "45.148.10.0/24", + "34.187.200.119": "34.187.128.0/17", + "4.223.65.51": "4.208.0.0/12", + "5.226.149.150": "5.226.144.0/21", + "20.220.150.193": "20.192.0.0/10", + "201.78.128.89": "201.78.128.0/24", + "34.91.243.226": "34.91.240.0/20", + "85.204.70.90": "85.204.70.0/24", + "20.100.187.246": "20.64.0.0/10", + "185.177.72.67": "185.177.72.0/24", + "180.95.231.78": "180.95.128.0/17", + "34.186.83.134": "34.186.64.0/18", + "4.223.77.241": "4.208.0.0/12", + "146.190.83.166": "146.190.80.0/20", + "3.77.92.222": "3.64.0.0/12", + "185.177.72.205": "185.177.72.0/24", + "193.47.62.168": "193.47.62.0/24", + "20.208.40.114": "20.192.0.0/10", + "20.100.175.1": "20.64.0.0/10", + "192.133.77.14": "192.133.76.0/23", + "38.248.95.248": "38.248.95.0/24", + "136.108.11.195": "136.108.0.0/16", + "123.178.210.228": "123.178.0.0/15", + "194.5.82.59": "194.5.82.0/24", + "20.197.192.193": "20.192.0.0/10", + "64.227.32.66": "64.227.32.0/20", + "132.196.11.13": "132.196.0.0/16", + "136.110.80.8": "136.110.64.0/18", + "85.204.70.116": "85.204.70.0/24", + "81.171.72.93": "81.171.72.0/24", + "118.81.207.202": "118.80.0.0/15", + "20.226.26.5": "20.192.0.0/10", + "20.203.219.190": "20.192.0.0/10", + "158.158.32.226": "158.158.0.0/16", + "20.215.211.95": "20.192.0.0/10", + "2600:3c15::2000:53ff:fe31:8c0b": "2600:3c15::/32", + "23.178.112.107": "23.178.112.0/24", + "95.85.238.18": "95.85.238.0/24", + "35.231.100.82": "35.231.0.0/17", + "108.130.204.104": "108.128.0.0/13", + "206.81.12.187": "206.81.0.0/20", + "206.189.19.19": "206.189.16.0/20", + "20.230.87.222": "20.192.0.0/10", + "58.212.237.231": "58.208.0.0/12", + "185.177.72.9": "185.177.72.0/24", + "85.204.70.114": "85.204.70.0/24", + "2a09:bac5:9443:3af::5e:88": "2a09:bac5::/32", + "94.154.43.188": "94.154.43.0/24", + "34.95.239.96": "34.95.224.0/20", + "4.204.201.85": "4.192.0.0/12", + "2a0a:4cc0:2000:8528:c4e8:3ff:febb:3ee9": "2a0a:4cc0:2000::/48", + "220.202.112.100": "220.202.96.0/19", + "20.104.96.117": "20.64.0.0/10", + "35.192.232.168": "35.192.224.0/20", + "34.59.187.134": "34.59.0.0/16", + "216.73.216.24": "216.73.216.0/22", + "139.59.118.17": "139.59.112.0/20", + "171.104.81.195": "171.104.0.0/14", + "52.184.96.217": "52.160.0.0/11", + "216.73.216.133": "216.73.216.0/22", + "199.16.157.183": "199.16.157.0/24", + "110.179.80.63": "110.176.0.0/13", + "194.5.82.141": "194.5.82.0/24", + "216.73.216.25": "216.73.216.0/22", + "104.28.247.219": "104.28.247.0/24", + "146.70.194.230": "146.70.194.0/24", + "136.109.87.37": "136.109.0.0/17", + "93.123.109.152": "93.123.109.0/24", + "23.161.169.62": "23.161.169.0/24", + "2600:3000:2710:200::86": "2600:3000::/32", + "64.89.161.85": "64.89.161.0/24", + "185.93.89.147": "185.93.89.0/24", + "45.154.138.232": "45.154.138.0/24", + "13.57.15.41": "13.57.0.0/16", + "94.154.43.185": "94.154.43.0/24", + "20.113.56.22": "20.64.0.0/10", + "209.99.189.160": "209.99.184.0/21", + "34.86.126.198": "34.86.112.0/20", + "74.7.241.170": "74.7.0.0/16", + "136.115.17.125": "136.115.0.0/16", + "103.153.183.126": "103.153.182.0/23", + "20.104.16.169": "20.64.0.0/10", + "23.178.112.213": "23.178.112.0/24", + "194.5.82.100": "194.5.82.0/24", + "20.206.89.130": "20.192.0.0/10", + "69.30.85.194": "69.30.85.0/24", + "35.95.56.230": "35.80.0.0/12", + "68.183.180.73": "68.183.176.0/20", + "213.111.158.220": "213.111.158.0/23", + "185.177.72.54": "185.177.72.0/24", + "18.227.73.7": "18.224.0.0/14", + "73.116.194.16": "73.116.0.0/16", + "146.70.194.252": "146.70.194.0/24", + "23.178.112.104": "23.178.112.0/24", + "35.237.138.104": "35.237.0.0/16", + "124.117.194.69": "124.117.0.0/16", + "20.9.85.55": "20.0.0.0/11", + "51.79.208.246": "51.79.128.0/17", + "20.196.209.159": "20.192.0.0/10", + "185.93.89.167": "185.93.89.0/24", + "45.148.10.21": "45.148.10.0/24", + "20.220.15.170": "20.192.0.0/10", + "74.248.24.145": "74.248.0.0/15", + "209.99.189.152": "209.99.184.0/21", + "45.154.98.101": "45.154.98.0/24", + "216.73.217.176": "216.73.217.176/32", + "45.134.20.116": "45.134.20.116/32", + "2a00:6340:2008:ec80:be24:11ff:fe3b:4058": "2a00:6340:2008:ec80:be24:11ff:fe3b:4058/128", + "35.199.183.190": "35.199.183.190/32", + "91.192.10.86": "91.192.10.0/24", + "45.148.10.120": "45.148.10.0/24", + "159.89.174.87": "159.89.160.0/20", + "23.178.112.219": "23.178.112.0/24", + "52.253.111.48": "52.224.0.0/11", + "51.103.152.96": "51.103.152.96/32", + "193.19.109.122": "193.19.109.0/24", + "35.243.223.147": "35.243.208.0/20", + "158.101.144.198": "158.101.128.0/19", + "149.56.150.98": "149.56.0.0/16", + "20.119.205.224": "20.64.0.0/10", + "2600:3000:2710:200::83": "2600:3000::/32", + "194.233.91.21": "194.233.88.0/22", + "185.177.72.17": "185.177.72.0/24", + "185.151.21.124": "185.151.21.0/24", + "35.196.22.146": "35.196.16.0/20", + "4.185.41.66": "4.176.0.0/12", + "192.253.248.142": "192.253.248.142/32", + "5.253.247.16": "5.253.247.0/24", + "2602:fa59:10:a3a::1": "2602:fa59::/40", + "216.73.217.13": "216.73.216.0/22", + "23.180.120.147": "23.180.120.0/24", + "161.118.251.181": "161.118.192.0/18", + "51.103.133.211": "51.103.0.0/16", + "74.248.33.172": "74.248.33.172/32", + "157.143.3.35": "157.143.0.0/17", + "20.226.60.151": "20.192.0.0/10", + "159.65.144.72": "159.65.144.0/20", + "158.158.105.63": "158.158.0.0/16", + "51.120.83.160": "51.120.0.0/16", + "185.177.72.29": "185.177.72.29/32", + "144.76.19.27": "144.76.0.0/16", + "45.45.237.214": "45.45.237.0/24", + "172.213.8.106": "172.208.0.0/13", + "45.139.104.172": "45.139.104.0/24", + "23.161.169.127": "23.161.169.0/24", + "103.215.75.19": "103.215.75.19/32", + "82.102.18.188": "82.102.18.0/24", + "36.106.166.223": "36.106.166.0/23", + "130.12.180.126": "130.12.180.0/24", + "185.177.72.38": "185.177.72.0/24", + "82.102.18.190": "82.102.18.190/32", + "172.171.198.214": "172.160.0.0/11", + "43.228.157.102": "43.228.157.0/24", + "173.249.34.197": "173.249.34.0/23", + "193.148.57.33": "193.148.57.0/24", + "192.133.77.15": "192.133.77.15/32", + "195.178.110.48": "195.178.110.0/24", + "217.217.252.197": "217.217.252.0/22", + "3.237.185.144": "3.224.0.0/12", + "216.73.217.173": "216.73.216.0/22", + "3.250.52.216": "3.250.0.0/15", + "91.239.79.146": "91.239.78.0/23", + "146.70.194.220": "146.70.194.0/24", + "123.158.61.179": "123.158.0.0/16", + "172.213.225.181": "172.208.0.0/13", + "20.215.189.213": "20.215.189.213/32", + "20.200.216.112": "20.192.0.0/10", + "74.7.230.46": "74.7.0.0/16", + "185.177.72.52": "185.177.72.0/24", + "104.28.215.219": "104.28.215.0/24", + "147.93.171.188": "147.93.171.188/32", + "91.124.88.36": "91.124.88.0/24", + "34.118.184.2": "34.118.128.0/18", + "168.144.100.227": "168.144.96.0/20", + "45.148.10.247": "45.148.10.0/24", + "94.154.43.187": "94.154.43.0/24", + "13.221.106.198": "13.221.106.198/32", + "104.28.215.220": "104.28.215.0/24", + "2a04:4e40:e000:0:d09:97ab:43b1:a59a": "2a04:4e40:e000::/48", + "194.5.82.98": "194.5.82.0/24", + "54.163.186.39": "54.163.128.0/18", + "223.199.187.171": "223.199.0.0/16", + "143.244.57.123": "143.244.57.123/32", + "167.99.181.249": "167.99.176.0/20", + "216.73.216.229": "216.73.216.0/22", + "34.47.183.174": "34.47.128.0/17", + "2600:3000:2710:200::81": "2600:3000::/32", + "43.228.157.191": "43.228.157.0/24", + "104.208.94.19": "104.208.94.19/32", + "2600:3000:2710:200::84": "2600:3000::/32", + "146.70.40.68": "146.70.40.0/24", + "2600:3000:1305:4212::84": "2600:3000::/32", + "157.245.105.107": "157.245.96.0/20", + "68.221.69.72": "68.220.0.0/15", + "195.178.110.211": "195.178.110.211/32", + "103.168.66.237": "103.168.66.0/23", + "162.243.79.174": "162.243.0.0/17", + "172.213.144.209": "172.208.0.0/13", + "138.197.191.87": "138.197.176.0/20", + "54.216.227.171": "54.216.192.0/18", + "172.213.216.126": "172.213.216.126/32", + "209.99.190.126": "209.99.184.0/21", + "134.199.148.147": "134.199.144.0/20", + "110.179.81.90": "110.176.0.0/13", + "134.209.162.128": "134.209.160.0/20", + "195.178.110.223": "195.178.110.223/32", + "93.123.109.166": "93.123.109.0/24", + "185.177.72.12": "185.177.72.0/24", + "165.245.185.158": "165.245.176.0/20", + "172.213.27.135": "172.208.0.0/13", + "77.90.185.230": "77.90.185.0/24", + "20.9.4.9": "20.0.0.0/11", + "74.248.139.103": "74.248.0.0/15", + "117.14.153.168": "117.8.0.0/13", + "45.153.34.43": "45.153.34.0/24", + "191.237.255.150": "191.232.0.0/13", + "194.5.82.92": "194.5.82.0/24", + "118.212.120.78": "118.212.0.0/16", + "20.151.205.204": "20.150.0.0/15", + "185.177.72.10": "185.177.72.0/24", + "35.230.52.27": "35.230.0.0/17", + "31.56.58.134": "31.56.58.0/24", + "20.230.47.94": "20.192.0.0/10", + "44.245.216.107": "44.224.0.0/11", + "52.167.144.160": "52.160.0.0/11", + "34.11.127.22": "34.11.0.0/17", + "85.204.70.96": "85.204.70.0/24", + "35.157.251.190": "35.156.0.0/14", + "20.100.178.92": "20.64.0.0/10", + "4.184.121.217": "4.176.0.0/12", + "35.198.180.224": "35.198.176.0/20", + "44.251.113.120": "44.224.0.0/11", + "91.230.225.189": "91.230.225.0/24", + "34.47.194.183": "34.47.128.0/17", + "82.102.18.182": "82.102.18.0/24", + "172.213.155.173": "172.208.0.0/13", + "82.102.18.180": "82.102.18.0/24", + "23.178.112.106": "23.178.112.0/24", + "5.255.119.161": "5.255.96.0/19", + "158.158.96.17": "158.158.0.0/16", + "2600:3000:1305:4212::81": "2600:3000::/32", + "192.133.77.16": "192.133.76.0/23", + "172.232.175.132": "172.232.160.0/19", + "193.37.33.117": "193.37.33.0/24", + "2a04:c300:400::5d": "2a04:c300::/29", + "20.198.120.225": "20.192.0.0/10", + "192.253.248.7": "192.253.248.0/24", + "2400:8901::2000:62ff:fe9a:e1c2": "2400:8901::/32", + "20.215.218.234": "20.192.0.0/10", + "3.79.41.12": "3.64.0.0/12", + "94.154.43.183": "94.154.43.0/24", + "43.98.201.177": "43.98.201.0/24", + "20.104.50.116": "20.64.0.0/10", + "52.165.196.84": "52.160.0.0/11", + "185.177.72.53": "185.177.72.0/24", + "91.92.47.101": "91.92.47.0/24", + "216.73.216.140": "216.73.216.0/22", + "35.187.9.30": "35.187.0.0/20", + "191.237.250.106": "191.232.0.0/13", + "104.28.240.187": "104.28.240.0/24", + "66.187.5.228": "66.187.5.0/24", + "2a00:6340:2008:ec80:be24:11ff:fe82:2775": "2a00:6340::/32", + "113.57.186.101": "113.56.0.0/15", + "34.244.177.56": "34.240.0.0/13", + "2600:3000:1305:4212::85": "2600:3000::/32", + "20.9.15.100": "20.0.0.0/11", + "80.91.86.169": "80.91.86.0/24", + "20.91.140.156": "20.64.0.0/10", + "139.59.231.238": "139.59.224.0/20", + "23.102.123.231": "23.102.0.0/16", + "20.104.22.47": "20.64.0.0/10", + "216.73.216.209": "216.73.216.0/22", + "4.223.71.149": "4.208.0.0/12", + "192.133.77.18": "192.133.76.0/23", + "93.123.109.163": "93.123.109.0/24", + "4.225.166.222": "4.224.0.0/12", + "193.148.56.148": "193.148.56.0/24", + "23.178.112.212": "23.178.112.0/24", + "194.5.82.123": "194.5.82.0/24", + "34.171.75.208": "34.171.0.0/17", + "185.177.72.22": "185.177.72.0/24", + "20.197.178.120": "20.192.0.0/10", + "158.158.45.224": "158.158.0.0/16", + "85.204.70.100": "85.204.70.0/24", + "2600:3000:2710:200::82": "2600:3000::/32", + "34.65.14.246": "34.65.0.0/20", + "185.177.72.5": "185.177.72.0/24", + "20.195.181.10": "20.192.0.0/10", + "110.179.80.213": "110.176.0.0/13", + "34.69.107.154": "34.69.96.0/20", + "195.178.110.199": "195.178.110.0/24", + "85.204.70.118": "85.204.70.0/24", + "20.226.43.166": "20.192.0.0/10", + "130.12.182.254": "130.12.182.0/24", + "20.226.5.174": "20.192.0.0/10", + "20.91.208.34": "20.64.0.0/10", + "68.221.130.241": "68.220.0.0/15", + "85.204.70.112": "85.204.70.0/24", + "165.245.190.205": "165.245.176.0/20", + "51.120.79.193": "51.120.0.0/16", + "216.73.216.41": "216.73.216.0/22", + "216.73.216.66": "216.73.216.0/22", + "34.105.30.251": "34.105.16.0/20", + "45.148.10.244": "45.148.10.0/24", + "2600:3000:2710:200::85": "2600:3000::/32", + "20.215.216.94": "20.192.0.0/10", + "18.142.90.109": "18.142.0.0/15", + "34.242.226.103": "34.240.0.0/13", + "158.158.41.79": "158.158.0.0/16", + "195.178.110.31": "195.178.110.0/24", + "20.203.209.227": "20.192.0.0/10", + "209.99.189.190": "209.99.184.0/21", + "148.251.182.79": "148.251.0.0/16", + "16.146.99.254": "16.146.0.0/17", + "167.99.210.137": "167.99.208.0/20", + "45.134.20.128": "45.134.20.0/24", + "69.164.220.232": "69.164.208.0/20", + "34.11.108.163": "34.11.0.0/17", + "104.248.151.168": "104.248.144.0/20", + "128.199.182.55": "128.199.128.0/18", + "23.178.112.103": "23.178.112.0/24", + "91.92.41.115": "91.92.41.0/24", + "20.151.225.23": "20.150.0.0/15", + "91.92.241.196": "91.92.241.0/24", + "185.177.72.81": "185.177.72.0/24", + "20.100.176.141": "20.64.0.0/10", + "129.212.237.196": "129.212.224.0/20", + "2a09:bac5:9443:3af::5e:4d": "2a09:bac5::/32", + "20.100.169.152": "20.64.0.0/10", + "20.151.10.161": "20.150.0.0/15", + "34.73.128.17": "34.73.128.0/20", + "143.244.57.84": "143.244.56.0/23", + "20.198.114.210": "20.192.0.0/10", + "45.88.138.44": "45.88.138.0/24", + "167.99.182.39": "167.99.176.0/20", + "185.177.72.58": "185.177.72.0/24", + "54.82.69.43": "54.82.0.0/17", + "172.202.29.124": "172.200.0.0/13", + "64.89.161.82": "64.89.161.0/24", + "20.197.195.24": "20.192.0.0/10", + "20.251.48.208": "20.192.0.0/10", + "150.255.94.250": "150.255.0.0/16", + "82.165.82.154": "82.165.0.0/16", + "5.34.213.142": "5.34.208.0/21", + "45.154.138.181": "45.154.138.0/24", + "200.162.152.170": "200.162.144.0/20", + "4.193.178.28": "4.192.0.0/12", + "2a01:2a8:a0dd:1::1012": "2a01:2a8::/29", + "216.73.216.98": "216.73.216.0/22", + "20.48.234.177": "20.48.0.0/12", + "134.122.94.138": "134.122.80.0/20", + "35.231.41.225": "35.231.0.0/17", + "2.58.56.101": "2.58.56.0/24", + "146.190.63.248": "146.190.32.0/19", + "34.31.88.250": "34.31.0.0/16", + "20.220.210.52": "20.192.0.0/10", + "52.139.36.144": "52.136.0.0/13", + "194.5.82.87": "194.5.82.0/24", + "35.199.111.12": "35.199.96.0/20", + "85.204.70.106": "85.204.70.0/24", + "20.203.133.142": "20.192.0.0/10", + "91.124.88.29": "91.124.88.0/24", + "216.73.216.10": "216.73.216.0/22", + "198.98.52.229": "198.98.48.0/20", + "93.123.109.101": "93.123.109.0/24", + "34.139.141.56": "34.139.128.0/20", + "143.244.168.161": "143.244.160.0/20", + "216.73.216.230": "216.73.216.0/22", + "194.5.82.89": "194.5.82.0/24", + "195.178.110.104": "195.178.110.0/24", + "34.86.217.208": "34.86.208.0/20", + "20.214.106.237": "20.192.0.0/10", + "54.206.147.115": "54.206.128.0/18", + "91.224.92.190": "91.224.92.0/24", + "20.48.236.161": "20.48.0.0/12", + "34.181.134.189": "34.181.128.0/17", + "157.245.197.36": "157.245.192.0/20", + "45.61.150.67": "45.61.148.0/22", + "143.244.57.92": "143.244.56.0/23", + "94.154.43.184": "94.154.43.0/24", + "20.203.148.31": "20.192.0.0/10", + "34.65.18.71": "34.65.16.0/20", + "192.133.77.17": "192.133.76.0/23", + "84.17.60.251": "84.17.60.0/24", + "138.197.104.211": "138.197.96.0/20", + "206.81.24.227": "206.81.16.0/20", + "20.46.121.148": "20.40.0.0/13", + "36.106.166.92": "36.106.166.0/23", + "20.63.63.128": "20.48.0.0/12", + "20.151.200.100": "20.150.0.0/15", + "216.73.217.11": "216.73.216.0/22", + "130.12.180.39": "130.12.180.0/24", + "146.70.194.254": "146.70.194.0/24", + "91.148.244.131": "91.148.244.0/22", + "194.5.82.63": "194.5.82.0/24", + "20.205.229.184": "20.192.0.0/10", + "2600:3000:1305:4212::83": "2600:3000::/32", + "206.189.95.232": "206.189.80.0/20", + "130.131.231.171": "130.131.0.0/16", + "185.177.72.24": "185.177.72.0/24", + "20.220.212.5": "20.192.0.0/10", + "146.70.194.228": "146.70.194.0/24", + "20.203.135.57": "20.192.0.0/10", + "20.206.105.145": "20.192.0.0/10", + "2a04:c300:400::58": "2a04:c300::/29", + "20.100.203.84": "20.64.0.0/10", + "91.224.92.159": "91.224.92.0/24", + "104.196.62.35": "104.196.62.35/32", + "153.75.89.249": "153.75.89.0/24", + "104.168.93.75": "104.168.0.0/17", + "185.177.72.13": "185.177.72.0/24", + "91.92.42.86": "91.92.42.0/24", + "45.153.34.182": "45.153.34.182/32", + "185.177.72.100": "185.177.72.0/24", + "178.16.54.137": "178.16.54.0/24", + "103.215.75.66": "103.215.74.0/23", + "81.171.72.135": "81.171.72.135/32", + "20.251.58.190": "20.192.0.0/10", + "195.178.110.159": "195.178.110.0/24", + "23.178.112.211": "23.178.112.0/24", + "185.177.72.56": "185.177.72.0/24", + "143.110.178.165": "143.110.178.165/32", + "20.127.18.0": "20.64.0.0/10", + "185.177.72.69": "185.177.72.0/24", + "185.177.72.30": "185.177.72.0/24", + "147.182.149.75": "147.182.149.75/32", + "94.154.43.179": "94.154.43.0/24", + "194.5.82.95": "194.5.82.0/24", + "172.213.232.128": "172.208.0.0/13", + "118.81.196.142": "118.80.0.0/15", + "93.123.109.10": "93.123.109.0/24", + "66.228.38.221": "66.228.32.0/20", + "91.148.245.81": "91.148.244.0/22", + "74.248.35.25": "74.248.0.0/15", + "45.45.237.69": "45.45.237.0/24", + "34.17.159.73": "34.17.0.0/16", + "45.148.10.194": "45.148.10.0/24", + "8.229.1.130": "8.229.0.0/16", + "104.199.182.104": "104.199.176.0/20", + "2001:4ca0:108:42::24": "2001:4ca0::/32", + "111.113.89.107": "111.112.0.0/15", + "4.184.185.91": "4.176.0.0/12", + "34.146.12.12": "34.146.0.0/20", + "34.131.251.163": "34.131.240.0/20", + "45.138.16.114": "45.138.16.0/24", + "182.119.230.145": "182.112.0.0/12", + "51.120.76.82": "51.120.0.0/16", + "34.40.240.124": "34.40.128.0/17", + "111.113.88.210": "111.112.0.0/15", + "34.42.48.100": "34.42.0.0/16", + "20.220.203.117": "20.192.0.0/10", + "45.148.10.60": "45.148.10.0/24", + "8.234.157.73": "8.234.128.0/17", + "74.208.40.151": "74.208.0.0/16", + "165.22.235.3": "165.22.224.0/20", + "20.240.137.17": "20.192.0.0/10", + "144.172.97.164": "144.172.96.0/23", + "142.111.152.16": "142.111.152.0/24", + "216.73.163.148": "216.73.163.0/24", + "4.223.226.192": "4.208.0.0/12", + "93.152.221.13": "93.152.221.0/24", + "20.63.10.215": "20.48.0.0/12", + "136.116.215.248": "136.116.0.0/16", + "103.76.88.36": "103.76.88.0/24", + "213.35.113.47": "213.35.96.0/19", + "160.179.166.222": "160.176.0.0/14", + "216.81.248.20": "216.81.248.0/22", + "104.28.254.131": "104.28.254.0/24", + "3.27.206.50": "3.24.0.0/14", + "91.193.232.61": "91.193.232.0/24", + "136.66.128.85": "136.66.0.0/15", + "5.161.225.116": "5.161.0.0/16", + "34.187.20.70": "34.187.0.0/17", + "185.223.152.40": "185.223.152.0/24", + "20.172.218.20": "20.160.0.0/12", + "31.56.58.124": "31.56.58.0/24", + "173.239.240.222": "173.239.240.0/24", + "2001:ba0:24c:a500::1": "2001:ba0::/32", + "185.223.152.126": "185.223.152.0/24", + "62.93.179.146": "62.93.179.0/24", + "104.248.5.228": "104.248.0.0/20", + "35.182.17.8": "35.182.0.0/15", + "20.46.252.52": "20.40.0.0/13", + "34.116.255.65": "34.116.240.0/20", + "172.213.208.20": "172.208.0.0/13", + "163.7.13.205": "163.7.13.0/24", + "46.232.235.6": "46.232.235.0/24", + "4.231.36.50": "4.224.0.0/12", + "45.130.203.154": "45.130.202.0/23", + "20.100.173.28": "20.64.0.0/10", + "176.227.240.61": "176.227.240.0/24", + "2a04:c300::3ce": "2a04:c300::/29", + "185.251.19.30": "185.251.19.0/24", + "92.118.39.29": "92.118.39.0/24", + "185.192.71.16": "185.192.71.0/24", + "138.197.34.12": "138.197.32.0/20", + "34.90.88.37": "34.90.80.0/20", + "165.22.24.78": "165.22.16.0/20", + "144.172.101.168": "144.172.101.0/24", + "216.24.212.243": "216.24.212.0/24", + "4.236.158.50": "4.224.0.0/12", + "168.144.174.42": "168.144.160.0/20", + "2406:da18:525:0:a23e:e8f6:3ea1:4c2c": "2406:da18:400::/38", + "52.159.244.176": "52.152.0.0/13", + "34.24.155.203": "34.24.0.0/16", + "37.140.223.58": "37.140.223.0/24", + "194.156.79.68": "194.156.79.0/24", + "46.23.108.234": "46.23.108.0/24", + "74.248.96.101": "74.248.0.0/15", + "31.58.51.130": "31.58.51.0/24", + "34.94.172.72": "34.94.160.0/20", + "200.162.145.220": "200.162.144.0/20", + "13.126.86.18": "13.126.0.0/15", + "143.198.25.99": "143.198.16.0/20", + "104.28.254.134": "104.28.254.0/24", + "167.86.82.167": "167.86.82.0/23", + "91.92.243.245": "91.92.243.0/24", + "106.63.26.8": "106.63.0.0/17", + "2a09:bac1:76c0:460::5e:6a": "2a09:bac1::/32", + "136.66.153.34": "136.66.0.0/15", + "20.52.125.110": "20.48.0.0/12", + "15.206.146.240": "15.206.0.0/15", + "194.32.120.151": "194.32.120.0/24", + "135.232.177.160": "135.232.0.0/14", + "20.171.55.167": "20.160.0.0/12", + "64.225.63.129": "64.225.48.0/20", + "172.202.95.21": "172.200.0.0/13", + "136.144.43.127": "136.144.43.0/24", + "24.144.85.55": "24.144.80.0/20", + "128.199.182.152": "128.199.128.0/18", + "152.42.137.175": "152.42.128.0/20", + "87.121.79.42": "87.121.79.0/24", + "82.39.206.154": "82.39.206.0/24", + "167.71.39.61": "167.71.32.0/20", + "193.19.109.68": "193.19.109.0/24", + "195.178.110.18": "195.178.110.0/24", + "193.36.225.180": "193.36.225.0/24", + "136.144.33.102": "136.144.33.0/24", + "167.71.0.47": "167.71.0.0/20", + "104.28.240.188": "104.28.240.0/24", + "45.146.54.80": "45.146.54.0/24", + "193.36.225.133": "193.36.225.0/24", + "103.177.150.66": "103.177.150.0/24", + "172.86.122.106": "172.86.120.0/22", + "37.140.223.6": "37.140.223.0/24", + "216.73.163.119": "216.73.163.0/24", + "179.43.134.114": "179.43.134.0/24", + "130.12.182.12": "130.12.182.0/24", + "159.89.46.165": "159.89.32.0/20", + "62.60.130.227": "62.60.130.0/24", + "207.241.173.229": "207.241.172.0/23", + "142.93.62.168": "142.93.48.0/20", + "91.230.225.192": "91.230.225.0/24", + "54.210.219.163": "54.210.0.0/16", + "45.131.195.127": "45.131.195.0/24", + "20.199.181.219": "20.192.0.0/10", + "193.37.33.55": "193.37.33.0/24", + "216.73.163.138": "216.73.163.0/24", + "188.213.202.116": "188.213.202.0/24", + "85.204.70.98": "85.204.70.0/24", + "193.36.225.117": "193.36.225.0/24", + "188.213.202.121": "188.213.202.0/24", + "45.131.195.175": "45.131.195.0/24", + "34.75.4.161": "34.75.0.0/20", + "20.125.176.178": "20.64.0.0/10", + "34.158.83.192": "34.158.64.0/18", + "45.130.203.177": "45.130.202.0/23", + "216.24.212.233": "216.24.212.0/24", + "168.144.167.238": "168.144.160.0/20", + "216.73.163.132": "216.73.163.0/24", + "191.232.199.39": "191.232.0.0/13", + "20.203.200.218": "20.192.0.0/10", + "20.12.214.177": "20.0.0.0/11", + "98.159.37.147": "98.159.37.0/24", + "34.21.144.169": "34.21.128.0/17", + "45.146.54.137": "45.146.54.0/24", + "193.37.32.159": "193.37.32.0/24", + "172.98.33.251": "172.98.33.0/24", + "207.154.197.113": "207.154.192.0/20", + "104.251.181.106": "104.251.180.0/22", + "34.26.191.118": "34.26.0.0/16", + "193.37.33.251": "193.37.33.0/24", + "135.119.74.228": "135.119.0.0/16", + "20.226.83.28": "20.192.0.0/10", + "2a04:4e40:e000:0:566:f8d7:adbe:f536": "2a04:4e40:e000::/48", + "168.144.83.150": "168.144.80.0/20", + "68.221.137.211": "68.220.0.0/15", + "167.172.192.26": "167.172.192.0/20", + "143.110.217.244": "143.110.208.0/20", + "88.184.210.4": "88.184.192.0/19", + "167.172.53.28": "167.172.48.0/20", + "20.55.86.49": "20.48.0.0/12", + "35.199.115.51": "35.199.112.0/20", + "34.154.122.183": "34.154.0.0/17", + "85.204.70.92": "85.204.70.0/24", + "2a13:adc0::2ed:89ff:fe58:19c5": "2a13:adc0::/48", + "157.245.113.227": "157.245.112.0/20", + "45.131.195.153": "45.131.195.0/24", + "34.82.10.34": "34.82.0.0/20", + "47.85.160.47": "47.85.160.0/24", + "85.11.167.225": "85.11.167.0/24", + "4.246.70.112": "4.240.0.0/12", + "143.20.64.236": "143.20.64.0/24", + "185.192.71.76": "185.192.71.0/24", + "134.209.185.72": "134.209.176.0/20", + "144.126.207.57": "144.126.192.0/20", + "172.202.37.160": "172.200.0.0/13", + "118.194.253.208": "118.194.253.0/24", + "144.172.93.117": "144.172.92.0/22", + "4.225.203.146": "4.224.0.0/12", + "209.126.2.173": "209.126.0.0/20", + "35.204.88.252": "35.204.80.0/20", + "2600:3c15::2000:9cff:fef0:8ad6": "2600:3c15::/32", + "2a06:a880:5:7fb3::1": "2a06:a880:5::/48", + "94.154.43.177": "94.154.43.0/24", + "176.227.240.17": "176.227.240.0/24", + "91.230.225.186": "91.230.225.0/24", + "8.235.114.162": "8.235.0.0/17", + "159.223.132.86": "159.223.128.0/20", + "91.92.47.153": "91.92.47.0/24", + "2602:fa59:10:e70::1": "2602:fa59::/40", + "188.213.202.84": "188.213.202.0/24", + "45.146.55.48": "45.146.55.0/24", + "45.131.193.139": "45.131.193.0/24", + "34.55.200.45": "34.55.0.0/16", + "20.215.191.139": "20.192.0.0/10", + "8.234.207.247": "8.234.128.0/17", + "9.205.16.190": "9.205.0.0/16", + "35.204.141.252": "35.204.128.0/20", + "136.144.35.201": "136.144.35.0/24", + "2a04:4e40:e000:0:bfc:4b0:174b:d52d": "2a04:4e40:e000::/48", + "208.84.102.195": "208.84.102.0/24", + "160.179.166.124": "160.176.0.0/14", + "74.248.20.32": "74.248.0.0/15", + "93.123.109.167": "93.123.109.0/24", + "52.165.250.117": "52.160.0.0/11", + "5.161.62.209": "5.161.0.0/18", + "2602:fa59:10:4c9::1": "2602:fa59::/40", + "45.154.98.134": "45.154.98.0/24", + "172.245.102.64": "172.245.102.0/24", + "2a04:4e40:4400:0:537:cdd2:7250:95db": "2a04:4e40:4400::/48", + "216.24.210.46": "216.24.210.0/24", + "143.198.6.225": "143.198.0.0/20", + "91.230.225.2": "91.230.225.0/24", + "142.93.155.36": "142.93.144.0/20", + "20.220.225.223": "20.192.0.0/10", + "68.221.186.136": "68.220.0.0/15", + "172.170.254.20": "172.160.0.0/11", + "13.83.161.80": "13.64.0.0/11", + "103.76.88.37": "103.76.88.0/24", + "20.100.186.69": "20.64.0.0/10", + "185.192.71.247": "185.192.71.0/24", + "52.242.193.144": "52.224.0.0/11", + "92.119.121.196": "92.119.120.0/22", + "185.223.152.60": "185.223.152.0/24", + "20.184.150.240": "20.184.0.0/13", + "52.225.25.48": "52.224.0.0/11", + "94.154.43.178": "94.154.43.0/24", + "216.24.212.14": "216.24.212.0/24", + "2a04:4e40:e000:0:33c:a689:be95:5a4": "2a04:4e40:e000::/48", + "35.184.157.121": "35.184.144.0/20", + "185.192.71.5": "185.192.71.0/24", + "104.28.240.189": "104.28.240.0/24", + "147.93.169.103": "147.93.168.0/22", + "164.90.169.163": "164.90.160.0/20", + "46.232.235.5": "46.232.235.0/24", + "155.2.215.98": "155.2.215.0/24", + "134.209.25.199": "134.209.16.0/20", + "185.238.231.167": "185.238.231.0/24", + "20.199.136.34": "20.192.0.0/10", + "216.24.212.45": "216.24.212.0/24", + "147.93.168.91": "147.93.168.0/22", + "185.223.152.103": "185.223.152.0/24", + "159.89.117.109": "159.89.112.0/20", + "140.248.75.106": "140.248.75.0/24", + "193.37.32.157": "193.37.32.0/24", + "128.90.128.13": "128.90.128.0/24", + "185.192.71.3": "185.192.71.0/24", + "106.63.26.7": "106.63.0.0/17", + "158.158.41.78": "158.158.0.0/16", + "185.192.71.13": "185.192.71.0/24", + "85.204.70.102": "85.204.70.0/24", + "152.53.242.93": "152.53.240.0/22", + "144.172.100.174": "144.172.100.0/24", + "178.62.196.238": "178.62.192.0/18", + "159.203.13.62": "159.203.0.0/20", + "31.184.236.8": "31.184.236.0/24", + "128.90.141.10": "128.90.141.0/24", + "45.146.55.11": "45.146.55.0/24", + "4.232.188.49": "4.224.0.0/12", + "45.148.10.119": "45.148.10.0/24", + "172.204.17.66": "172.200.0.0/13", + "34.7.160.235": "34.7.128.0/17", + "35.201.159.247": "35.201.144.0/20", + "34.148.8.63": "34.148.0.0/20", + "4.223.140.16": "4.208.0.0/12", + "165.22.147.152": "165.22.144.0/20", + "34.90.87.32": "34.90.80.0/20", + "106.63.26.22": "106.63.0.0/17", + "138.124.103.234": "138.124.103.0/24", + "34.64.173.205": "34.64.160.0/20", + "93.123.109.178": "93.123.109.0/24", + "91.193.232.155": "91.193.232.0/24", + "209.87.169.142": "209.87.169.0/24", + "2a04:4e40:e000:0:954:9789:20b1:d5fc": "2a04:4e40:e000::/48", + "34.187.82.183": "34.187.0.0/17", + "193.37.32.149": "193.37.32.0/24", + "20.52.54.143": "20.48.0.0/12", + "106.63.26.21": "106.63.0.0/17", + "2a05:4140:3::37cd": "2a05:4140:3::/48", + "193.36.224.57": "193.36.224.0/24", + "2a09:bac5:55fb:18c8::278:a7": "2a09:bac5::/32", + "195.178.110.72": "195.178.110.0/24", + "200.162.145.50": "200.162.144.0/20", + "31.56.58.122": "31.56.58.0/24", + "2a01:4f8:c014:7f9e::1": "2a01:4f8::/32", + "216.24.212.43": "216.24.212.0/24", + "195.178.110.100": "195.178.110.0/24", + "102.37.110.74": "102.37.0.0/17", + "20.106.182.98": "20.64.0.0/10", + "172.202.44.182": "172.200.0.0/13", + "216.26.233.39": "216.26.233.0/24", + "52.242.217.80": "52.224.0.0/11", + "167.172.31.131": "167.172.16.0/20", + "45.8.196.200": "45.8.196.0/24", + "195.178.110.241": "195.178.110.0/24", + "41.143.154.1": "41.140.0.0/14", + "23.94.155.57": "23.94.155.0/24", + "159.89.123.252": "159.89.112.0/20", + "104.238.222.26": "104.238.222.0/24", + "23.95.226.104": "23.94.0.0/15", + "159.89.97.19": "159.89.96.0/20", + "35.255.40.65": "35.255.0.0/16", + "34.145.62.28": "34.145.48.0/20", + "185.192.71.2": "185.192.71.0/24", + "193.149.176.55": "193.149.176.0/24", + "34.7.193.250": "34.7.128.0/17", + "152.39.198.115": "152.39.128.0/17", + "18.236.136.230": "18.236.0.0/15", + "188.213.202.113": "188.213.202.0/24", + "136.144.19.51": "136.144.19.0/24", + "68.183.9.16": "68.183.0.0/20", + "62.60.130.128": "62.60.130.0/24", + "185.192.71.6": "185.192.71.0/24", + "18.191.56.164": "18.191.0.0/16", + "104.234.19.67": "104.234.19.0/24", + "172.196.32.64": "172.192.0.0/13", + "185.213.175.168": "185.213.175.0/24", + "46.232.235.3": "46.232.235.0/24", + "94.154.43.229": "94.154.43.0/24", + "34.18.38.137": "34.18.0.0/16", + "132.196.83.33": "132.196.0.0/16", + "91.193.232.168": "91.193.232.0/24", + "31.56.58.129": "31.56.58.0/24", + "141.98.11.171": "141.98.11.0/24", + "216.73.163.122": "216.73.163.0/24", + "142.111.152.57": "142.111.152.0/24", + "98.159.36.147": "98.159.36.0/24", + "2a00:6340:2008:ec80:64cc:9116:beb2:1931": "2a00:6340::/32", + "172.212.171.144": "172.208.0.0/13", + "46.232.235.4": "46.232.235.0/24", + "34.65.138.195": "34.65.128.0/20", + "159.89.12.166": "159.89.0.0/20", + "193.143.1.112": "193.143.1.0/24", + "145.132.103.19": "145.132.0.0/15", + "209.99.190.99": "209.99.184.0/21", + "20.104.18.253": "20.64.0.0/10", + "178.128.176.132": "178.128.176.0/20" +} \ No newline at end of file diff --git a/scan.py b/scan.py new file mode 100644 index 0000000..6d663a4 --- /dev/null +++ b/scan.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Consolidated intrusion detection log scanner. + +Runs both copyparty and Nginx Proxy Manager (NPM) log analyzers, +and outputs a single consolidated MikroTik RouterOS blocklist script. +""" + +from __future__ import annotations + +import argparse +import os +from datetime import datetime + +import utils +from copyparty import CopypartyAnalyzer, DEFAULT_LOG_PATH as CP_DEFAULT_LOG, DEFAULT_REPORT_DIR as CP_DEFAULT_REPORT +from npm import NpmAnalyzer, DEFAULT_LOG_PATH as NPM_DEFAULT_LOG, DEFAULT_REPORT_DIR as NPM_DEFAULT_REPORT + +DEFAULT_CONSOLIDATED_REPORT_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "reports") + + +def main() -> None: + parser = argparse.ArgumentParser( + description="Consolidated IDS Scanner: run copyparty + NPM analyzers and generate unified blocklist." + ) + + # copyparty arguments + parser.add_argument("--copyparty-log-path", default=CP_DEFAULT_LOG, + help=f"copyparty log directory (default: {CP_DEFAULT_LOG})") + parser.add_argument("--copyparty-report-dir", default=CP_DEFAULT_REPORT, + help=f"copyparty report directory (default: {CP_DEFAULT_REPORT})") + parser.add_argument("--min-denied", type=int, default=1, + help="minimum copyparty denied hits before flagging (default: 1)") + + # NPM arguments + parser.add_argument("--npm-log-path", default=NPM_DEFAULT_LOG, + help=f"NPM log directory (default: {NPM_DEFAULT_LOG})") + parser.add_argument("--npm-report-dir", default=NPM_DEFAULT_REPORT, + help=f"NPM report directory (default: {NPM_DEFAULT_REPORT})") + parser.add_argument("--min-errors", type=int, default=5, + help="minimum NPM error responses to flag (default: 5)") + parser.add_argument("--error-ratio", type=float, default=0.5, + help="minimum NPM error ratio to flag (default: 0.5)") + + # Shared arguments + parser.add_argument("--report-dir", default=DEFAULT_CONSOLIDATED_REPORT_DIR, + help=f"consolidated reports directory (default: {DEFAULT_CONSOLIDATED_REPORT_DIR})") + parser.add_argument("--top", type=int, default=30, + help="rows in the top-IP tables (default: 30)") + parser.add_argument("--quiet", action="store_true", + help="write files, suppress console reports") + parser.add_argument("--no-rsc", action="store_true", + help="skip MikroTik consolidated .rsc generation") + + args = parser.parse_args() + + # 1. Initialize Analyzers + # We turn off inner write_rsc of individual analyzers to avoid duplicate/redundant rsc writes, + # or let them write their individual ones if they want to. Writing individual scripts is harmless. + # We'll match the individual tools' default write behavior, but generate the consolidated one. + cp_analyzer = CopypartyAnalyzer( + log_path=args.copyparty_log_path, + report_dir=args.copyparty_report_dir, + top=args.top, + quiet=args.quiet, + write_rsc=not args.no_rsc, + min_denied=args.min_denied, + ) + + npm_analyzer = NpmAnalyzer( + log_path=args.npm_log_path, + report_dir=args.npm_report_dir, + top=args.top, + quiet=args.quiet, + write_rsc=not args.no_rsc, + min_errors=args.min_errors, + error_ratio=args.error_ratio, + ) + + # 2. Run copyparty analysis + print("=== Running copyparty scan ===") + cp_exists = True + try: + cp_analyzer.run() + except Exception as e: + print(f"Error running copyparty scan: {e}") + cp_exists = False + + # 3. Run NPM analysis + print("\n=== Running NPM scan ===") + npm_exists = True + try: + npm_analyzer.run() + except Exception as e: + print(f"Error running NPM scan: {e}") + npm_exists = False + + if not cp_exists and not npm_exists: + print("\nBoth scans failed. Unable to generate consolidated blocklist.") + return + + # 4. Consolidate block candidates + consolidated_candidates: set[str] = set() + if cp_exists: + consolidated_candidates.update(cp_analyzer.block_candidates()) + if npm_exists: + consolidated_candidates.update(npm_analyzer.block_candidates()) + + # Exclude whitelisted IPs + final_blocked_ips = { + ip for ip in consolidated_candidates if not utils.is_whitelisted(ip)} + + # Group into subnets via WHOIS if multiple malicious IPs in same subnet (threshold = 3) + print("\nGrouping blocked IPs into subnets using WHOIS...") + final_blocked_targets = utils.consolidate_by_subnet( + final_blocked_ips, threshold=3) + + print("\n=== Consolidated Summary ===") + print( + f"copyparty block candidates: {len(cp_analyzer.block_candidates()) if cp_exists else 0}") + print( + f"NPM block candidates: {len(npm_analyzer.block_candidates()) if npm_exists else 0}") + print(f"Total unified blocked IPs: {len(final_blocked_ips)}") + print(f"Total target blocks/subnets: {len(final_blocked_targets)}") + + # 5. Write Consolidated MikroTik RouterOS Script + if not args.no_rsc: + rsc_path = os.path.join(args.report_dir, "mikrotik-block.rsc") + utils.ensure_dirs(args.report_dir) + path, n_blocked, n_whitelisted = utils.write_mikrotik( + rsc_path, + "ids", + final_blocked_targets + ) + print(f"\nConsolidated MikroTik script: {path}") + print( + f" {n_blocked} targets blocked, {n_whitelisted} CIDR ranges whitelisted") + + +if __name__ == "__main__": + main() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..9e37bb3 --- /dev/null +++ b/utils.py @@ -0,0 +1,886 @@ +#!/usr/bin/env python3 +"""Shared core for the local-dev intrusion-detection scripts. + +`copyparty.py` and `npm.py` are thin adapters. They only provide: + + * a log-file matcher (glob-ish pattern / suffixes) + * a line parser producing `Request` objects + * a flagging rule (which IPs look hostile) + * column definitions for their CSV output + +Everything else -- IP aggregation, whitelisting, console + file reports, +MikroTik RouterOS export -- lives in this module. +""" + +from __future__ import annotations + +import argparse +import csv +import gzip +import ipaddress +import json +import os +import re +from collections import Counter +from dataclasses import dataclass, field +from datetime import datetime +from typing import Any, Callable, Iterable, Iterator, Sequence + +# --------------------------------------------------------------------------- +# constants +# --------------------------------------------------------------------------- + +ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + +# Whitelists (imported from dedicated module) +from whitelists import ( + WHITELISTS, + WHITELIST_NETS as _WHITELIST_NETS, + refresh_whitelists, +) + +# Malicious pattern detection +import bad_patterns + +#: (header, getter) pair used for CSV + console tables +Column = tuple[str, Callable[[Any], Any]] + + +# --------------------------------------------------------------------------- +# List download helpers +# --------------------------------------------------------------------------- + +# download helpers and whitelist refresh now live in whitelists.py + +# --------------------------------------------------------------------------- +# IP helpers +# --------------------------------------------------------------------------- + +def parse_ip(value: str): + """Return an ip_address object, or None if `value` is not an IP.""" + try: + return ipaddress.ip_address(value) + except (ValueError, TypeError): + return None + + +def ip_version(value: str) -> int | None: + addr = parse_ip(value) + return addr.version if addr else None + + +def is_ipv6(value: str) -> bool: + return ip_version(value) == 6 + + +def is_ipv4(value: str) -> bool: + return ip_version(value) == 4 + + +def whitelist_of(value: str) -> str | None: + """Return the whitelist name an IP belongs to, else None.""" + addr = parse_ip(value) + if addr is None: + return None + for name, nets in _WHITELIST_NETS.items(): + for net in nets: + if addr.version == net.version and addr in net: + return name + return None + + +def is_whitelisted(value: str) -> bool: + return whitelist_of(value) is not None + + +# WHOIS Cache file path +WHOIS_CACHE_PATH = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "reports", "whois_cache.json" +) + +_whois_cache: dict[str, str] = {} + + +def load_whois_cache() -> None: + global _whois_cache + if not _whois_cache and os.path.exists(WHOIS_CACHE_PATH): + try: + with open(WHOIS_CACHE_PATH, "r", encoding="utf-8") as f: + _whois_cache = json.load(f) + except Exception: + _whois_cache = {} + + +def save_whois_cache() -> None: + if _whois_cache: + os.makedirs(os.path.dirname(WHOIS_CACHE_PATH), exist_ok=True) + try: + with open(WHOIS_CACHE_PATH, "w", encoding="utf-8") as f: + json.dump(_whois_cache, f, indent=2) + except Exception: + pass + + +def query_whois_subnet(ip: str) -> str | None: + """Query whois.radb.net for the route/subnet of an IP address.""" + addr = parse_ip(ip) + if addr is None: + return None + + load_whois_cache() + if ip in _whois_cache: + return _whois_cache[ip] or None + + import socket + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s.settimeout(3.0) + s.connect(("whois.radb.net", 43)) + s.sendall(f"{ip}\r\n".encode()) + res = "" + while True: + chunk = s.recv(4096).decode(errors="ignore") + if not chunk: + break + res += chunk + s.close() + + # Find route/route6 patterns + routes = re.findall(r"^(?:route|route6):\s*(\S+)", + res, re.IGNORECASE | re.MULTILINE) + if routes: + # Sort by subnet length descending to find the most specific network (longest prefix) + routes_parsed = [] + for r in routes: + try: + routes_parsed.append(ipaddress.ip_network(r)) + except ValueError: + continue + if routes_parsed: + # Get the most specific one (highest prefixlen) + best_route = max(routes_parsed, key=lambda n: n.prefixlen) + subnet_str = str(best_route) + _whois_cache[ip] = subnet_str + save_whois_cache() + return subnet_str + + except Exception: + pass + + # Fallback to single host subnet + subnet_str = f"{ip}/32" if addr.version == 4 else f"{ip}/128" + _whois_cache[ip] = subnet_str + save_whois_cache() + return subnet_str + + +def consolidate_by_subnet(ips: Iterable[str], threshold: int = 3) -> set[str]: + """Group IPs by WHOIS subnet; if >= threshold IPs in a subnet, block the subnet. + + Otherwise, block individual IPs. + """ + ip_list = list(ips) + subnet_to_ips: dict[str, set[str]] = {} + ip_to_subnet: dict[str, str] = {} + + for ip in ip_list: + subnet = query_whois_subnet(ip) + if subnet: + # Verify the IP is actually in the subnet + try: + net = ipaddress.ip_network(subnet, strict=False) + addr = ipaddress.ip_address(ip) + if addr in net: + subnet_to_ips.setdefault(subnet, set()).add(ip) + ip_to_subnet[ip] = subnet + continue + except ValueError: + pass + # Fallback to self + fallback_net = f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128" + subnet_to_ips.setdefault(fallback_net, set()).add(ip) + ip_to_subnet[ip] = fallback_net + + blocked_targets: set[str] = set() + for subnet, mapped_ips in subnet_to_ips.items(): + if len(mapped_ips) >= threshold: + blocked_targets.add(subnet) + else: + for ip in mapped_ips: + blocked_targets.add( + f"{ip}/32" if ip_version(ip) == 4 else f"{ip}/128") + + return blocked_targets + + +def sort_ips(ips: Iterable[str]) -> list[str]: + """Numeric sort, IPv4 before IPv6, unparseable last.""" + def key(ip: str): + addr = parse_ip(ip) + if addr is None: + return (9, b"", ip) + return (addr.version, addr.packed, "") + return sorted(ips, key=key) + + +def split_versions(ips: Iterable[str]) -> tuple[list[str], list[str]]: + """Split an IP iterable into (sorted IPv4, sorted IPv6).""" + ips = list(ips) + return (sort_ips(ip for ip in ips if is_ipv4(ip)), + sort_ips(ip for ip in ips if is_ipv6(ip))) + + +# --------------------------------------------------------------------------- +# data model +# --------------------------------------------------------------------------- + +@dataclass(slots=True) +class Request: + """One normalised request line, produced by any log parser.""" + + ts: str = "" + ip: str = "" + method: str | None = None + path: str | None = None + status: int | None = None + upstream_status: int | None = None + host: str | None = None + user: str | None = None + ua: str | None = None + referer: str | None = None + denied: bool = False + banned: bool = False + extra: dict = field(default_factory=dict) + + @property + def is_client_error(self) -> bool: + return self.status is not None and 400 <= self.status < 500 + + @property + def is_server_error(self) -> bool: + return self.status is not None and 500 <= self.status < 600 + + @property + def is_error(self) -> bool: + return self.denied or self.is_client_error or self.is_server_error + + +@dataclass +class IPStats: + """Aggregated per-IP counters.""" + + ip: str + version: int = 0 + total: int = 0 + errors: int = 0 + c4xx: int = 0 + c5xx: int = 0 + denied: int = 0 + banned: int = 0 + first: str = "" + last: str = "" + paths: Counter = field(default_factory=Counter) + methods: Counter = field(default_factory=Counter) + hosts: Counter = field(default_factory=Counter) + uas: Counter = field(default_factory=Counter) + users: set[str] = field(default_factory=set) + malicious_hits: int = 0 + malicious_patterns: dict[str, int] = field(default_factory=dict) + + @property + def error_ratio(self) -> float: + return round(self.errors / self.total, 3) if self.total else 0.0 + + @property + def authenticated(self) -> bool: + """True if the IP ever presented a real (non-anonymous) user.""" + return any(u and u != "*" for u in self.users) + + @property + def whitelist(self) -> str: + return whitelist_of(self.ip) or "" + + @property + def users_str(self) -> str: + return ",".join(sorted(self.users)) or "-" + + @property + def methods_str(self) -> str: + return ",".join(sorted(self.methods)) or "-" + + def to_dict(self) -> dict: + return { + "ip": self.ip, + "version": self.version, + "total": self.total, + "errors": self.errors, + "error_ratio": self.error_ratio, + "status_4xx": self.c4xx, + "status_5xx": self.c5xx, + "denied": self.denied, + "banned": self.banned, + "first": self.first, + "last": self.last, + "whitelist": self.whitelist or None, + "users": sorted(self.users), + "methods": dict(self.methods), + "hosts": dict(self.hosts), + "paths": dict(self.paths.most_common()), + "uas": dict(self.uas.most_common(10)), + "malicious_hits": self.malicious_hits, + "malicious_patterns": dict(self.malicious_patterns), + } + + +def aggregate(requests: Iterable[Request]) -> dict[str, IPStats]: + """Fold request records into per-IP statistics.""" + stats: dict[str, IPStats] = {} + for r in requests: + s = stats.get(r.ip) + if s is None: + s = stats[r.ip] = IPStats( + ip=r.ip, version=ip_version(r.ip) or 0, first=r.ts) + s.total += 1 + s.last = r.ts + if r.denied: + s.denied += 1 + if r.banned: + s.banned += 1 + if r.is_client_error: + s.c4xx += 1 + if r.is_server_error: + s.c5xx += 1 + if r.is_error: + s.errors += 1 + if r.path: + s.paths[r.path] += 1 + if r.method: + s.methods[r.method] += 1 + if r.host: + s.hosts[r.host] += 1 + if r.ua and r.ua != "-": + s.uas[r.ua] += 1 + if r.user: + s.users.add(r.user) + if r.path and bad_patterns.is_malicious(r.path): + s.malicious_hits += 1 + for p in bad_patterns.classify_path(r.path): + s.malicious_patterns[p.name] = s.malicious_patterns.get(p.name, 0) + 1 + return stats + + +# --- flagging rules -------------------------------------------------------- + +def flag_scanners(stats: dict[str, IPStats], min_errors: int = 5, + min_ratio: float = 0.5) -> dict[str, IPStats]: + """IPs with many errors AND a high error ratio.""" + return { + ip: s for ip, s in stats.items() + if s.errors >= min_errors and s.error_ratio >= min_ratio + } + + +def flag_denied_only(stats: dict[str, IPStats], + min_denied: int = 1) -> dict[str, IPStats]: + """IPs that only ever hit denied paths and never authenticated.""" + return { + ip: s for ip, s in stats.items() + if s.denied >= min_denied and not s.authenticated + } + + +def flag_malicious_patterns(stats: dict[str, IPStats], + min_severity: int = 2, + min_hits: int = 1) -> dict[str, IPStats]: + """IPs with hits against malicious path patterns at or above severity threshold.""" + return { + ip: s for ip, s in stats.items() + if s.malicious_hits >= min_hits and any( + bad_patterns.PATTERNS_BY_NAME[name].severity >= min_severity + for name in s.malicious_patterns + ) + } + + +def by_total(stats: dict[str, IPStats]) -> list[IPStats]: + """Stats sorted by request count, descending.""" + return sorted(stats.values(), key=lambda s: (-s.total, s.ip)) + + +# --------------------------------------------------------------------------- +# filesystem helpers +# --------------------------------------------------------------------------- + +def validate_dir(path: str, need_write: bool = False) -> str: + if not os.path.exists(path): + raise FileNotFoundError(f"path does not exist: {path}") + if not os.path.isdir(path): + raise NotADirectoryError(f"path is not a directory: {path}") + if not os.access(path, os.R_OK): + raise PermissionError(f"path not readable: {path}") + if need_write and not os.access(path, os.W_OK): + raise PermissionError(f"path not writable: {path}") + return path + + +def ensure_dirs(*paths: str) -> None: + for p in paths: + os.makedirs(p, exist_ok=True) + + +def find_logs(path: str, pattern: re.Pattern | None = None, + suffixes: Sequence[str] = (), + exclude_prefixes: Sequence[str] = ()) -> list[str]: + """List matching log file names (not full paths), sorted.""" + out = [] + for name in os.listdir(path): + if not os.path.isfile(os.path.join(path, name)): + continue + if any(name.startswith(p) for p in exclude_prefixes): + continue + if pattern is not None and not pattern.match(name): + continue + if suffixes and not name.endswith(tuple(suffixes)): + continue + out.append(name) + return sorted(out) + + +def read_lines(path: str) -> Iterator[str]: + """Yield ANSI-stripped, newline-stripped lines. Transparently gunzips.""" + opener = gzip.open if path.endswith(".gz") else open + with opener(path, "rt", encoding="utf-8", errors="replace") as f: + for line in f: + yield ANSI_RE.sub("", line.rstrip("\r\n")) + + +# --------------------------------------------------------------------------- +# output helpers +# --------------------------------------------------------------------------- + +def write_csv(path: str, header: Sequence[str], + rows: Iterable[Sequence[Any]]) -> str: + with open(path, "w", newline="", encoding="utf-8") as f: + w = csv.writer(f) + w.writerow(header) + w.writerows(rows) + return path + + +def write_json(path: str, obj: Any) -> str: + with open(path, "w", encoding="utf-8") as f: + json.dump(obj, f, indent=2, default=str) + return path + + +def write_lines(path: str, lines: Iterable[str]) -> str: + with open(path, "w", encoding="utf-8") as f: + for line in lines: + f.write(line + "\n") + return path + + +def columns_header(columns: Sequence[Column]) -> list[str]: + return [c[0] for c in columns] + + +def columns_row(columns: Sequence[Column], obj: Any) -> list[Any]: + return [fn(obj) for _, fn in columns] + + +def render_table(headers: Sequence[str], rows: Sequence[Sequence[Any]], + max_width: int = 48) -> str: + """Aligned fixed-width text table. Numeric columns right-aligned.""" + def cell(v: Any) -> str: + s = "" if v is None else str(v) + return s if len(s) <= max_width else s[:max_width - 1] + "~" + + body = [[cell(v) for v in row] for row in rows] + aligns = [] + for i in range(len(headers)): + numeric = bool(rows) and all( + isinstance(row[i], (int, float)) and not isinstance(row[i], bool) + for row in rows) + aligns.append(">" if numeric else "<") + widths = [len(h) for h in headers] + for row in body: + for i, v in enumerate(row): + widths[i] = max(widths[i], len(v)) + + def line(values: Sequence[str]) -> str: + return " ".join( + f"{v:{a}{w}}" for v, a, w in zip(values, aligns, widths)).rstrip() + + out = [line(list(headers)), "-" * (sum(widths) + 2 * (len(widths) - 1))] + out += [line(row) for row in body] + return "\n".join(out) + + +def render_kv(rows: Sequence[tuple[str, Any]]) -> str: + width = max((len(k) for k, _ in rows), default=0) + 1 + return "\n".join(f"{k + ':':<{width + 1}} {v}" for k, v in rows) + + +def timestamp_name() -> str: + return datetime.now().strftime("%Y-%m-%d-%H-%M-%S") + + +# --------------------------------------------------------------------------- +# MikroTik RouterOS export +# --------------------------------------------------------------------------- + +def split_cidr_versions(targets: Iterable[str]) -> tuple[list[str], list[str]]: + """Split target CIDRs/IPs into (sorted IPv4 list, sorted IPv6 list).""" + v4 = [] + v6 = [] + for t in targets: + # t might have CIDR suffix + ip = t.split("/")[0] + if is_ipv4(ip): + v4.append(t) + elif is_ipv6(ip): + v6.append(t) + + # Sort CIDR/IP strings + def key_cidr(cidr: str): + ip = cidr.split("/")[0] + addr = parse_ip(ip) + if addr is None: + return (9, b"", cidr) + return (addr.version, addr.packed, cidr) + + return sorted(v4, key=key_cidr), sorted(v6, key=key_cidr) + + +def mikrotik_script(tag: str, blocked_ips_or_subnets: Iterable[str], + whitelists: dict[str, list[str]] | None = None, + generated: str | None = None) -> tuple[str, int, int]: + """Build a RouterOS .rsc script. + + Whitelisted ranges become `accept` rules placed above the `drop` rules. + Blocked targets can be individual IPs (e.g. 1.2.3.4/32) or CIDR blocks (e.g. 1.2.3.0/24). + + Returns (script_text, blocked_count, whitelisted_cidr_count). + """ + whitelists = WHITELISTS if whitelists is None else whitelists + generated = generated or datetime.now().isoformat(timespec="seconds") + + # Filter out whitelisted entries (check individual IP representation or base subnet) + def clean_blocked(targets: Iterable[str]) -> list[str]: + cleaned = [] + for t in targets: + ip = t.split("/")[0] + if not is_whitelisted(ip): + cleaned.append(t) + return cleaned + + blocked = clean_blocked(set(blocked_ips_or_subnets)) + block_v4, block_v6 = split_cidr_versions(blocked) + + lines: list[str] = [f"# {tag} firewall export - generated {generated}", ""] + wl_count = 0 + + for name, cidrs in whitelists.items(): + nets = [ipaddress.ip_network(c) for c in cidrs] + wl_count += len(nets) + for version, cmd in ((4, "/ip firewall address-list"), + (6, "/ipv6 firewall address-list")): + chunk = [n for n in nets if n.version == version] + if not chunk: + continue + lines.append(f"# {name} IPv{version} ranges (whitelisted)") + lines.append(cmd) + lines += [f"add list={name} address={n}" for n in chunk] + lines.append("") + + if block_v4: + lines.append(f"# Blocked IPv4 ({len(block_v4)}, non-whitelisted)") + lines.append("/ip firewall address-list") + lines += [f"add list=Blocked address={t}" for t in block_v4] + lines.append("") + if block_v6: + lines.append(f"# Blocked IPv6 ({len(block_v6)}, non-whitelisted)") + lines.append("/ipv6 firewall address-list") + lines += [f"add list=Blocked address={t}" for t in block_v6] + lines.append("") + + for version, cmd, blocks in ((4, "/ip firewall filter", block_v4), + (6, "/ipv6 firewall filter", block_v6)): + lines.append(cmd) + for name in whitelists: + lines.append( + f"add chain=input action=accept src-address-list={name} " + f'comment="{tag} {name.lower()} whitelist (v{version})"') + if blocks: + for chain in ("input", "forward"): + lines.append( + f"add chain={chain} action=drop src-address-list=Blocked " + f'comment="{tag} blocked (v{version})"') + lines.append("") + + return "\n".join(lines), len(blocked), wl_count + + +def write_mikrotik(path: str, tag: str, + blocked_ips_or_subnets: Iterable[str]) -> tuple[str, int, int]: + text, n_blocked, n_whitelisted = mikrotik_script( + tag, blocked_ips_or_subnets) + with open(path, "w", encoding="utf-8") as f: + f.write(text) + return path, n_blocked, n_whitelisted + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def arg_parser(description: str, default_log_path: str, + default_report_dir: str) -> argparse.ArgumentParser: + p = argparse.ArgumentParser(description=description) + p.add_argument("--log-path", default=default_log_path, + help=f"log directory (default: {default_log_path})") + p.add_argument("--report-dir", default=default_report_dir, + help=f"report output dir (default: {default_report_dir})") + p.add_argument("--top", type=int, default=30, + help="rows in the top-IP table (default: 30)") + p.add_argument("--quiet", action="store_true", + help="write files, print nothing but the paths") + p.add_argument("--no-rsc", action="store_true", + help="skip MikroTik .rsc generation") + return p + + +# --------------------------------------------------------------------------- +# analyzer +# --------------------------------------------------------------------------- + +class Analyzer: + """Log-source-agnostic analysis pipeline. + + Subclasses must set the class attributes below and implement + `parse_line()` and `flag()`. + """ + + name = "ids" # short id, used in file names / tags + title = "IDS log analysis report" + log_pattern: re.Pattern | None = None + log_suffixes: tuple[str, ...] = () + exclude_prefixes: tuple[str, ...] = () + ip_columns: Sequence[Column] = () + request_columns: Sequence[Column] = () + paths_file = "error_paths.csv" + paths_label = "error paths" + flag_files = ("flagged_ips.txt", "flagged_ipv6.txt") + flag_label = "Flagged IPs" + flag_note = "" + + def __init__(self, log_path: str, report_dir: str, top: int = 30, + quiet: bool = False, write_rsc: bool = True): + self.log_path = log_path + self.report_dir = report_dir + self.data_dir = os.path.join(report_dir, "data") + self.top = top + self.quiet = quiet + self.write_rsc = write_rsc + self.files: list[str] = [] + self.requests: list[Request] = [] + self.stats: dict[str, IPStats] = {} + self.flagged: dict[str, IPStats] = {} + self.per_file: dict[str, int] = {} + + # --- hooks ------------------------------------------------------------- + + def parse_line(self, line: str) -> Request | None: + raise NotImplementedError + + def flag(self, stats: dict[str, IPStats]) -> dict[str, IPStats]: + raise NotImplementedError + + def flag_comment(self, s: IPStats) -> str: + return (f"errors={s.errors} total={s.total} ratio={s.error_ratio} " + f"first={s.first} last={s.last}") + + def block_candidates(self) -> set[str]: + malicious = flag_malicious_patterns(self.stats) + return set(self.flagged) | set(malicious) + + def extra_summary_rows(self) -> list[tuple[str, Any]]: + return [] + + def write_extra(self, data_dir: str) -> list[str]: + return [] + + # --- pipeline ---------------------------------------------------------- + + def discover(self) -> list[str]: + self.files = find_logs(self.log_path, self.log_pattern, + self.log_suffixes, self.exclude_prefixes) + if not self.files: + raise FileNotFoundError(f"no log files found in: {self.log_path}") + return self.files + + def load(self) -> list[Request]: + for name in self.files: + count = 0 + for line in read_lines(os.path.join(self.log_path, name)): + req = self.parse_line(line) + if req is not None: + self.requests.append(req) + count += 1 + self.per_file[name] = count + return self.requests + + def analyze(self) -> dict[str, IPStats]: + self.stats = aggregate(self.requests) + self.flagged = self.flag(self.stats) + return self.stats + + # --- derived data ------------------------------------------------------ + + def error_paths(self) -> Counter: + return Counter(r.path for r in self.requests if r.is_error and r.path) + + def flagged_split(self) -> tuple[list[IPStats], list[IPStats]]: + v4 = [s for s in self.flagged.values() if s.version == 4] + v6 = [s for s in self.flagged.values() if s.version == 6] + def key(s): return (-s.errors, s.ip) + return sorted(v4, key=key), sorted(v6, key=key) + + def summary_rows(self) -> list[tuple[str, Any]]: + v4, v6 = split_versions(self.stats) + rows = [ + ("log path", self.log_path), + ("log files", len(self.files)), + ("total requests", len(self.requests)), + ("unique IPs", + f"{len(self.stats)} (IPv4 {len(v4)} / IPv6 {len(v6)})"), + ("error requests", sum(1 for r in self.requests if r.is_error)), + (f"unique {self.paths_label}", len(self.error_paths())), + ] + rows += self.extra_summary_rows() + rows += [ + (self.flag_label.lower(), len(self.flagged)), + ("whitelisted IPs seen", + sum(1 for ip in self.stats if is_whitelisted(ip))), + ("block candidates", len(self.block_candidates())), + ] + return rows + + # --- reporting (one text body, used for console AND file) -------------- + + def text_report(self) -> str: + out = [self.title, + f"generated: {datetime.now().isoformat(timespec='seconds')}", + "", + render_kv(self.summary_rows()), ""] + + out.append(f"== top {self.top} IPs ==") + rows = [columns_row(self.ip_columns, s) + for s in by_total(self.stats)[:self.top]] + out.append(render_table(columns_header(self.ip_columns), rows)) + if len(self.stats) > self.top: + out.append(f"... {len(self.stats) - self.top} more IPs omitted") + out.append("") + + out.append(f"== top 20 {self.paths_label} ==") + for path, n in self.error_paths().most_common(20): + out.append(f" {n:>5} {path}") + out.append("") + + v4, v6 = self.flagged_split() + out.append(f"== {self.flag_label} ({len(self.flagged)}) ==") + if self.flag_note: + out.append(self.flag_note) + for label, group in (("IPv4", v4), ("IPv6", v6)): + if not group: + continue + out.append(f"-- {label} ({len(group)}) --") + rows = [columns_row(self.ip_columns, s) for s in group] + out.append(render_table(columns_header(self.ip_columns), rows)) + out.append("") + return "\n".join(out) + + # --- writing ----------------------------------------------------------- + + def write_data(self) -> list[str]: + ensure_dirs(self.report_dir, self.data_dir) + written: list[str] = [] + d = self.data_dir + header = columns_header(self.ip_columns) + ordered = by_total(self.stats) + + for version, fname in ((4, "ips_ipv4.csv"), (6, "ips_ipv6.csv")): + rows = [columns_row(self.ip_columns, s) + for s in ordered if s.version == version] + written.append(write_csv(os.path.join(d, fname), header, rows)) + + written.append(write_json( + os.path.join(d, "ips.json"), + {s.ip: s.to_dict() for s in ordered})) + + written.append(write_csv( + os.path.join(d, self.paths_file), ["path", "count"], + self.error_paths().most_common())) + + v4, v6 = self.flagged_split() + for group, fname in zip((v4, v6), self.flag_files): + written.append(write_lines( + os.path.join(d, fname), + [f"{s.ip} # {self.flag_comment(s)}" for s in group])) + + blocked_v4, blocked_v6 = split_versions( + ip for ip in self.block_candidates() if not is_whitelisted(ip)) + written.append(write_lines( + os.path.join(d, "blocked_ipv4.txt"), blocked_v4)) + written.append(write_lines( + os.path.join(d, "blocked_ipv6.txt"), blocked_v6)) + + written.append(write_csv( + os.path.join(d, "requests.csv"), + columns_header(self.request_columns), + (columns_row(self.request_columns, r) for r in self.requests))) + + written += self.write_extra(d) + return written + + def write_report(self) -> str: + ensure_dirs(self.report_dir) + path = os.path.join(self.report_dir, f"{timestamp_name()}.txt") + with open(path, "w", encoding="utf-8") as f: + f.write(self.text_report()) + return path + + def write_rsc_file(self) -> tuple[str, int, int]: + ensure_dirs(self.data_dir) + return write_mikrotik( + os.path.join(self.data_dir, "mikrotik-block.rsc"), + f"{self.name}-ids", self.block_candidates()) + + # --- entrypoint -------------------------------------------------------- + + def run(self) -> dict[str, Any]: + validate_dir(self.log_path) + ensure_dirs(self.report_dir, self.data_dir) + refresh_whitelists() + self.discover() + self.load() + self.analyze() + + if not self.quiet: + print(self.text_report()) + + self.write_data() + report_path = self.write_report() + print(f"data files: {self.data_dir}") + print(f"summary report: {report_path}") + + result: dict[str, Any] = {"report": report_path, + "data_dir": self.data_dir} + if self.write_rsc: + rsc, n_blocked, n_whitelisted = self.write_rsc_file() + print(f"mikrotik script: {rsc}") + print(f" {n_blocked} IPs blocked, " + f"{n_whitelisted} CIDR ranges whitelisted") + result.update(rsc=rsc, blocked=n_blocked, + whitelisted=n_whitelisted) + return result diff --git a/whitelists.py b/whitelists.py new file mode 100644 index 0000000..e5edea5 --- /dev/null +++ b/whitelists.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python3 +"""Whitelist management for the intrusion-detection system. + +Owns provider CIDR constants, remote download/refresh logic, and +CIDR extraction/validation. Imported by `utils.py` which provides +the convenience lookups `whitelist_of()` and `is_whitelisted()`. + +Whitelist loading order (per provider): + 1. Local file in `data//` (if present and readable) + 2. Remote download (if network available) + 3. Built-in hardcoded fallback +""" + +from __future__ import annotations + +import ipaddress +import os +import re +from typing import Any + +# --------------------------------------------------------------------------- +# Paths +# --------------------------------------------------------------------------- + +_HERE = os.path.dirname(os.path.abspath(__file__)) +_DATA_DIR = os.path.join(_HERE, "data") + +# --------------------------------------------------------------------------- +# CIDR extraction regex +# --------------------------------------------------------------------------- + +CIDR_RE = re.compile( + r"(? CIDR ranges (v4 + v6 mixed, split at render time) +WHITELISTS: dict[str, list[str]] = { + "Openfactory": OPENFACTORY_IPV4, + "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6, + "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6, +} + +#: pre-parsed networks, so matching does not re-parse CIDRs per IP +WHITELIST_NETS: dict[str, tuple[Any, ...]] = { + name: tuple(ipaddress.ip_network(c) for c in cidrs) + for name, cidrs in WHITELISTS.items() +} + +#: guard so refresh runs at most once per process +_refreshed = False + + +# --------------------------------------------------------------------------- +# CIDR extraction / validation +# --------------------------------------------------------------------------- + +def extract_cidrs(text: str) -> list[str]: + """Extract valid CIDR strings from plain text, CSV, or HTML.""" + seen: set[str] = set() + cidrs: list[str] = [] + for match in CIDR_RE.finditer(text): + candidate = match.group(1).strip() + try: + network = ipaddress.ip_network(candidate, strict=False) + except ValueError: + continue + rendered = str(network) + if rendered not in seen: + seen.add(rendered) + cidrs.append(rendered) + return cidrs + + +# --------------------------------------------------------------------------- +# Local file loading +# --------------------------------------------------------------------------- + +def _read_local_file(path: str) -> list[str]: + """Read a local whitelist file and extract valid CIDRs.""" + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + return extract_cidrs(f.read()) + except Exception: + return [] + + +def _load_local_whitelists() -> dict[str, list[str]]: + """Load whitelist CIDRs from local data/ files. + + Returns a dict of {name: [CIDR strings]}. Only keys with + successfully parsed CIDRs are included. + """ + out: dict[str, list[str]] = {} + cidrs = _read_local_file(OPENFACTORY_FILE) + if cidrs: + out["Openfactory"] = cidrs + cf_v4 = _read_local_file(CLOUDFLARE_V4_FILE) + cf_v6 = _read_local_file(CLOUDFLARE_V6_FILE) + if cf_v4 or cf_v6: + out["Cloudflare"] = cf_v4 + cf_v6 + cidrs = _read_local_file(PALOALTO_FILE) + if cidrs: + out["PaloAlto"] = cidrs + return out + + +# --------------------------------------------------------------------------- +# HTTP download helpers +# --------------------------------------------------------------------------- + +def _download_text(url: str) -> str: + """Download a text file from a URL, return as string.""" + import urllib.request + request = urllib.request.Request( + url, + headers={ + "User-Agent": "Mozilla/5.0 (compatible; intrusion-detection/1.0)", + "Accept": "text/plain,text/csv,text/html;q=0.9,*/*;q=0.8", + }, + ) + with urllib.request.urlopen(request) as response: + return response.read().decode("utf-8", errors="replace") + + +def _download_cidrs(url: str) -> list[str]: + """Download a whitelist source and extract valid CIDRs from it.""" + return extract_cidrs(_download_text(url)) + + +# --------------------------------------------------------------------------- +# Refresh pipeline +# --------------------------------------------------------------------------- + +def _download_whitelists() -> dict[str, list[str]]: + """Download the latest whitelists from the Internet and save to local files. + + Returns a dict of {name: [CIDR strings]}. Only keys with + successfully parsed CIDRs are included; failures are logged + and skipped so built-in fallbacks remain untouched. + Saves each download to the corresponding local file on success. + """ + out: dict[str, list[str]] = {} + try: + text = _download_text(OPENFACTORY_URL) + cidrs = extract_cidrs(text) + if cidrs: + out["Openfactory"] = cidrs + _save_local_file(OPENFACTORY_FILE, text) + except Exception as e: + print(f"Error downloading Openfactory whitelist: {e}") + try: + text_v4 = _download_text(CLOUDFLARE_URL_V4) + text_v6 = _download_text(CLOUDFLARE_URL_V6) + cidrs = extract_cidrs(text_v4) + extract_cidrs(text_v6) + if cidrs: + out["Cloudflare"] = cidrs + _save_local_file(CLOUDFLARE_V4_FILE, text_v4) + _save_local_file(CLOUDFLARE_V6_FILE, text_v6) + except Exception as e: + print(f"Error downloading Cloudflare whitelist: {e}") + # Do not download PaloAlto whitelist for now, as it is not a reliable source of CIDRs and often returns HTML instead of plain text. + # try: + # text = _download_text(PALOALTO_URL) + # cidrs = extract_cidrs(text) + # if cidrs and len(cidrs) > 0: + # out["PaloAlto"] = cidrs + # _save_local_file(PALOALTO_FILE, text) + # except Exception as e: + # print(f"Error downloading PaloAlto whitelist: {e}") + return out + + +def _save_local_file(path: str, text: str) -> None: + """Save downloaded text to a local file, creating parent dirs as needed.""" + try: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8") as f: + f.write(text) + except Exception as e: + print(f"Error saving whitelist file {path}: {e}") + + +def update_whitelists() -> None: + """Load whitelists (local files first, then remote, then built-in fallback). + + Priority per provider: + 1. Local file in data// + 2. Remote download (saves to local file on success) + 3. Built-in hardcoded constants (always available) + """ + global WHITELISTS, WHITELIST_NETS + + # Start from built-in fallbacks, then layer local, then remote + WHITELISTS = { + "Openfactory": OPENFACTORY_IPV4, + "Cloudflare": CLOUDFLARE_IPV4 + CLOUDFLARE_IPV6, + "PaloAlto": PALOALTO_IPV4 + PALOALTO_IPV6, + } + + # Layer 1: local files + local = _load_local_whitelists() + if local: + WHITELISTS.update(local) + + # Layer 2: remote download (highest priority, saves local) + remote = _download_whitelists() + if remote: + WHITELISTS.update(remote) + + # Rebuild parsed networks + WHITELIST_NETS = { + name: tuple(ipaddress.ip_network(c) for c in cidrs) + for name, cidrs in WHITELISTS.items() + } + + +def refresh_whitelists(force: bool = False) -> bool: + """Refresh provider whitelists once per process, keeping built-in fallbacks. + + Returns True if a refresh was attempted, False if already done. + """ + global _refreshed + if _refreshed and not force: + return False + update_whitelists() + _refreshed = True + return True