feat: make MikroTik blocklist cumulative via git history

This commit is contained in:
zwitschi committed 2026-08-06 13:21:02 +02:00
1 parent c131759a7c
commit 894a2eec33
12 files changed
+3582 -129

No files matched your search

+20
View File
@@ -184,6 +184,26 @@ python -m app.copyparty --top 100
python app.py scan --quiet --update-firewall --dry-run
```
## Blocklist behavior (cumulative / grow-only)
The MikroTik `Blocked` address list is **cumulative**: it only grows over
time and is never trimmed automatically.
- An IP (or subnet) that the system has ever blocked stays blocked — even if
it stops appearing in recent logs (for example after it leaves the NPM
~30-day log-retention window, or once WHOIS subnet consolidation groups it
into a larger range).
- The **source of truth** for "all blocked IPs" is the git history of the
consolidated blocklist files `reports/app/data/blocklist_ipv4.txt` and
`blocklist_ipv6.txt` (these files are git-tracked). After each scan the
updated blocklist files should be committed so the history keeps growing.
- The sync only ever **adds** entries. The only automatic removals are router
entries that were never blocked by the system — i.e. absent from every git
version of the blocklist files (manual / rogue entries).
- To explicitly unblock an IP, remove it manually via the web UI
(`/blocklist` → remove) or directly on the router; the automated sync will
not re-add entries that are not part of a scan.
## Web UI
`python app.py` starts a **localhost-only** Flask web UI (default
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env python3
"""Git-backed cumulative blocklist history.
The consolidated blocklist lives in git-tracked files
(``reports/app/data/blocklist_ipv4.txt`` / ``blocklist_ipv6.txt``). To make the
MikroTik blocklist grow-only, the "complete set of all blocked IPs" is
reconstructed from **every git revision** of those files (the union of all
lines ever present), plus the current working files and the latest scan
targets.
This keeps IPs blocked even after they leave the NPM ~30-day log window, and
only ever removes router entries that were never blocked by the system.
"""
from __future__ import annotations
import ipaddress
import os
import re
import subprocess
from typing import Any, Iterable
#: A ``+ 1.2.3.4/32`` added line in a git patch; ``+++ b/...`` is the file
#: header and is excluded by the negative lookahead.
_ADDED_LINE_RE = re.compile(r"^\+(?!\+)(.*)$")
def _run_git(*args: str) -> subprocess.CompletedProcess[str] | None:
"""Run a git command, returning None on any failure (never raises)."""
try:
return subprocess.run(
["git", *args], capture_output=True, text=True, check=False)
except (OSError, subprocess.SubprocessError):
return None
def _valid_line(line: str) -> str | None:
"""Normalize a blocklist line to a canonical CIDR string, or None."""
line = line.strip()
if not line:
return None
try:
return str(ipaddress.ip_network(line, strict=False))
except ValueError:
return None
def committed_lines(path: str) -> set[str]:
"""Return the union of all lines ever present in ``path`` across git.
Uses a single ``git log --all -p -- <path>`` and collects the ``+`` added
lines (excluding the ``+++`` file header). A line present in any revision
appears as an added line in some commit's diff, so the union equals "all
lines ever present".
Returns an empty set when git is unavailable, the path is untracked, or the
repository has no history — callers then fall back to the working file.
"""
proc = _run_git("log", "--all", "-p", "--", path)
if proc is None or proc.returncode != 0:
return set()
lines: set[str] = set()
for raw in proc.stdout.splitlines():
m = _ADDED_LINE_RE.match(raw)
if not m:
continue
clean = _valid_line(m.group(1))
if clean is not None:
lines.add(clean)
return lines
def current_file_lines(path: str) -> set[str]:
"""Return the validated lines currently in ``path`` (missing file -> {})."""
try:
with open(path, "r", encoding="utf-8") as f:
return {clean for clean in (_valid_line(x) for x in f)
if clean is not None}
except OSError:
return set()
def cumulative_blocklist(v4_path: str, v6_path: str,
desired_v4: Iterable[str],
desired_v6: Iterable[str],
) -> tuple[set[str], set[str]]:
"""Complete set of all blocked targets (history | working file | desired).
Returns ``(complete_v4, complete_v6)`` of canonical CIDR strings.
"""
complete_v4 = (
committed_lines(v4_path)
| current_file_lines(v4_path)
| {c for c in (_valid_line(x) for x in desired_v4) if c is not None})
complete_v6 = (
committed_lines(v6_path)
| current_file_lines(v6_path)
| {c for c in (_valid_line(x) for x in desired_v6) if c is not None})
return complete_v4, complete_v6
def compute_changes(current_v4: set[str], current_v6: set[str],
complete_v4: set[str], complete_v6: set[str],
whitelisted_v4: set[str], whitelisted_v6: set[str],
) -> dict[str, Any]:
"""Compute the add/remove diff for a cumulative (grow-only) blocklist.
Entries are added when present in the complete set but not yet on the
router and not whitelisted. Entries are removed only when they sit on the
router but are absent from the complete set — i.e. they were never blocked
by the system (manual/rogue entries). Callers may attach extra keys (e.g.
``dry_run``).
"""
return {
"added_ipv4": sorted(complete_v4 - current_v4 - whitelisted_v4),
"added_ipv6": sorted(complete_v6 - current_v6 - whitelisted_v6),
"removed_ipv4": sorted(current_v4 - complete_v4),
"removed_ipv6": sorted(current_v6 - complete_v6),
}
+1 -1
View File
@@ -108,7 +108,7 @@ DEFAULT_MIN_DENIED = int(os.environ.get(
"DEFAULT_MIN_DENIED", "1")) # copyparty
DEFAULT_MIN_ERRORS = int(os.environ.get("DEFAULT_MIN_ERRORS", "5")) # npm
DEFAULT_ERROR_RATIO = float(os.environ.get(
"DEFAULT_ERROR_RATIO", "0.5")) # npm
"DEFAULT_ERROR_RATIO", "0.4")) # npm
DEFAULT_TOP_PATHS = int(os.environ.get("DEFAULT_TOP_PATHS", "20"))
# ---------------------------------------------------------------------------
+23 -8
View File
@@ -182,7 +182,16 @@ class MikrotikFirewall:
blocked_ipv6: Iterable[str],
dry_run: bool = False,
report_dir: str = REPORT_DIR) -> dict[str, list[str]]:
"""Synchronize the Blocked list with scan targets and audit the result."""
"""Synchronize the Blocked list cumulatively and audit the result.
The blocklist is grow-only: targets are added when present in the
complete set (all git history of the blocklist files + the current
working files + this scan's targets) and never removed automatically.
Only router entries absent from the complete set — i.e. never blocked
by the system — are pruned. See ``app.blocklist_history``.
"""
from .blocklist_history import compute_changes, cumulative_blocklist
desired_ipv4 = set(blocked_ipv4)
desired_ipv6 = set(blocked_ipv6)
current_ipv4 = self.get_address_list_ipv4(BLOCKLIST_NAME)
@@ -193,13 +202,19 @@ class MikrotikFirewall:
whitelisted_ipv4.update(self.get_address_list_ipv4(name))
whitelisted_ipv6.update(self.get_address_list_ipv6(name))
changes = {
"added_ipv4": sorted(desired_ipv4 - current_ipv4 - whitelisted_ipv4),
"added_ipv6": sorted(desired_ipv6 - current_ipv6 - whitelisted_ipv6),
"removed_ipv4": sorted(current_ipv4 - desired_ipv4),
"removed_ipv6": sorted(current_ipv6 - desired_ipv6),
"dry_run": dry_run,
}
v4_path = os.path.join(
report_dir, "app", "data", "blocklist_ipv4.txt")
v6_path = os.path.join(
report_dir, "app", "data", "blocklist_ipv6.txt")
complete_v4, complete_v6 = cumulative_blocklist(
v4_path, v6_path, desired_ipv4, desired_ipv6)
changes = compute_changes(
current_ipv4, current_ipv6,
complete_v4, complete_v6,
whitelisted_ipv4, whitelisted_ipv6)
changes["dry_run"] = dry_run
if not dry_run:
for ip in changes["removed_ipv4"]:
self.execute_command(
+6 -1
View File
@@ -348,7 +348,12 @@ def create_app() -> Flask:
@app.post("/api/firewall/apply")
def firewall_apply():
"""Apply the blocklist sync (add new / remove stale entries)."""
"""Apply the blocklist sync (cumulative — adds new entries only).
The blocklist is grow-only: it adds entries present in the complete
set (git history + working files + scan targets) and only prunes
router entries never seen in any git version of the blocklist files.
"""
return _sync_firewall(dry_run=False)
# --- whitelist ---------------------------------------------------------
+54
View File
@@ -3790,5 +3790,59 @@
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "147.93.168.91",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "147.93.169.103",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "207.175.20.232",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "23.180.120.134",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "34.156.139.95",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "34.48.168.223",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "51.120.79.193",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "35.196.132.85",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "66.249.74.164",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
}
]
+6
View File
@@ -349,5 +349,11 @@
"creation-time": "2026-08-05",
"dynamic": "no",
"list": "Blocked"
},
{
"address": "2a00:6340::/32",
"creation-time": "2026-08-06",
"dynamic": "no",
"list": "Blocked"
}
]
+3331
View File
File diff suppressed because it is too large. Load diff
+7 -1
View File
@@ -984,5 +984,11 @@
"45.148.10.40": "45.148.10.0/24",
"35.234.181.205": "35.234.176.0/20",
"4.225.203.22": "4.224.0.0/12",
"188.166.227.85": "188.166.224.0/20"
"188.166.227.85": "188.166.224.0/20",
"207.175.20.232": "207.175.0.0/17",
"2a09:bac5:3e0d:11c3::1c5:145": "2a09:bac5::/32",
"34.48.168.223": "34.48.0.0/16",
"20.91.199.21": "20.64.0.0/10",
"34.156.139.95": "34.156.128.0/17",
"2a00:6340:2008:ec80:1ca2:6634:b8cd:fb01": "2a00:6340::/32"
}
+13 -114
View File
@@ -7,9 +7,7 @@
103.221.220.62/32
103.76.88.36/32
103.76.88.37/32
104.168.93.75/32
104.196.62.35/32
104.199.182.104/32
104.208.94.19/32
104.234.19.67/32
104.238.222.26/32
@@ -24,32 +22,24 @@
104.28.247.220/32
104.28.254.131/32
104.28.254.134/32
106.117.109.102/32
106.63.0.0/17
107.170.9.103/32
108.130.204.104/32
110.176.0.0/13
111.113.88.210/32
111.113.89.107/32
112.94.252.15/32
113.57.186.101/32
117.14.153.168/32
118.194.253.208/32
118.212.120.78/32
118.81.196.142/32
118.81.207.202/32
121.29.51.109/32
123.158.61.179/32
123.160.175.254/32
123.163.114.150/32
123.178.210.228/32
124.117.194.69/32
124.88.113.10/32
128.199.182.152/32
128.199.182.55/32
128.90.128.13/32
128.90.141.10/32
129.212.237.196/32
13.126.86.18/32
13.221.106.198/32
13.57.15.41/32
@@ -63,7 +53,6 @@
132.196.83.33/32
134.122.94.138/32
134.199.148.147/32
134.209.162.128/32
134.209.185.72/32
134.209.25.199/32
135.119.63.61/32
@@ -71,18 +60,14 @@
135.225.93.215/32
135.232.177.160/32
136.107.208.33/32
136.107.217.8/32
136.108.11.195/32
136.108.15.119/32
136.109.87.37/32
136.110.80.8/32
136.115.17.125/32
136.116.215.248/32
136.144.19.51/32
136.144.33.102/32
136.144.35.201/32
136.144.43.127/32
136.64.132.60/32
136.64.51.224/32
136.66.128.85/32
136.66.153.34/32
@@ -95,7 +80,6 @@
139.59.118.17/32
139.59.231.238/32
140.248.75.106/32
141.148.153.213/32
141.98.11.171/32
142.111.152.16/32
142.111.152.57/32
@@ -106,9 +90,7 @@
143.198.25.99/32
143.198.6.225/32
143.20.64.236/32
143.244.162.58/32
143.244.168.161/32
143.244.186.167/32
143.244.56.0/23
143.244.57.123/32
144.126.207.57/32
@@ -124,13 +106,13 @@
146.70.194.0/24
146.70.40.68/32
147.182.149.75/32
147.93.168.0/22
147.93.168.91/32
147.93.169.103/32
147.93.171.188/32
148.251.182.79/32
149.56.150.98/32
15.206.146.240/32
150.255.94.250/32
151.240.45.69/32
152.39.198.115/32
152.42.137.175/32
152.53.242.93/32
@@ -141,15 +123,10 @@
157.245.113.227/32
157.245.197.36/32
157.245.204.205/32
157.245.63.62/32
158.101.144.198/32
158.158.0.0/16
158.51.126.144/32
159.203.13.62/32
159.223.132.86/32
159.223.79.25/32
159.65.10.116/32
159.65.144.72/32
159.89.117.109/32
159.89.12.166/32
159.89.123.252/32
@@ -159,19 +136,12 @@
16.146.99.254/32
160.179.166.124/32
160.179.166.222/32
161.118.245.12/32
161.118.251.181/32
162.141.167.14/32
162.141.167.71/32
162.243.79.174/32
163.7.11.71/32
163.7.12.0/24
163.7.13.0/24
164.90.169.163/32
165.22.147.152/32
165.22.235.3/32
165.22.24.78/32
165.245.185.158/32
165.245.190.205/32
167.172.192.26/32
167.172.31.131/32
@@ -181,7 +151,6 @@
167.86.82.167/32
167.99.181.249/32
167.99.182.39/32
167.99.204.6/32
167.99.210.137/32
168.144.100.227/32
168.144.106.212/32
@@ -190,7 +159,6 @@
168.144.83.150/32
168.62.180.116/32
170.64.144.116/32
170.64.156.114/32
171.104.81.195/32
172.170.254.20/32
172.171.198.214/32
@@ -200,12 +168,8 @@
172.213.216.126/32
172.232.175.132/32
172.245.102.64/32
172.86.119.249/32
172.86.122.106/32
172.98.33.251/32
173.239.196.33/32
173.239.196.35/32
173.239.240.222/32
173.249.34.197/32
176.227.240.17/32
176.227.240.61/32
@@ -213,25 +177,20 @@
178.16.54.137/32
178.62.196.238/32
179.43.134.114/32
18.142.90.109/32
18.191.56.164/32
18.227.73.7/32
18.236.136.230/32
180.95.231.78/32
182.119.230.145/32
182.88.226.154/32
185.151.21.124/32
185.177.72.0/24
185.177.72.29/32
185.192.71.0/24
185.198.240.238/32
185.213.175.168/32
185.223.152.0/24
185.238.231.167/32
185.251.19.30/32
185.93.89.147/32
185.93.89.167/32
188.166.227.85/32
188.166.245.173/32
188.213.202.0/24
191.232.0.0/13
@@ -245,15 +204,12 @@
193.148.57.33/32
193.149.176.55/32
193.19.109.0/24
193.36.224.57/32
193.36.225.0/24
193.37.32.0/24
193.37.33.0/24
193.47.62.168/32
194.102.104.29/32
194.156.79.68/32
194.233.91.21/32
194.26.192.156/32
194.32.120.151/32
194.5.82.0/24
195.178.110.0/24
@@ -261,7 +217,6 @@
195.178.110.223/32
198.98.52.229/32
199.16.157.0/24
2.57.122.13/32
2.57.122.173/32
2.58.56.101/32
20.0.0.0/11
@@ -269,7 +224,6 @@
20.160.0.0/12
20.184.150.240/32
20.192.0.0/10
20.215.189.213/32
20.46.121.148/32
20.46.252.52/32
20.48.0.0/12
@@ -286,30 +240,25 @@
207.241.173.229/32
208.84.102.195/32
209.126.2.173/32
209.87.169.0/24
209.97.166.251/32
209.87.169.142/32
209.99.184.0/21
211.175.22.56/32
213.111.158.220/32
213.35.113.47/32
216.24.210.46/32
216.24.212.0/24
216.26.233.39/32
216.73.163.0/24
216.73.216.0/22
216.73.217.176/32
216.81.248.20/32
217.217.252.197/32
220.202.112.100/32
221.13.93.71/32
222.176.200.21/32
222.176.200.251/32
223.199.187.171/32
23.102.123.231/32
23.161.169.127/32
23.161.169.62/32
23.178.112.0/24
23.180.120.0/24
23.180.120.134/32
23.234.98.171/32
23.94.155.57/32
23.95.226.104/32
@@ -324,30 +273,18 @@
31.184.236.8/32
31.56.58.0/24
31.58.51.130/32
31.58.51.184/32
31.59.129.193/32
34.105.30.251/32
34.11.108.163/32
34.11.127.22/32
34.116.255.65/32
34.118.184.2/32
34.125.213.146/32
34.125.219.56/32
34.125.42.15/32
34.131.251.163/32
34.139.141.56/32
34.141.160.140/32
34.145.62.28/32
34.146.12.12/32
34.148.8.63/32
34.148.98.62/32
34.154.122.183/32
34.158.83.192/32
34.17.159.73/32
34.17.191.102/32
34.171.75.208/32
34.175.82.8/32
34.176.154.83/32
34.18.38.137/32
34.181.134.189/32
34.186.83.134/32
@@ -355,22 +292,15 @@
34.187.200.119/32
34.187.82.183/32
34.21.144.169/32
34.23.104.90/32
34.24.155.203/32
34.242.226.103/32
34.244.177.56/32
34.26.191.118/32
34.26.219.248/32
34.31.88.250/32
34.39.210.94/32
34.40.240.124/32
34.42.48.100/32
34.47.183.174/32
34.47.194.183/32
34.55.200.45/32
34.57.157.110/32
34.59.187.134/32
34.62.38.199/32
34.64.173.205/32
34.65.138.195/32
34.65.14.246/32
@@ -379,29 +309,21 @@
34.7.160.235/32
34.7.193.250/32
34.73.128.17/32
34.74.203.130/32
34.75.4.161/32
34.82.10.34/32
34.83.69.198/32
34.84.59.119/32
34.86.126.198/32
34.86.217.208/32
34.90.87.32/32
34.90.88.37/32
34.91.134.67/32
34.91.243.226/32
34.94.172.72/32
34.95.238.121/32
34.95.239.96/32
35.157.251.190/32
35.182.17.8/32
35.184.157.121/32
35.185.21.88/32
35.187.221.219/32
35.187.9.30/32
35.192.232.168/32
35.196.132.85/32
35.196.22.146/32
35.198.180.224/32
35.199.111.12/32
35.199.115.51/32
35.199.183.190/32
@@ -409,15 +331,13 @@
35.204.141.252/32
35.204.88.252/32
35.230.52.27/32
35.231.0.0/17
35.233.191.179/32
35.234.181.205/32
35.231.100.82/32
35.231.41.225/32
35.237.138.104/32
35.237.178.50/32
35.243.173.211/32
35.243.223.147/32
35.255.40.65/32
35.95.56.230/32
36.106.166.223/32
36.106.166.92/32
37.140.223.58/32
@@ -439,11 +359,10 @@
45.131.195.0/24
45.134.20.116/32
45.134.20.128/32
45.138.16.114/32
45.139.104.172/32
45.146.54.137/32
45.146.54.80/32
45.146.55.0/24
45.146.55.48/32
45.148.10.0/24
45.153.34.16/32
45.153.34.182/32
@@ -452,15 +371,12 @@
45.154.138.232/32
45.154.98.0/24
45.45.237.214/32
45.45.237.69/32
45.61.150.67/32
45.61.187.30/32
45.67.85.190/32
45.8.196.200/32
45.88.138.44/32
46.23.108.234/32
46.232.235.0/24
47.85.160.47/32
5.161.225.116/32
5.161.62.209/32
5.253.247.16/32
@@ -468,30 +384,25 @@
5.34.213.142/32
51.103.133.211/32
51.103.152.96/32
51.107.190.246/32
51.116.233.22/32
51.120.0.0/16
51.120.79.193/32
51.79.208.246/32
52.139.36.144/32
52.159.244.176/32
52.160.0.0/11
52.224.0.0/11
52.64.227.143/32
54.163.186.39/32
54.169.103.174/32
54.179.129.22/32
54.206.147.115/32
54.210.219.163/32
54.216.227.171/32
54.251.19.144/32
54.82.69.43/32
58.212.237.231/32
62.60.130.128/32
62.60.130.227/32
62.93.179.146/32
64.225.63.129/32
64.225.75.246/32
64.226.123.37/32
64.227.32.66/32
64.23.218.208/32
64.89.161.82/32
@@ -499,40 +410,30 @@
65.0.122.218/32
66.187.5.228/32
66.187.6.102/32
66.228.38.221/32
66.249.64.0/20
66.249.74.164/32
66.249.74.165/32
68.183.180.73/32
68.183.9.16/32
68.220.0.0/15
69.164.220.232/32
69.164.220.85/32
69.30.85.194/32
72.11.155.223/32
72.146.20.230/32
72.146.33.199/32
73.116.194.16/32
74.208.40.151/32
74.248.0.0/15
74.248.33.172/32
74.7.0.0/16
77.90.185.230/32
8.229.1.130/32
8.229.130.50/32
8.231.78.19/32
8.234.157.73/32
8.234.207.247/32
8.235.114.162/32
80.91.86.169/32
80.94.92.167/32
81.171.72.135/32
81.171.72.93/32
81.171.74.60/32
82.102.18.0/24
82.102.18.190/32
82.165.82.154/32
82.39.206.154/32
84.17.60.251/32
84.235.249.236/32
85.11.167.225/32
85.204.70.0/24
87.121.79.42/32
@@ -542,8 +443,6 @@
91.124.88.36/32
91.134.35.95/32
91.148.244.131/32
91.148.245.81/32
91.192.10.243/32
91.192.10.86/32
91.193.232.0/24
91.224.92.159/32
@@ -554,11 +453,11 @@
91.92.243.245/32
91.92.41.115/32
91.92.42.86/32
91.92.47.0/24
91.92.47.101/32
91.92.47.153/32
92.118.39.126/32
92.118.39.29/32
92.119.121.196/32
92.165.170.72/32
93.123.109.0/24
93.152.221.13/32
94.154.43.0/24
-4
View File
@@ -1,7 +1,5 @@
2001:4ca0:108:42::24/128
2001:ba0:24c:a500::1/128
2400:8901::2000:62ff:fe9a:e1c2/128
2400:8901::2000:beff:fea3:5a44/128
2406:da18:525:0:a23e:e8f6:3ea1:4c2c/128
2600:3000::/32
2600:3c09::2000:5cff:fe99:c802/128
@@ -13,11 +11,9 @@
2a00:6340:2008:ec80:be24:11ff:fe82:2775/128
2a01:2a8:a0dd:1::1012/128
2a01:4f8:c014:7f9e::1/128
2a04:4e40:4400:0:537:cdd2:7250:95db/128
2a04:4e40:e000::/48
2a04:c300::/29
2a05:4140:3::37cd/128
2a06:a880:5:7fb3::1/128
2a09:bac1:76c0:460::5e:6a/128
2a09:bac5::/32
2a0a:4cc0:2000:8528:c4e8:3ff:febb:3ee9/128
+2
View File
@@ -0,0 +1,2 @@
## Start the application in a virtual environment
& .\.venv\Scripts\python.exe .\app.py