feat: add incremental log scanning with resumable byte offsets

Track last processed byte offset per log file so scans only read new
entries, add --full flag to force a rescan, and persist state atomically.

app/analyzer.py

@@ -13,13 +13,14 @@ from collections import Counter
 from datetime import datetime
 from typing import Any, Sequence
 
-from .files import ensure_dirs, find_logs, read_lines, validate_dir, write_csv, write_json, write_lines
+from .files import ensure_dirs, find_logs, read_lines, read_lines_from, validate_dir, write_csv, write_json, write_lines
 from .flags import by_total, flag_malicious_patterns
 from .ip_utils import is_whitelisted, split_versions
 from .mikrotik import write_mikrotik
 from .models import IPStats, Request, aggregate
 from .reporting import Column, columns_header, columns_row, render_kv, render_table, timestamp_name
 from .constants import DEFAULT_TOP, DEFAULT_TOP_PATHS
+from . import scan_state
 from .whitelists import refresh_whitelists
 
 
@@ -44,13 +45,15 @@ class Analyzer:
     flag_note = ""
 
     def __init__(self, log_path: str, report_dir: str, top: int = DEFAULT_TOP,
-                 quiet: bool = False, write_rsc: bool = True):
+                 quiet: bool = False, write_rsc: bool = True,
+                 full: bool = False):
         self.log_path = log_path
         self.report_dir = report_dir
         self.data_dir = os.path.join(report_dir, "data")
         self.top = top
         self.quiet = quiet
         self.write_rsc = write_rsc
+        self.full = full
         self.files: list[str] = []
         self.requests: list[Request] = []
         self.stats: dict[str, IPStats] = {}
@@ -89,14 +92,46 @@ class Analyzer:
         return self.files
 
     def load(self) -> list[Request]:
+        """Read log entries, resuming from the last processed offset.
+
+        Non-gzip logs are read incrementally (only new bytes since the last
+        scan) unless ``full`` is set; gzip archives are always fully read
+        because they cannot be seeked.
+        """
+        state: dict[str, int] = {}
+        if not self.full:
+            state = scan_state.load_state()
+            # Prune entries whose log file no longer exists (safe across
+            # analyzers: only removes dead keys, keeps live ones).
+            state = {k: v for k, v in state.items() if os.path.exists(k)}
         for name in self.files:
+            full_path = os.path.join(self.log_path, name)
             count = 0
-            for line in read_lines(os.path.join(self.log_path, name)):
-                req = self.parse_line(line)
-                if req is not None:
-                    self.requests.append(req)
-                    count += 1
+            if self.full or full_path.endswith(".gz"):
+                for line in read_lines(full_path):
+                    req = self.parse_line(line)
+                    if req is not None:
+                        self.requests.append(req)
+                        count += 1
+            else:
+                try:
+                    size = os.path.getsize(full_path)
+                except OSError:
+                    size = 0
+                recorded = state.get(full_path, 0)
+                # Reset to the start if the file shrank or was truncated.
+                start = recorded if 0 < recorded <= size else 0
+                last_resume = start
+                for line, resume in read_lines_from(full_path, start):
+                    req = self.parse_line(line)
+                    if req is not None:
+                        self.requests.append(req)
+                        count += 1
+                    last_resume = resume
+                state[full_path] = last_resume
             self.per_file[name] = count
+        if not self.full:
+            scan_state.save_state(state)
         return self.requests
 
     def analyze(self) -> dict[str, IPStats]: