feat: add incremental log scanning with resumable byte offsets
Track last processed byte offset per log file so scans only read new entries, add --full flag to force a rescan, and persist state atomically.
This commit is contained in:
1 parent
f5abaaf7e5
commit
51f57f7a04
14 files changed
+133
-9
No files matched your search
@@ -0,0 +1,33 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Persistent state for incremental log scanning.
|
||||
|
||||
Tracks the last processed byte offset per absolute log file path so a scan
|
||||
only reads new entries instead of re-reading the whole file. The state file
|
||||
is written atomically (temp file + rename) to avoid corruption.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from .constants import SCAN_STATE_PATH
|
||||
|
||||
|
||||
def load_state() -> dict[str, int]:
|
||||
"""Return ``{absolute log path: byte offset}`` from the scan-state file."""
|
||||
try:
|
||||
with open(SCAN_STATE_PATH, "r", encoding="utf-8") as f:
|
||||
data = json.load(f)
|
||||
return {k: int(v) for k, v in data.items()}
|
||||
except (OSError, ValueError, TypeError):
|
||||
return {}
|
||||
|
||||
|
||||
def save_state(state: dict[str, int]) -> None:
|
||||
"""Write the scan-state file atomically (temp file + rename)."""
|
||||
os.makedirs(os.path.dirname(SCAN_STATE_PATH), exist_ok=True)
|
||||
tmp = SCAN_STATE_PATH + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
json.dump(state, f, indent=2, sort_keys=True)
|
||||
os.replace(tmp, SCAN_STATE_PATH)
|
||||
Reference in new issue
Block a user