feat: add incremental log scanning with resumable byte offsets

Track last processed byte offset per log file so scans only read new
entries, add --full flag to force a rescan, and persist state atomically.
This commit is contained in:
zwitschi committed 2026-08-05 21:41:00 +02:00
1 parent f5abaaf7e5
commit 51f57f7a04
14 files changed
+133 -9

No files matched your search

+20
View File
@@ -60,6 +60,26 @@ def read_lines(path: str) -> Iterator[str]:
yield ANSI_RE.sub("", line.rstrip("\r\n"))
def read_lines_from(path: str, start: int = 0) -> Iterator[tuple[str, int]]:
"""Yield ``(line, resume_offset)`` for a plain-text log from byte ``start``.
Only seekable (non-gzip) files can be resumed. ``resume_offset`` points
after the last complete line; a trailing partial line (no newline yet) is
yielded but resets the resume offset to its start so it is re-read once
completed. The last yielded ``resume_offset`` is the value to store.
"""
with open(path, "rb") as f:
f.seek(start)
pos = start
for raw in f:
line_start = pos
pos += len(raw)
complete = raw.endswith(b"\n") or raw.endswith(b"\r")
text = ANSI_RE.sub(
"", raw.decode("utf-8", errors="replace").rstrip("\r\n"))
yield text, (pos if complete else line_start)
# -- writers ----------------------------------------------------------------
def write_csv(path: str, header: Sequence[str],