feat: add incremental log scanning with resumable byte offsets
Track last processed byte offset per log file so scans only read new entries, add --full flag to force a rescan, and persist state atomically.
This commit is contained in:
1 parent
f5abaaf7e5
commit
51f57f7a04
14 files changed
+133
-9
No files matched your search
@@ -60,6 +60,26 @@ def read_lines(path: str) -> Iterator[str]:
|
||||
yield ANSI_RE.sub("", line.rstrip("\r\n"))
|
||||
|
||||
|
||||
def read_lines_from(path: str, start: int = 0) -> Iterator[tuple[str, int]]:
|
||||
"""Yield ``(line, resume_offset)`` for a plain-text log from byte ``start``.
|
||||
|
||||
Only seekable (non-gzip) files can be resumed. ``resume_offset`` points
|
||||
after the last complete line; a trailing partial line (no newline yet) is
|
||||
yielded but resets the resume offset to its start so it is re-read once
|
||||
completed. The last yielded ``resume_offset`` is the value to store.
|
||||
"""
|
||||
with open(path, "rb") as f:
|
||||
f.seek(start)
|
||||
pos = start
|
||||
for raw in f:
|
||||
line_start = pos
|
||||
pos += len(raw)
|
||||
complete = raw.endswith(b"\n") or raw.endswith(b"\r")
|
||||
text = ANSI_RE.sub(
|
||||
"", raw.decode("utf-8", errors="replace").rstrip("\r\n"))
|
||||
yield text, (pos if complete else line_start)
|
||||
|
||||
|
||||
# -- writers ----------------------------------------------------------------
|
||||
|
||||
def write_csv(path: str, header: Sequence[str],
|
||||
|
||||
Reference in new issue
Block a user