feat(reports): store and browse report artifacts

This commit is contained in:
zwitschi committed 2026-11-06 12:27:00 +01:00
1 parent 5967a43eb5
commit 6f2e479958
13 files changed
+875 -93

No files matched your search

+104
View File
@@ -10,6 +10,7 @@ from __future__ import annotations
import json
import sqlite3
import base64
from contextlib import contextmanager
from datetime import datetime, timedelta, timezone
import ipaddress
@@ -233,6 +234,109 @@ def count_scans(conn: sqlite3.Connection) -> int:
return int(conn.execute("SELECT COUNT(*) FROM scans").fetchone()[0])
# ---------------------------------------------------------------------------
# Report artifacts
# ---------------------------------------------------------------------------
def insert_report_artifact(conn: sqlite3.Connection, *, source: str,
report_type: str, generated_at: str,
payload: Any, original_path: str,
content_sha256: str, scan_id: int | None = None,
schema_version: int = 1) -> int:
"""Insert a structured report artifact, idempotent by path and hash."""
with transaction(conn):
conn.execute(
"INSERT OR IGNORE INTO report_artifacts "
"(source, scan_id, report_type, generated_at, schema_version, "
"payload_json, original_path, content_sha256, created_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(source, scan_id, report_type, generated_at, schema_version,
json.dumps(payload, ensure_ascii=False), original_path,
content_sha256, utcnow()),
)
row = conn.execute(
"SELECT id FROM report_artifacts "
"WHERE original_path=? AND content_sha256=?",
(original_path, content_sha256),
).fetchone()
return int(row["id"])
def list_report_artifacts(conn: sqlite3.Connection, *,
source: str | None = None,
report_type: str | None = None,
since: str | None = None,
until: str | None = None,
limit: int = 50,
cursor: str | None = None) -> dict[str, Any]:
"""List report metadata using bounded keyset pagination."""
limit = min(max(int(limit), 1), 200)
clauses: list[str] = []
args: list[Any] = []
if source:
clauses.append("source=?")
args.append(source)
if report_type:
clauses.append("report_type=?")
args.append(report_type)
if since:
clauses.append("generated_at>=?")
args.append(since)
if until:
clauses.append("generated_at<=?")
args.append(until)
if cursor:
generated_at, artifact_id = _decode_report_cursor(cursor)
clauses.append("(generated_at<? OR (generated_at=? AND id<?))")
args.extend((generated_at, generated_at, artifact_id))
sql = (
"SELECT id, source, scan_id, report_type, generated_at, "
"schema_version, original_path, content_sha256 "
"FROM report_artifacts"
)
if clauses:
sql += " WHERE " + " AND ".join(clauses)
sql += " ORDER BY generated_at DESC, id DESC LIMIT ?"
rows = [dict(row) for row in conn.execute(sql, (*args, limit + 1))]
has_more = len(rows) > limit
rows = rows[:limit]
next_cursor = (
_encode_report_cursor(rows[-1]["generated_at"], rows[-1]["id"])
if has_more and rows else None
)
return {"items": rows, "next_cursor": next_cursor}
def get_report_artifact(conn: sqlite3.Connection,
artifact_id: int) -> dict[str, Any] | None:
"""Fetch one artifact and decode its structured payload."""
row = conn.execute(
"SELECT id, source, scan_id, report_type, generated_at, "
"schema_version, payload_json, original_path, content_sha256, created_at "
"FROM report_artifacts WHERE id=?", (artifact_id,)).fetchone()
if row is None:
return None
artifact = dict(row)
artifact["payload"] = json.loads(artifact.pop("payload_json"))
return artifact
def _encode_report_cursor(generated_at: str, artifact_id: int) -> str:
raw = f"{generated_at}\n{artifact_id}".encode("utf-8")
return base64.urlsafe_b64encode(raw).decode("ascii").rstrip("=")
def _decode_report_cursor(cursor: str) -> tuple[str, int]:
try:
padded = cursor + "=" * (-len(cursor) % 4)
generated_at, artifact_id = base64.urlsafe_b64decode(
padded.encode("ascii")).decode("utf-8").split("\n", 1)
return generated_at, int(artifact_id)
except (ValueError, UnicodeError) as exc:
raise ValueError("invalid report cursor") from exc
def correlate_across_sources(conn: sqlite3.Connection,
limit: int = 100) -> list[dict[str, Any]]:
"""IPs seen in more than one log source, with per-source counts."""