115 lines
4.5 KiB
Python
115 lines
4.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Tests for legacy report artifact backfill."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import tempfile
|
|
import unittest
|
|
from pathlib import Path
|
|
|
|
from app.db import connect
|
|
from app.db.backfill import backfill_report_artifacts
|
|
from app.db import repositories as repo
|
|
|
|
|
|
class ReportBackfillTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.temp_dir = tempfile.TemporaryDirectory()
|
|
self.root = Path(self.temp_dir.name)
|
|
self.conn = connect(str(self.root / "test.db"))
|
|
self.roots = {
|
|
"copyparty": str(self.root / "copyparty"),
|
|
"npm": str(self.root / "npm"),
|
|
"app": str(self.root / "app"),
|
|
"mikrotik": str(self.root / "mikrotik"),
|
|
}
|
|
for path in self.roots.values():
|
|
Path(path).mkdir(parents=True)
|
|
|
|
def tearDown(self):
|
|
self.conn.close()
|
|
self.temp_dir.cleanup()
|
|
|
|
def test_dry_run_and_import_are_idempotent(self):
|
|
Path(self.roots["copyparty"], "2026-11-06-11-30-10.txt").write_text(
|
|
"copyparty summary\n", encoding="utf-8")
|
|
Path(self.roots["npm"], "2026-11-06.csv").write_text(
|
|
"ip,count\n192.0.2.1,3\n", encoding="utf-8")
|
|
Path(self.roots["app"], "2026-11-06.json").write_text(
|
|
'{"summary":{"blocked":1}}', encoding="utf-8")
|
|
Path(self.roots["app"], "data").mkdir()
|
|
Path(self.roots["app"], "data", "requests.csv").write_text(
|
|
"ip\n192.0.2.2\n", encoding="utf-8")
|
|
|
|
preview = backfill_report_artifacts(
|
|
self.conn, dry_run=True, roots=self.roots)["reports"]
|
|
self.assertEqual(preview["would_import"], 3)
|
|
self.assertEqual(preview["skipped_canonical"], [
|
|
{"source": "app", "path": "data/requests.csv"}])
|
|
self.assertEqual(preview["by_source"]["copyparty"]["would_import"], 1)
|
|
self.assertEqual(preview["by_format"]["csv"]["would_import"], 1)
|
|
self.assertEqual(repo.list_report_artifacts(self.conn)["items"], [])
|
|
|
|
imported = backfill_report_artifacts(
|
|
self.conn, roots=self.roots)["reports"]
|
|
self.assertEqual(imported["imported"], 3)
|
|
self.assertEqual(imported["unmapped"], [])
|
|
|
|
repeated = backfill_report_artifacts(
|
|
self.conn, roots=self.roots)["reports"]
|
|
self.assertEqual(repeated["imported"], 0)
|
|
self.assertEqual(repeated["already_imported"], 3)
|
|
|
|
items = repo.list_report_artifacts(
|
|
self.conn, source="copyparty")["items"]
|
|
self.assertEqual(items[0]["original_path"],
|
|
"copyparty/2026-11-06-11-30-10.txt")
|
|
self.assertEqual(items[0]["generated_at"],
|
|
"2026-11-06T11:30:10+00:00")
|
|
|
|
def test_malformed_json_is_preserved_with_parse_error(self):
|
|
Path(self.roots["app"], "2026-11-06.json").write_text(
|
|
'{"incomplete":', encoding="utf-8")
|
|
|
|
result = backfill_report_artifacts(
|
|
self.conn, roots=self.roots)["reports"]
|
|
self.assertEqual(result["imported"], 1)
|
|
artifact = repo.list_report_artifacts(
|
|
self.conn, source="app")["items"][0]
|
|
detail = repo.get_report_artifact(self.conn, artifact["id"])
|
|
self.assertEqual(detail["payload"]["content"], '{"incomplete":')
|
|
self.assertIn("parse_error", detail["payload"])
|
|
|
|
def test_artifacts_survive_sqlite_backup_and_restore(self):
|
|
artifact_id = repo.insert_report_artifact(
|
|
self.conn, source="app", report_type="summary",
|
|
generated_at="2026-11-06T00:00:00+00:00",
|
|
payload={"summary": {"count": 4}},
|
|
original_path="app/2026-11-06.json", content_sha256="c" * 64,
|
|
)
|
|
restored = connect(str(self.root / "restored.db"))
|
|
try:
|
|
self.conn.backup(restored)
|
|
artifact = repo.get_report_artifact(restored, artifact_id)
|
|
self.assertEqual(artifact["payload"], {"summary": {"count": 4}})
|
|
finally:
|
|
restored.close()
|
|
|
|
def test_oversized_file_is_reported_without_deleting_source(self):
|
|
source = Path(self.roots["npm"], "large.txt")
|
|
source.write_text("keep", encoding="utf-8")
|
|
from unittest.mock import patch
|
|
|
|
with patch("app.db.backfill.MAX_REPORT_ARTIFACT_BYTES", 1):
|
|
result = backfill_report_artifacts(
|
|
self.conn, roots=self.roots)["reports"]
|
|
|
|
self.assertEqual(len(result["unmapped"]), 1)
|
|
self.assertTrue(source.exists())
|
|
self.assertEqual(result["unmapped"][0]["path"], "large.txt")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|