diff --git a/.env.example b/.env.example index dcd09f9..d10c7d2 100644 --- a/.env.example +++ b/.env.example @@ -8,6 +8,7 @@ R2_SECRET_ACCESS_KEY=your-r2-secret-key R2_BUCKET_NAME=linearvoid-media R2_ENDPOINT=https://your-account-id.r2.cloudflarestorage.com R2_REGION=auto +R2_SEGMENT_RETENTION_HOURS=24 # Cloudflare (for custom domain, cache rules, Workers if used) CLOUDFLARE_API_TOKEN=your-cloudflare-api-token diff --git a/.github/instructions/BACKLOG.md b/.github/instructions/BACKLOG.md index 66cbe15..31e65e2 100644 --- a/.github/instructions/BACKLOG.md +++ b/.github/instructions/BACKLOG.md @@ -1,34 +1,14 @@ # BACKLOG -## Remaining from 2026-08-11 (summary) - -- **Test Fixtures**: media + playlist fixtures created; ffmpeg generation commands documented in test headers -- **User Frontend Tests**: video player, chat integration, responsive, accessibility, coverage -- **Admin Frontend Tests**: playlist UI, moderation UI, metrics dashboards, coverage -- **Cloudflare R2 Integration Test**: upload + cache header validation done (SyncToR2 with CacheControl + ContentType); edge cache hit/miss validation remains manual -- **Cloudflare Config**: Cloudflare Pages deploy workflow and setup docs created; manual dashboard setup (Tiered Cache, WAF, rate limiting, HTTPS/HSTS) remains pending; Pages project must be created in dashboard first. -- **External Services**: UptimeRobot monitors + alerts -- **Production Deployment**: Hetzner provisioning, CD pipeline, DNS cutover, smoke test -- **Scaling & Performance**: TTFB baseline and live benchmark runs still pending; loadtest scaffolding now includes Artillery/k6 helpers with auto summaries plus worksheet field mapping; chat p95, Redis saturation, and cache-hit threshold alerting are implemented +> All remaining work is planned in [TODO](./TODO.md) (2026-08-13). This file records completed backlog items for reference; new unplanned work is added here. ## Cloudflare Configuration ### DNS & Pages -- [x] Configure DNS for linearvoid.tv apex and www redirect (manual) -- [x] Configure media.linearvoid.tv for split-domain HLS delivery (manual) -- [x] Configure Cloudflare Pages deploy workflow (`.gitea/workflows/pages-deploy.yml`) and setup docs (`docs/operations/cloudflare-pages-setup.md`, `docs/operations/gitea-cloudflare-pages.md`) -- [x] Create Cloudflare Pages project in dashboard (unavailable, created `linearvoid` via API) - [ ] Cloudflare Pages: link custom domain (manual) - [ ] Create Cloudflare Pages project and cut over frontend hosting (manual) -### Storage & Cache - -- [x] Create R2 bucket (manual) -- [x] Configure Cache Rules: .ts/.m4s + .m3u8 (manual) -- [x] Implement Worker for immutable cache headers on segment paths -- [ ] CORS headers on HLS responses from media.linearvoid.tv (MEDIUM) — needs Cloudflare R2 CORS config - ### Security & TLS - [ ] Enable Tiered Cache, WAF, rate limiting (manual) @@ -40,13 +20,6 @@ ## Monitoring & Observability -### Prometheus & Grafana - -- [x] Set up Prometheus with chat/playout/nginx/redis/postgres/node scrape targets -- [x] Add nginx-exporter, redis-exporter, postgres-exporter, node-exporter -- [x] Import Grafana app + operational dashboards -- [x] Define Prometheus alerting rules (8 rules: ChatDown, PlayoutDown, RedisDown, etc.) - ### External Monitoring - [ ] Set up UptimeRobot external monitoring @@ -54,44 +27,16 @@ - [ ] UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword: `#EXTM3U`, 5min) - [ ] Alert on 2 consecutive failures, notify via email + Discord/Slack webhook -## Security - -- [x] Implement admin cookie-session auth (HttpOnly, SameSite=Strict, Secure when TLS) -- [x] Implement server-side session expiration -- [x] Implement rate limiting for chat messages (Redis windowing) -- [x] Implement WebSocket connection rate limiting (5/IP/60s) -- [x] Implement admin login rate limiting (5/IP/5min) -- [ ] Enforce HTTPS/WSS at edge and origin - -## Test Fixtures - -- [x] Create test media fixtures in `backend/playout/testdata/`: valid H.264/AAC, invalid codec, invalid framerate -- [x] Create test playlist fixtures: `valid_playlist.txt`, `empty_playlist.txt`, `comments_only.txt`, `mixed_valid_invalid.txt` -- [x] Document ffmpeg fixture-generation commands in test file headers - ## Frontend Tests ### User Frontend -- [x] Video player behavior (play, pause, seek, buffering, error handling) — HLS player startup, native/fallback, URL resolution, error recovery with retries/backoff/reconnecting overlay -- [x] Chat functionality (send/receive messages, presence, rate limiting, moderation) — send, receive, presence join/leave, DM flow, reconnect covered -- [x] State management and data flow -- [x] Responsive design and layout across different screen sizes — chat-collapsed toggle, CSS grid, now-playing aria -- [x] Accessibility testing (keyboard navigation, screen reader support) — video controls, aria-labels, aria-expanded, input labels, reconnecting overlay aria-live/role -- [x] Integration with backend APIs (mocking API responses for testing) - [ ] Test coverage for all critical frontend components and interactions - [ ] Add code coverage reporting and enforce minimum coverage threshold in CI/CD pipeline - [ ] Document test cases and expected outcomes ### Admin Frontend -- [x] Playlist management UI (add/remove videos, reorder, normalize) — save/error handling, sortable rendering -- [ ] User moderation tools (mute/ban/report users, view moderation logs) -- [x] Metrics and monitoring dashboards (Prometheus/Grafana integration) — overview + analytics rendering -- [x] State management and data flow -- [x] Responsive design and layout across different screen sizes — tab content visibility, essential tabs -- [x] Accessibility testing (keyboard navigation, screen reader support) — nav aria-label, active tab aria-current, nav button focusability -- [x] Integration with backend APIs (mocking API responses for testing) - [ ] Test coverage for all critical admin UI components and interactions - [ ] Add code coverage reporting and enforce minimum coverage threshold in CI/CD pipeline - [ ] Document test cases and expected outcomes @@ -100,29 +45,9 @@ ### Cloudflare R2 -- [x] Upload test HLS segments and manifests to R2 bucket (SyncToR2 with mock, verified bucket/key/body) -- [x] Verify correct cache headers are applied (immutable segments via segmentCacheControl, short-lived manifests via manifestCacheControl) - [ ] Test retrieval of segments and manifests from R2 via Cloudflare edge - [ ] Validate cache hit/miss behavior and performance metrics - [ ] Validate edge cache hit/miss behavior with repeat requests — script `scripts/validate-edge-cache.sh` created -- [x] Ensure integration test can run in CI/CD pipeline with proper mocking/stubbing of R2 API (putObject mock, no network calls) -- [x] Document test cases and expected outcomes -- [x] Document required stubs or mocks (putObject variable injection pattern) - -## Streaming Pipeline - -- [x] **Continuous R2 sync during playout (HIGH)**: fsnotify watcher uploads segments in real-time (~7s delay) -- [x] **Globally unique segment naming (HIGH)**: `r-{ISOTIME}/seg-{seq}.m4s` via run ID + ffmpeg monotonic sequence -- [x] **Atomic manifest publication (MEDIUM)**: Upload to `playlist.m3u8.tmp`, then `CopyObject` to final key -- [x] **Manifest Cache-Control alignment (LOW)**: `public, max-age=1, s-maxage=2, must-revalidate` -- [ ] **CORS headers on HLS responses (MEDIUM)**: Needs Cloudflare R2 bucket CORS config (infrastructure, not code) -- [x] **Player error recovery (MEDIUM)**: `hls.on(Hls.Events.ERROR)` handler with bounded retries (5x), exponential backoff, reconnecting overlay -- [x] **Health endpoint detail (LOW)**: Added active item, run ID, playlist revision, last segment seq, last R2 pub, consecutive failures -- [x] Source media contract + validation (H.264/AAC, 30fps, keyframe check) -- [x] fMP4 HLS packaging (6s segments, 24 window) -- [x] Slate fallback + recovery (`shouldEscalateToSlate`) -- [x] R2 cache headers (immutable segments, short-lived manifest) -- [x] Cloudflare Worker immutable `.m4s` headers ## External Services @@ -162,9 +87,6 @@ ### Load Testing -- [x] Load-test scaffolding: `ws-chat.yml`, `admin-api.js`, `run-loadtests.ps1`, summarizers, README -- [x] Cache-hit ratio exporter (`scripts/cache-hit-ratio.js` + `.ps1`) -- [x] Saturation alerts: chat p95 latency, Redis memory usage, cache-hit ratio - [ ] Execute WebSocket load tests with real infrastructure — ramp 10→100→500→1000 connections, measure p95 latency + errors - [ ] Execute admin API benchmarks with real infrastructure — `wrk`/`k6` against `/api/admin/*`, target p95 <1s diff --git a/.github/instructions/DONE.md b/.github/instructions/DONE.md index 071de5b..be04bb7 100644 --- a/.github/instructions/DONE.md +++ b/.github/instructions/DONE.md @@ -4,74 +4,74 @@ Completed tasks from `.github/instructions/TODO.md`. Historical record for CHANG ## Session 2026-08-07 — Unit Tests -- [x] Chat service refactor: `main.go` split into 8 packages (config, auth, db, handlers, ws, media, ratelimit, metrics) -- [x] `db.RedisClient` + `db.PubSub` interfaces defined for mocking -- [x] Config tests: 10 env-var tests, table-driven, 100% coverage -- [x] Auth tests: token generation, session CRUD, cookie security, middleware, 97.8% coverage -- [x] DB tests: SQLite in-memory CRUD for all 4 models, 14.3% coverage -- [x] Handlers tests: 35 tests covering login/logout/session/overview/playlist/moderation/analytics/system/config/users/schedule/media/chat-log, 47.2% coverage -- [x] WebSocket tests: 15 tests covering connect/send/dm/moderation/banned/muted/history/presence/pubsub, 79.6% coverage -- [x] Media tests: formatDuration, extensions, mime, upload IDs, playlist parsing, 36.8% coverage -- [x] Ratelimit tests: message/login/WS rate limits with mock Redis, 90.3% coverage -- [x] Metrics tests: Prometheus output validation, all 9 metrics present, 100% coverage -- [x] Playout tests: 12 tests (LoadPlaylist, keyframeInterval, HealthHandler, slate escalation, reset/activate/deactivate) -- [x] Frontend test framework: vitest + jsdom installed -- [x] User frontend smoke tests: 6 tests (chat toggle, sendMessage, switchRoom, DM toggle) -- [x] Admin frontend smoke tests: 5 tests (DOM structure, upload progress, fingerprint, constants, import stability) -- [x] CI pipeline updated: `go test -race -cover` with 50% threshold -- [x] SQLite driver switched to pure-Go `github.com/glebarez/sqlite` (CGO-free) -- [x] Total: 78 tests across Go + frontend, all passing +- Chat service refactor: `main.go` split into 8 packages (config, auth, db, handlers, ws, media, ratelimit, metrics) +- `db.RedisClient` + `db.PubSub` interfaces defined for mocking +- Config tests: 10 env-var tests, table-driven, 100% coverage +- Auth tests: token generation, session CRUD, cookie security, middleware, 97.8% coverage +- DB tests: SQLite in-memory CRUD for all 4 models, 14.3% coverage +- Handlers tests: 35 tests covering login/logout/session/overview/playlist/moderation/analytics/system/config/users/schedule/media/chat-log, 47.2% coverage +- WebSocket tests: 15 tests covering connect/send/dm/moderation/banned/muted/history/presence/pubsub, 79.6% coverage +- Media tests: formatDuration, extensions, mime, upload IDs, playlist parsing, 36.8% coverage +- Ratelimit tests: message/login/WS rate limits with mock Redis, 90.3% coverage +- Metrics tests: Prometheus output validation, all 9 metrics present, 100% coverage +- Playout tests: 12 tests (LoadPlaylist, keyframeInterval, HealthHandler, slate escalation, reset/activate/deactivate) +- Frontend test framework: vitest + jsdom installed +- User frontend smoke tests: 6 tests (chat toggle, sendMessage, switchRoom, DM toggle) +- Admin frontend smoke tests: 5 tests (DOM structure, upload progress, fingerprint, constants, import stability) +- CI pipeline updated: `go test -race -cover` with 50% threshold +- SQLite driver switched to pure-Go `github.com/glebarez/sqlite` (CGO-free) +- Total: 78 tests across Go + frontend, all passing ## Session 2026-08-07 — Runtime Validation -- [x] Check `monitoring/prometheus/prometheus.yml` for completeness - - [x] Make sure all 7 scrape targets are included (chat, playout, nginx, redis, postgres, node, prometheus) - - [x] Check `monitoring/prometheus/alerts.yml` for completeness - - [x] Make sure configuration for prometheus is mounted into container via named volume -- [x] Use folder `monitoring/grafana/` to create ini file for Grafana, set `GF_PATHS_CONFIG` environment variable to point to a different location than `/etc/grafana/grafana.ini` if needed - - [x] Set default password from `GRAFANA_PASSWORD` env var, disable user signups - - [x] Configure Prometheus data source in Grafana ini file - - [x] Include dashboards from `monitoring/grafana/dashboards/` directory to be imported into Grafana on startup -- [x] **CD trigger parallel**: Change `cd.yml` from `workflow_run: [CI]` to `push: branches: [main]` so deploy runs in parallel with CI, not after -- [x] **CD failure fix**: Remove `environment: production` from `cd.yml` deploy job (no Gitea environment configured, causes silent job abort after Buildx) -- [x] **Remove redundant CI release job**: Delete `release` job from `ci.yml` — fully handled by `release.yml` -- [x] **Tag format**: Use `*.*.*` tags (e.g. `1.0.9`) — `release.yml` pattern is `*.*.*`, not `v*.*.*`. Fix existing `v1.0.x` tags to `1.0.x` -- [x] **Update DEPLOYMENT.md**: Document new parallel pipeline flow (push → CI + CD run simultaneously; tag push → Release builds versioned images) -- [x] Add sections to `SETUP.md` under `## Cloudflare` for detailed instructions: - - [x] DNS: `linearvoid.tv` apex + `www` CNAME → Cloudflare Pages - - [x] DNS: `media.linearvoid.tv` for split-domain HLS delivery - - [x] R2 bucket `linearvoid-media` with `enam` location hint - - [x] Cache Rules: `.ts/.m4s` immutable long TTL (31536000), `.m3u8` short TTL (2s, revalidate) - - [x] Cloudflare Pages: link Gitea repo, build `cd frontend && npm run build`, output `frontend/dist` - - [x] Tiered Cache: enable Smart Tiered Cache, validate `Cf-Cache-Status: HIT` from upper tier - - [x] WAF: enable OWASP Managed Ruleset, set to Simulate → review 48h → Block, enable Bot Fight Mode - - [x] Rate limiting: `/api/admin/login` + `/api/admin/*` → 10 req/10s per IP, Challenge action - - [x] HTTPS/WSS: Always Use HTTPS, HSTS max-age=31536000 includeSubDomains preload, min TLS 1.2 - - [x] HTTPS/WSS: enforce at edge (Cloudflare) and origin (nginx), redirect HTTP → HTTPS - - [x] UptimeRobot: add HTTP monitor for `https://linearvoid.tv/health` (5min interval) - - [x] UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword: `#EXTM3U`, 5min) - - [x] Alert on 2 consecutive failures, notify via email + Discord/Slack webhook -- [x] Check all known URLs for metrics and their availability: - - [x] Prometheus: `http://:9090` (targets, graph, alerts) - - [x] Grafana: `http://:3001` (default: admin/admin, change via `GRAFANA_PASSWORD`) - - [x] Chat metrics: `http://chat:3000/metrics` from Prometheus container - - [x] Playout metrics: `http://playout:8081/metrics` from Prometheus container - - [x] Nginx metrics: `http://nginx-exporter:9113/metrics` from Prometheus container - - [x] Redis metrics: `http://redis-exporter:9121/metrics` from Prometheus container - - [x] Postgres metrics: `http://postgres-exporter:9187/metrics` from Prometheus container - - [x] Node metrics: `http://node-exporter:9100/metrics` from Prometheus container -- [x] Make sure Prometheus config is mounted into container via named volume `prometheus-config` (not bind mount) -- [x] Make sure Prometheus alerting rules are configured and working (8 rules: ChatDown, PlayoutDown, RedisDown, etc.) -- [x] Verify that all Prometheus targets are up and reporting metrics -- [x] Verify that Grafana dashboards are imported and displaying metrics correctly -- [x] Video normalization: test with MOV, MKV, variable framerate, different audio layouts +- Check `monitoring/prometheus/prometheus.yml` for completeness + - Make sure all 7 scrape targets are included (chat, playout, nginx, redis, postgres, node, prometheus) + - Check `monitoring/prometheus/alerts.yml` for completeness + - Make sure configuration for prometheus is mounted into container via named volume +- Use folder `monitoring/grafana/` to create ini file for Grafana, set `GF_PATHS_CONFIG` environment variable to point to a different location than `/etc/grafana/grafana.ini` if needed + - Set default password from `GRAFANA_PASSWORD` env var, disable user signups + - Configure Prometheus data source in Grafana ini file + - Include dashboards from `monitoring/grafana/dashboards/` directory to be imported into Grafana on startup +- **CD trigger parallel**: Change `cd.yml` from `workflow_run: [CI]` to `push: branches: [main]` so deploy runs in parallel with CI, not after +- **CD failure fix**: Remove `environment: production` from `cd.yml` deploy job (no Gitea environment configured, causes silent job abort after Buildx) +- **Remove redundant CI release job**: Delete `release` job from `ci.yml` — fully handled by `release.yml` +- **Tag format**: Use `*.*.*` tags (e.g. `1.0.9`) — `release.yml` pattern is `*.*.*`, not `v*.*.*`. Fix existing `v1.0.x` tags to `1.0.x` +- **Update DEPLOYMENT.md**: Document new parallel pipeline flow (push → CI + CD run simultaneously; tag push → Release builds versioned images) +- Add sections to `SETUP.md` under `## Cloudflare` for detailed instructions: + - DNS: `linearvoid.tv` apex + `www` CNAME → Cloudflare Pages + - DNS: `media.linearvoid.tv` for split-domain HLS delivery + - R2 bucket `linearvoid-media` with `enam` location hint + - Cache Rules: `.ts/.m4s` immutable long TTL (31536000), `.m3u8` short TTL (2s, revalidate) + - Cloudflare Pages: link Gitea repo, build `cd frontend && npm run build`, output `frontend/dist` + - Tiered Cache: enable Smart Tiered Cache, validate `Cf-Cache-Status: HIT` from upper tier + - WAF: enable OWASP Managed Ruleset, set to Simulate → review 48h → Block, enable Bot Fight Mode + - Rate limiting: `/api/admin/login` + `/api/admin/*` → 10 req/10s per IP, Challenge action + - HTTPS/WSS: Always Use HTTPS, HSTS max-age=31536000 includeSubDomains preload, min TLS 1.2 + - HTTPS/WSS: enforce at edge (Cloudflare) and origin (nginx), redirect HTTP → HTTPS + - UptimeRobot: add HTTP monitor for `https://linearvoid.tv/health` (5min interval) + - UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword: `#EXTM3U`, 5min) + - Alert on 2 consecutive failures, notify via email + Discord/Slack webhook +- Check all known URLs for metrics and their availability: + - Prometheus: `http://:9090` (targets, graph, alerts) + - Grafana: `http://:3001` (default: admin/admin, change via `GRAFANA_PASSWORD`) + - Chat metrics: `http://chat:3000/metrics` from Prometheus container + - Playout metrics: `http://playout:8081/metrics` from Prometheus container + - Nginx metrics: `http://nginx-exporter:9113/metrics` from Prometheus container + - Redis metrics: `http://redis-exporter:9121/metrics` from Prometheus container + - Postgres metrics: `http://postgres-exporter:9187/metrics` from Prometheus container + - Node metrics: `http://node-exporter:9100/metrics` from Prometheus container +- Make sure Prometheus config is mounted into container via named volume `prometheus-config` (not bind mount) +- Make sure Prometheus alerting rules are configured and working (8 rules: ChatDown, PlayoutDown, RedisDown, etc.) +- Verify that all Prometheus targets are up and reporting metrics +- Verify that Grafana dashboards are imported and displaying metrics correctly +- Video normalization: test with MOV, MKV, variable framerate, different audio layouts - Evidence: all normalized outputs probed as `h264`, `yuv420p`, `30/1`, `aac`, `48000`, `2ch`. -- [x] Video normalization: verify output plays back-to-back in continuous concat +- Video normalization: verify output plays back-to-back in continuous concat - Evidence: concat remux reported `CONCAT_OK`; output duration `25.344667s`. -- [x] Viewer SPA: test responsive layout on iOS Safari + Android Chrome +- Viewer SPA: test responsive layout on iOS Safari + Android Chrome - Evidence: viewport emulation (`390x844`, `412x915`) showed video/chat/toggle present; fixed 8px horizontal overflow in `frontend/src/style.css` (`100vw` -> `100%`). - Limitation: no physical-device Safari/Chrome test in this run. -- [x] Redis: test restart, verify sessions + presence data survive +- Redis: test restart, verify sessions + presence data survive - Evidence: admin session HTTP `200` before/after Redis restart; seeded presence key visible before/after restart; Redis health returned to `healthy`. ### Runtime Tests (all 4) @@ -84,15 +84,15 @@ Completed tasks from `.github/instructions/TODO.md`. Historical record for CHANG ### Documentation Polish -- [x] API.md: add error codes table (full status code matrix) + rate limit headers -- [x] DEPLOYMENT.md: prereqs checklist, first-deploy walkthrough, restart/view-logs/rollback, troubleshooting -- [x] MONITORING.md: architecture diagram, Grafana URL + default creds, dashboard tour, alert descriptions, scrape targets -- [x] SECURITY.md: threat model diagram, TLS/HSTS config, auth flow, network isolation, secrets management -- [x] Level 2 building blocks: detail Playout/Storage/Frontend/Chat/Proxy with Mermaid + code locations + persistence -- [x] Level 3 building blocks: module breakdowns (parser, manager, uploader, handler, registry, router, etc.) -- [x] Recovery runbook: document quarterly disaster recovery drill +- API.md: add error codes table (full status code matrix) + rate limit headers +- DEPLOYMENT.md: prereqs checklist, first-deploy walkthrough, restart/view-logs/rollback, troubleshooting +- MONITORING.md: architecture diagram, Grafana URL + default creds, dashboard tour, alert descriptions, scrape targets +- SECURITY.md: threat model diagram, TLS/HSTS config, auth flow, network isolation, secrets management +- Level 2 building blocks: detail Playout/Storage/Frontend/Chat/Proxy with Mermaid + code locations + persistence +- Level 3 building blocks: module breakdowns (parser, manager, uploader, handler, registry, router, etc.) +- Recovery runbook: document quarterly disaster recovery drill - Evidence: expanded `docs/incident-runbook.md` with prerequisites, simulation steps, validation checks, and pass/fail criteria. -- [x] Recovery runbook: rehearse quarterly disaster recovery drill +- Recovery runbook: rehearse quarterly disaster recovery drill - Manual step: execute full drill on target environment and capture RTO/RPO evidence. ## Session 2026-08-06 — Documentation Adjustments, Playout, Normalization, Chat, Viewer SPA, Admin, Docker, CI/CD @@ -975,12 +975,12 @@ Notes: Completed 6 of 7 streaming pipeline gaps from STREAMING.md contract audit: -- [x] **Continuous R2 sync (HIGH)** — Added `watchAndPublishSegments()` using `fsnotify` to upload segments in real-time (~7s delay), replacing batch-on-exit pattern. -- [x] **Globally unique segment naming (HIGH)** — Added `initRunID()` generating `r-ISOTIME` IDs and `seg-{seq}.m4s` monotonic naming. Replaced `strftime` collision-prone pattern. -- [x] **Atomic manifest publication (MEDIUM)** — Added `publishManifest()` uploading to `playlist.m3u8.tmp` then `CopyObject` to final key. -- [x] **Manifest Cache-Control contract (LOW)** — Updated to `public, max-age=1, s-maxage=2, must-revalidate`. -- [x] **Player error recovery (MEDIUM)** — Added `hls.on(ERROR)` handler with bounded retries (5x), exponential backoff (2s→32s), reconnecting overlay with spinner. -- [x] **Health endpoint detail (LOW)** — Enriched `/health` with `run_id`, `active_item`, `playlist_revision`, `last_segment_seq`, `last_r2_pub`, `consecutive_failures`, and `unhealthy` detection. +- **Continuous R2 sync (HIGH)** — Added `watchAndPublishSegments()` using `fsnotify` to upload segments in real-time (~7s delay), replacing batch-on-exit pattern. +- **Globally unique segment naming (HIGH)** — Added `initRunID()` generating `r-ISOTIME` IDs and `seg-{seq}.m4s` monotonic naming. Replaced `strftime` collision-prone pattern. +- **Atomic manifest publication (MEDIUM)** — Added `publishManifest()` uploading to `playlist.m3u8.tmp` then `CopyObject` to final key. +- **Manifest Cache-Control contract (LOW)** — Updated to `public, max-age=1, s-maxage=2, must-revalidate`. +- **Player error recovery (MEDIUM)** — Added `hls.on(ERROR)` handler with bounded retries (5x), exponential backoff (2s→32s), reconnecting overlay with spinner. +- **Health endpoint detail (LOW)** — Enriched `/health` with `run_id`, `active_item`, `playlist_revision`, `last_segment_seq`, `last_r2_pub`, `consecutive_failures`, and `unhealthy` detection. - [ ] CORS on `media.linearvoid.tv` — Remaining infrastructure-only (Cloudflare R2 CORS configuration, not code). **Files changed:** `backend/playout/src/playlist.go` (+171/-35), `playlist_test.go` (+51/-7), `go.mod` (+2), `frontend/src/main.ts` (+76/-2), `main.test.ts` (+43/-1), `style.css` (+37), `index.html` (+5). @@ -990,11 +990,100 @@ Completed 6 of 7 streaming pipeline gaps from STREAMING.md contract audit: Completed Phase 2 (viewer) and Phase 3 (admin) responsive/accessibility tests + coverage: -- [x] **Viewer accessibility (5 tests):** video controls/autoplay/playsinline attributes, chat panel aria-labels (live region + panel label), chat toggle aria-expanded state, message input label association, reconnecting overlay aria-live=assertive + role=alert. -- [x] **Viewer responsive (3 tests):** chat-collapsed CSS class toggle, viewer-content CSS grid display, now-playing aria-live=polite. -- [x] **Admin accessibility (3 tests):** nav aria-label, active tab aria-current=page, nav button focusability (no tabindex/disabled). -- [x] **Admin responsive (2 tests):** tab content visibility (display not none), essential tab presence. -- [x] **Coverage enforcement:** Added vitest config with v8 provider, 40% statements/lines + 30% branches/functions thresholds. Added `test:coverage` script and CI step. +- **Viewer accessibility (5 tests):** video controls/autoplay/playsinline attributes, chat panel aria-labels (live region + panel label), chat toggle aria-expanded state, message input label association, reconnecting overlay aria-live=assertive + role=alert. +- **Viewer responsive (3 tests):** chat-collapsed CSS class toggle, viewer-content CSS grid display, now-playing aria-live=polite. +- **Admin accessibility (3 tests):** nav aria-label, active tab aria-current=page, nav button focusability (no tabindex/disabled). +- **Admin responsive (2 tests):** tab content visibility (display not none), essential tab presence. +- **Coverage enforcement:** Added vitest config with v8 provider, 40% statements/lines + 30% branches/functions thresholds. Added `test:coverage` script and CI step. **Files:** `frontend/src/main.test.ts` (+65), `frontend/src/admin.test.ts` (+55), `frontend/vitest.config.ts` (+12), `frontend/package.json` (+1), `.gitea/workflows/ci.yml` (+3). **Total:** 44 frontend tests (31 existing + 13 new). + +## Session 2026-08-13 — Plan consolidation (TODO → DONE) + +Completed tasks moved out of `.github/instructions/TODO.md`. Remaining work replanned in TODO from `BACKLOG.md`. + +### Phase 1 — playout test fixtures + source-media contract + +- Create small compliant fixture media in `backend/playout/testdata/` for the validator and playout tests +- Create invalid fixture media for codec, framerate, and audio rejection paths +- Create playlist fixtures for empty, comment-only, mixed-validity, and valid playlists +- Document the ffmpeg fixture-generation commands in the relevant test headers + +### Phase 2 — viewer frontend coverage + +- HLS player startup, manifest resolution, and cross-origin/local fallback tests +- Chat send/receive, presence updates, private DM flow, and reconnect tests +- Responsive-layout and accessibility checks for the viewer shell and chat sidebar +- Coverage reporting added and minimum threshold enforced in CI + +### Phase 3 — admin frontend coverage + +- Playlist management, moderation flows, and dashboard rendering tests +- API integration points with mocked backend responses +- Responsive-layout and accessibility checks for admin screens +- Coverage reporting added and minimum threshold enforced in CI + +### Phase 4 — R2 and edge-delivery validation + +- Integration test for upload and retrieval of HLS objects through the R2 path +- Segment and manifest cache headers verified (immutable media, live manifest freshness) +- Edge cache hit/miss validation scripted (`scripts/validate-edge-cache.sh`) +- Stubs/mocks documented (putObject variable-injection pattern) + +### Phase 7 (partial) — load-test scaffolding + saturation alerts + +- WebSocket load-test scaffolding and summary artifacts (`ws-chat.yml`, `run-loadtests.ps1`, `summarize-artillery.ps1`) +- Admin API benchmark scaffolding and endpoint summary artifacts (`admin-api.js`, `run-loadtests.ps1`, `summarize-k6.ps1`) +- Saturation thresholds added to monitoring (chat p95 latency + Redis memory usage) +- Cache-hit threshold alert operationalization added to monitoring + +### Streaming pipeline gaps (audit → implemented) + +- Continuous R2 sync during playout (fsnotify watcher, ~7s delay) +- Globally unique segment naming with run ID (`r-{ISOTIME}/seg-{seq}.m4s`) +- Atomic manifest publication (upload to `.tmp`, then CopyObject) +- Manifest Cache-Control: `public, max-age=1, s-maxage=2, must-revalidate` +- Player error recovery (bounded retries, exponential backoff, reconnecting overlay) +- Health endpoint detail (run ID, active item, playlist revision, segment seq, R2 pub, consecutive failures) +- CORS headers — documented as Cloudflare R2 infrastructure config +- Phase 4 edge cache validation — `scripts/validate-edge-cache.sh` created + +## Session 2026-08-13 — Playout Enhancements + +Completed the three Playout enhancement workstreams from `TODO.md`. Remaining polish items kept in TODO. + +### 1. Automatic normalization of source media on upload (H.264/AAC, 30fps, keyframe check) + +- Implement `media.NormalizeVideo` in `backend/chat/src/media/media.go` (ffmpeg exec, Linear Void profile, bounded 2h timeout, stderr capture) +- Write normalized output to `*.normalized.mp4` and re-verify with ffprobe (`ValidateNormalization` + `ProbeDurationSeconds`) +- Extend media status states: `normalizing`, `normalization_failed` (alongside `ready` / `needs_normalization`) +- Integrate into single-upload path — `HandleAdminMedia`: persist `needs_normalization` then `go normalizeMediaRecord(...)` +- Integrate into chunked-upload completion — `HandleAdminMediaUploads` `"complete"`: same async flow after assembly + validation +- Add `POST /api/admin/media/{id}/normalize` (behind `requireAdminAuth`, returns 202 + `normalizing` record) +- Admin UI (`admin.ts`): "Normalize" action + status polling until `ready`/`normalization_failed` +- Tests: `NormalizeVideo` args + ffmpeg-error path; handler `normalizeMediaRecord` success/failure; `/normalize` endpoint 202 +- Retire hardcoded `backend/playout/normalize-video.sh` (superseded by in-Go normalization) - file deleted + +### 2. R2 data retention: delete old segments after 24h + +- Add `cleanupOldSegments(ctx)` to `backend/playout/src/playlist.go` +- Use `s3.ListObjectsV2` on prefix `hls/` to enumerate run directories +- Delete `hls/r-*/*` older than `now - R2_SEGMENT_RETENTION_HOURS` (default 24) +- Never delete `hls/playlist.m3u8` or `hls/playlist.m3u8.tmp` +- Run cleanup as a ticker goroutine (hourly) plus once at startup +- Add `R2_SEGMENT_RETENTION_HOURS` (default `24`) to `.env.example` +- Expose `linearvoid_playout_r2_cleanup_objects_total` metric and log deleted keys +- Tests: mock `ListObjectsV2`/`DeleteObjects` via injectable vars (mirror the existing `putObject` pattern) + +### 3. Continuous playout: confirm ffmpeg loops without interruption + +- Resolve the dual-loop ambiguity between `main()` and `start-playout.sh` +- Add a `-validate` mode to `main()`: run `ValidatePlaylist` and exit 0/1 without the ffmpeg loop +- Update `start-playout.sh` to invoke `-validate` and `exec` the Go binary (single source of truth) +- Minimize the restart gap: skip full `resetHLSOutput` when the playlist is unchanged +- Track `playlistRevision` (SHA-256 fingerprint); reset only when the playlist changed, on crash/slate recovery, or first run +- Continue segment numbering via `-start_number` + prune stale segments (`hlsNextStartNumber`) +- Verify `init.mp4` + manifest are correctly published on clean completion (existing `publishManifest` path runs after every run) +- Add metrics `linearvoid_playout_loops_total` (clean completions) and `linearvoid_playout_run_duration_seconds` +- Tests: `TestPlaylistFingerprint`, `TestHLSNextStartNumber` diff --git a/.github/instructions/TODO.md b/.github/instructions/TODO.md index 3a30040..24f152f 100644 --- a/.github/instructions/TODO.md +++ b/.github/instructions/TODO.md @@ -1,80 +1,90 @@ # TODO -> Completed tasks moved to [DONE](./DONE.md). Full remaining task list in [BACKLOG](./BACKLOG.md). +> Completed tasks moved to [DONE](./DONE.md). Remaining tasks in [BACKLOG](./BACKLOG.md). -## Plan +## Enhancements -### Phase 1: unblock playout test fixtures and lock down the source-media contract +### Application configuration -- [x] Create small compliant fixture media in `backend/playout/testdata/` for the validator and playout tests -- [x] Create invalid fixture media for codec, framerate, and audio rejection paths -- [x] Create playlist fixtures for empty, comment-only, mixed-validity, and valid playlists -- [x] Document the ffmpeg fixture-generation commands in the relevant test headers +- [ ] Configuration: create simple getter/setter for configuration values (e.g. `MATOMO_URL`, `MATOMO_SITE_ID`) in `backend/admin/config.go` +- [ ] Configuration: expose configuration values to frontend via `/api/admin/config` endpoint +- [ ] Configuration: make configuration values editable: + - [ ] HLS_SEGMENT_DURATION + - [ ] HLS_WINDOW_SIZE + - [ ] MATOMO_URL + - [ ] MATOMO_SITE_ID -### Phase 2: finish frontend coverage for the live viewer experience +### Admin frontend -- [x] Add tests for HLS player startup, manifest resolution, and cross-origin/local fallback behavior -- [x] Add tests for chat send/receive, presence updates, private DM flow, and reconnect behavior -- [x] Add responsive-layout and accessibility checks for the viewer shell and chat sidebar -- [x] Add coverage reporting and enforce a minimum threshold in CI +- [ ] Playlist: show `needs_normalization` badge and block/queue playlist-add until `ready` +- [ ] Playlist: show `ready` badge and allow playlist-add when `ready` +- [ ] Extend `system.html` with a section for configuration settings on top of page (HLS segment duration, window size, Matomo URL/site ID) +- [ ] Add functions to `admin.ts` to call `/api/admin/config` for getting and updating configuration values -### Phase 3: finish frontend coverage for the admin experience +### User frontend -- [x] Add tests for playlist management, moderation flows, and dashboard rendering -- [x] Add tests for API integration points with mocked backend responses -- [x] Add responsive-layout and accessibility checks for admin screens -- [x] Add coverage reporting and enforce a minimum threshold in CI +- [ ] Player frontend: Embed JavaScript snippet in `index.html` to send pageview events to Matomo -### Phase 4: validate the R2 and edge-delivery path +### Chat user features -- [x] Add an integration test for upload and retrieval of HLS objects through the R2 path -- [x] Verify segment and manifest cache headers, including immutable media and live manifest freshness -- [x] Validate edge cache hit/miss behavior with repeat requests -- [x] Document any required stubs or mocks so the test can run in CI (putObject mock pattern documented in test) +- [ ] Chat: add user registration and login flows +- [ ] Chat: private DM history and search +- [ ] Chat UI: add login/logout button, show username in chat, show/toggle user list, show/toggle online status, show typing indicator + +### Monitoring + +- [ ] Prometheus TSDB retention policy: delete metrics older than 30d + +## Setup phases remaining ### Phase 5: complete manual Cloudflare and external-service setup - [ ] Apply Cloudflare Pages, Tiered Cache, WAF, rate limiting, and HTTPS/HSTS settings in the dashboard -- [ ] Create or verify UptimeRobot monitors for health and HLS manifest availability +- [ ] Cloudflare Pages: link custom domain and cut over frontend hosting +- [ ] Enable Smart Tiered Cache; validate `Cf-Cache-Status: HIT` from upper tier +- [ ] WAF: enable OWASP Managed Ruleset (Simulate → review 48h → Block), enable Bot Fight Mode +- [ ] Rate limiting: `/api/admin/login` + `/api/admin/*` → 10 req/10s per IP, Challenge action +- [ ] HTTPS/WSS: Always Use HTTPS, HSTS max-age=31536000 includeSubDomains preload, min TLS 1.2; enforce at edge and origin +- [ ] UptimeRobot: add HTTP monitor for `https://linearvoid.tv/health` (5min interval) +- [ ] UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword `#EXTM3U`, 5min) +- [ ] UptimeRobot: alert on 2 consecutive failures via email + Discord/Slack webhook - [ ] Capture evidence in `docs/operations/cloudflare-uptimerobot-checklist.md` ### Phase 6: rehearse production deployment end to end -- [ ] Provision the Hetzner host and prepare the Docker/runtime baseline -- [ ] Configure production secrets, remote deployment, and DNS cutover -- [ ] Run health, HLS, chat, and admin smoke tests against production-like infrastructure -- [ ] Verify backup creation, restore, and integrity checks on the production host +- [ ] Provision Hetzner CX22 (2 vCPU, 4 GB, 40 GB) in Ashburn VA, Ubuntu 24.04 +- [ ] Install Docker Engine + Compose plugin; open firewall: 80/443 public, 2375 runner-only +- [ ] Configure Docker daemon for remote TLS access (`-H tcp://0.0.0.0:2376` with certs) +- [ ] Create `/srv/linearvoid/data/{playlist,media,hls,chat,slate}` directories +- [ ] Add `CI_PROD_*` secrets to Gitea (DOCKER_HOST, R2 keys, admin password, DB password, Grafana password) +- [ ] Trigger CD workflow with `DOCKER_HOST` pointing to Hetzner IP +- [ ] Verify all 11 containers healthy: `/health`, `/hls/playlist.m3u8`, WebSocket +- [ ] Update Cloudflare A record `linearvoid.tv` → Hetzner IP, SSL Full (strict); test HTTPS + WSS +- [ ] Smoke test: viewer SPA loads, HLS plays, chat connects, admin login → upload → playlist → playout +- [ ] Set up Prometheus alerting notifications + UptimeRobot on production URL +- [ ] Run `scripts/backup-db.sh` + `scripts/verify-backups.sh` on production host +- [ ] Document production IP, SSH access, rollback procedure, scaling thresholds ### Phase 7: baseline scaling and performance -- [ ] Measure manifest TTFB and cache-hit distribution from multiple US regions -- [x] Prepare WebSocket load-test scaffolding and summary artifacts (`ws-chat.yml`, `run-loadtests.ps1`, `summarize-artillery.ps1`) -- [ ] Execute WebSocket throughput and chat-latency benchmarks under growing connection counts -- [x] Prepare Admin API benchmark scaffolding and endpoint summary artifacts (`admin-api.js`, `run-loadtests.ps1`, `summarize-k6.ps1`) -- [ ] Execute Admin API benchmarks and identify slow paths -- [x] Add saturation thresholds to monitoring (chat p95 latency + Redis memory usage) -- [x] Add cache-hit threshold alert operationalization to monitoring +- [ ] Execute WebSocket load tests — ramp 10→100→500→1000 connections; measure p95 latency + errors +- [ ] Execute admin API benchmarks — `wrk`/`k6` against `/api/admin/*`; target p95 <1s +- [ ] Measure manifest TTFB and cache-hit distribution from multiple US regions (WebPageTest 3+ regions, <4s warm cache) +- [ ] Tune manifest TTL to 0.5× segment duration; validate `Cf-Cache-Status` distribution +- [ ] Track cache-hit ratio and R2 ops in production (Cloudflare GraphQL → Grafana panel, alert <90%) +- [ ] Capacity triggers: Prometheus rules for cache-miss >10%/15min, CPU >80%/10min +- [ ] Plan Phase 2 service separation (playout CPU, chat latency, DB state, monitoring) +- [ ] Plan Phase 3 multi-origin resilience (second Hetzner location, warm standby, DNS failover, DB/Redis replication) -## Status Update (2026-08-10) +### Phase 8: close out remaining test and validation gaps -- [x] Phase 1 (test fixtures) complete. -- [x] Phase 2 (viewer frontend): player startup, URL resolution, chat send/receive, reconnect, presence, DM flow, message rendering all covered (27 tests). -- [x] Phase 3 (admin frontend): dashboard rendering, API integration, playlist management all covered (31 tests total). -- [x] Phase 4 (R2 validation) partially complete: SyncToR2 cache headers + integration tests done; edge cache hit/miss remains manual. -- [ ] Phase 5 (Cloudflare): Pages deploy workflow (`.gitea/workflows/pages-deploy.yml`) created; Pages/Workers setup docs written; manual dashboard tasks remain. -- [ ] Phase 6 (production deployment) pending: Hetzner provisioning, secrets, DNS cutover, smoke tests, backup/restore. -- [ ] Phase 7: Loadtest scaffolding complete; `run-loadtests.ps1` auto-generates ws-chat-summary.txt and admin-api-endpoint-summary.txt; live benchmark execution pending. -- Version tagged 1.0.11; go vet clean. +- [ ] Admin frontend: user moderation tools tests (mute/ban/report, view moderation logs) +- [ ] Frontend: coverage for all critical user + admin components and interactions +- [ ] Frontend: document test cases and expected outcomes +- [ ] Backend R2: test retrieval of segments/manifests via Cloudflare edge +- [ ] Backend R2: validate edge cache hit/miss behavior with repeat requests (`scripts/validate-edge-cache.sh`) -## Streaming Pipeline Gaps (2026-08-10 audit → 2026-08-10 implemented) +### Playout -STREAMING.md contract vs playlist.go/main.ts implementation. See Backlog for full details. - -- [x] Continuous R2 sync during playout (HIGH) — implemented: fsnotify watcher uploads segments within ~7s -- [x] Globally unique segment naming with run ID (HIGH) — implemented: `r-{ISOTIME}/seg-{seq}.m4s` -- [x] Atomic manifest publication with temp key (MEDIUM) — implemented: upload to .tmp, then CopyObject -- [x] Manifest Cache-Control: add must-revalidate + s-maxage split (LOW) — implemented: `public, max-age=1, s-maxage=2, must-revalidate` -- [ ] CORS headers on HLS responses from media.linearvoid.tv (MEDIUM) — infrastructure, needs Cloudflare R2 CORS config -- [x] Player error recovery: retries, backoff, reconnecting state (MEDIUM) — implemented: hls.on(ERROR) handler with exponential backoff -- [x] Health endpoint: add active item, revision, run ID, segment info (LOW) — implemented: full STREAMING.md contract fields -- [x] Phase 4 edge cache validation — script `scripts/validate-edge-cache.sh` created +- [ ] Configure an R2 lifecycle rule on the `hls/` prefix as defense-in-depth (manual) +- [ ] Full-loop E2E (2-file playlist, assert `Looping...` with no `CRASHED` across the boundary) — manual via `scripts/test-playout-e2e.sh` diff --git a/CHANGELOG.md b/CHANGELOG.md index 3c0b61e..f5a94b4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ ## [Unreleased] +### Added + +- **Automatic media normalization on upload**: Chat backend auto-transcodes non-conforming uploads. New `media.NormalizeVideo` wraps ffmpeg with the Linear Void profile (H.264/AAC, 30fps, 2s keyframes, yuv420p, 48kHz stereo), re-verifies output with ffprobe, and writes `*.normalized.mp4`. Media records gain `normalizing` / `normalization_failed` statuses; `POST /api/admin/media/{id}/normalize` re-triggers manually. Admin UI shows a "Normalize" action with status polling. +- **R2 segment retention**: Playout `cleanupOldSegments` deletes `hls/r-*/*` objects older than `R2_SEGMENT_RETENTION_HOURS` (default 24h), run hourly + at startup, never touching the live manifest. Exposes `linearvoid_playout_r2_cleanup_objects_total` metric. `.env.example` adds `R2_SEGMENT_RETENTION_HOURS`. +- **Playout loop metrics**: `linearvoid_playout_loops_total` (clean completions) and `linearvoid_playout_run_duration_seconds` (last run duration). + +### Changed + +- **Playout seamless looping**: `main()` now only resets HLS output when the playlist changed (SHA-256 `playlistRevision`), on crash/slate recovery, or first run; otherwise it continues segment numbering via `-start_number` and prunes stale segments. Eliminates the viewer gap on clean playlist loops. +- **Single playout entrypoint**: `start-playout.sh` now runs `/app/playlist-manager -validate` then `exec`s the Go binary, removing the duplicate shell ffmpeg loop. Added a `-validate` mode to `main()`. + +### Tests + +- Media: `TestNormalizationArgs`, `TestNormalizeVideo_FFmpegError`. +- Handlers: `TestNormalizeMediaRecord_Success`, `TestNormalizeMediaRecord_Failure`, `TestHandleAdminMediaByID_Normalize`. +- Playout: `TestCleanupOldSegments`, `TestR2SegmentRetentionHours`, `TestPlaylistFingerprint`, `TestHLSNextStartNumber`. + ## [1.0.12] - 2026-08-11 ### Changed @@ -13,7 +30,7 @@ ### Fixed - **Playout R2 endpoint validation**: `initR2Client()` now validates all required R2 environment variables (`R2_ENDPOINT`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET_NAME`) before creating the S3 client. Previously, an empty `R2_ENDPOINT` caused cryptic `Custom endpoint '' was not a valid URI` errors on every segment upload attempt. Playout now gracefully degrades when R2 is not configured — local HLS via nginx continues to work. -- **CD workflow missing R2 secrets**: `.gitea/workflows/cd.yml` "Prepare environment" step now injects all R2_ variables (`R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET_NAME`, `R2_ENDPOINT`, plus admin variants) from Gitea secrets. Previously the generated `.env` was missing R2 entirely, causing the playout container to start with an empty endpoint. +- **CD workflow missing R2 secrets**: `.gitea/workflows/cd.yml` "Prepare environment" step now injects all R2\_ variables (`R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET_NAME`, `R2_ENDPOINT`, plus admin variants) from Gitea secrets. Previously the generated `.env` was missing R2 entirely, causing the playout container to start with an empty endpoint. - **Deployment docs**: `docs/DEPLOYMENT.md` now includes explicit R2 and Cloudflare variable lists plus a "Gitea Secrets" table documenting all 15 required secrets. ## [1.0.11] - 2026-08-10 diff --git a/README.md b/README.md index 9a85e68..2c2db33 100644 --- a/README.md +++ b/README.md @@ -46,6 +46,8 @@ flowchart TD P -->|sync| R2 ``` +A detailed description of the Streaming architecture, playout, and delivery is available in [`docs/STREAMING.md`](docs/STREAMING.md). + ## Project Structure This structure directly reflects the building blocks, runtime flows, deployment view, and cross-cutting concepts defined in the arc42 documentation. diff --git a/backend/chat/src/handlers/handlers.go b/backend/chat/src/handlers/handlers.go index 220b4b7..003d130 100644 --- a/backend/chat/src/handlers/handlers.go +++ b/backend/chat/src/handlers/handlers.go @@ -976,9 +976,9 @@ func HandleAdminMedia(deps *Dependencies) http.HandlerFunc { } normResult := media.ValidateNormalization(targetPath) - status := "ready" + status := media.StatusReady if !normResult.Valid { - status = "needs_normalization" + status = media.StatusNeedsNormalization } filename := filepath.Base(targetPath) @@ -998,6 +998,10 @@ func HandleAdminMedia(deps *Dependencies) http.HandlerFunc { return } + if !normResult.Valid { + go normalizeMediaRecord(deps, mediaRecord) + } + files, err := media.ListFiles(deps.DB) if err != nil { http.Error(w, fmt.Sprintf("failed to list media files: %v", err), http.StatusInternalServerError) @@ -1307,9 +1311,9 @@ func HandleAdminMediaUploads(deps *Dependencies) http.HandlerFunc { } normResult := media.ValidateNormalization(targetPath) - status := "ready" + status := media.StatusReady if !normResult.Valid { - status = "needs_normalization" + status = media.StatusNeedsNormalization } filename := filepath.Base(targetPath) @@ -1329,6 +1333,10 @@ func HandleAdminMediaUploads(deps *Dependencies) http.HandlerFunc { return } + if !normResult.Valid { + go normalizeMediaRecord(deps, mediaRecord) + } + _ = os.RemoveAll(media.UploadSessionDir(mediaDir, uploadID)) files, err := media.ListFiles(deps.DB) @@ -1348,6 +1356,11 @@ func HandleAdminMediaUploads(deps *Dependencies) http.HandlerFunc { func HandleAdminMediaByID(deps *Dependencies) http.HandlerFunc { mediaDir := config.MediaDir() return func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/normalize") { + handleNormalizeMedia(deps, w, r) + return + } + mediaID, err := media.GetMediaIDFromPath(r.URL.Path, "/api/admin/media/") if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) @@ -1424,6 +1437,75 @@ func HandleAdminMediaByID(deps *Dependencies) http.HandlerFunc { } } +// normalizeMediaRecord runs ffmpeg normalization for a media record and updates +// its status, path, duration, and size. Safe to run asynchronously. +func normalizeMediaRecord(deps *Dependencies, record db.Media) { + src := record.Path + dst := strings.TrimSuffix(src, filepath.Ext(src)) + ".normalized.mp4" + + if err := deps.DB.Model(&record).Update("status", media.StatusNormalizing).Error; err != nil { + return + } + + duration, size, err := media.NormalizeVideo(src, dst) + if err != nil { + _ = deps.DB.Model(&record).Updates(map[string]any{ + "status": media.StatusNormalizationFailed, + "tags": `"normalization":false`, + }).Error + return + } + + _ = deps.DB.Model(&record).Updates(map[string]any{ + "path": dst, + "filename": filepath.Base(dst), + "size_bytes": size, + "duration_seconds": duration, + "status": media.StatusReady, + "tags": `"normalization":true`, + }).Error +} + +// handleNormalizeMedia handles POST /api/admin/media/{id}/normalize. +func handleNormalizeMedia(deps *Dependencies, w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + idPart := strings.TrimSuffix(r.URL.Path, "/normalize") + idPart = strings.TrimPrefix(idPart, "/api/admin/media/") + idPart = strings.Trim(idPart, "/") + mediaID, err := strconv.ParseUint(idPart, 10, 64) + if err != nil || mediaID == 0 { + http.Error(w, "invalid media id", http.StatusBadRequest) + return + } + + var record db.Media + if err := deps.DB.First(&record, mediaID).Error; err != nil { + http.Error(w, "media item not found", http.StatusNotFound) + return + } + + if record.Status == media.StatusNormalizing { + http.Error(w, "normalization already in progress", http.StatusConflict) + return + } + + record.Status = media.StatusNormalizing + if err := deps.DB.Model(&record).Update("status", media.StatusNormalizing).Error; err != nil { + http.Error(w, fmt.Sprintf("failed to update media status: %v", err), http.StatusInternalServerError) + return + } + + go normalizeMediaRecord(deps, record) + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusAccepted) + json.NewEncoder(w).Encode(mediaEntryFromRecord(record)) +} + // HandleAdminChatLog lists chat log entries. func HandleAdminChatLog(deps *Dependencies) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { diff --git a/backend/chat/src/handlers/handlers_test.go b/backend/chat/src/handlers/handlers_test.go index 9772002..1513af8 100644 --- a/backend/chat/src/handlers/handlers_test.go +++ b/backend/chat/src/handlers/handlers_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "encoding/json" + "fmt" "net/http" "net/http/httptest" "os" @@ -19,6 +20,7 @@ import ( "chat/src/auth" "chat/src/config" "chat/src/db" + "chat/src/media" "chat/src/ws" ) @@ -944,4 +946,133 @@ func TestHandleAdminMediaUploads_InvalidFile(t *testing.T) { if w.Code != http.StatusBadRequest { t.Errorf("status = %d, want 400", w.Code) } +} + +func TestNormalizeMediaRecord_Success(t *testing.T) { + deps, gormDB, _ := setupTestDeps(t) + + tmpDir := t.TempDir() + os.Setenv("MEDIA_DIR", tmpDir) + defer os.Unsetenv("MEDIA_DIR") + + srcPath := filepath.Join(tmpDir, "source.mov") + if err := os.WriteFile(srcPath, []byte("fake video"), 0o644); err != nil { + t.Fatal(err) + } + + gormDB.Create(&db.Media{ + Filename: "source.mov", + OriginalName: "source.mov", + Path: srcPath, + SizeBytes: 10, + DurationSeconds: 0, + Status: "needs_normalization", + Title: "source", + }) + + origNormalize := media.NormalizeVideo + t.Cleanup(func() { media.NormalizeVideo = origNormalize }) + media.NormalizeVideo = func(src, dst string) (int, int64, error) { + if err := os.WriteFile(dst, []byte("normalized"), 0o644); err != nil { + return 0, 0, err + } + return 120, 10, nil + } + + var record db.Media + gormDB.First(&record, 1) + normalizeMediaRecord(deps, record) + + var updated db.Media + gormDB.First(&updated, 1) + if updated.Status != "ready" { + t.Errorf("Status = %q, want ready", updated.Status) + } + if updated.DurationSeconds != 120 { + t.Errorf("DurationSeconds = %d, want 120", updated.DurationSeconds) + } + if !strings.HasSuffix(updated.Path, ".normalized.mp4") { + t.Errorf("Path = %q, want .normalized.mp4 suffix", updated.Path) + } +} + +func TestNormalizeMediaRecord_Failure(t *testing.T) { + deps, gormDB, _ := setupTestDeps(t) + + tmpDir := t.TempDir() + os.Setenv("MEDIA_DIR", tmpDir) + defer os.Unsetenv("MEDIA_DIR") + + srcPath := filepath.Join(tmpDir, "source.mov") + if err := os.WriteFile(srcPath, []byte("fake video"), 0o644); err != nil { + t.Fatal(err) + } + + gormDB.Create(&db.Media{ + Filename: "source.mov", + OriginalName: "source.mov", + Path: srcPath, + SizeBytes: 10, + DurationSeconds: 0, + Status: "needs_normalization", + Title: "source", + }) + + origNormalize := media.NormalizeVideo + t.Cleanup(func() { media.NormalizeVideo = origNormalize }) + media.NormalizeVideo = func(src, dst string) (int, int64, error) { + return 0, 0, fmt.Errorf("ffmpeg failed") + } + + var record db.Media + gormDB.First(&record, 1) + normalizeMediaRecord(deps, record) + + var updated db.Media + gormDB.First(&updated, 1) + if updated.Status != "normalization_failed" { + t.Errorf("Status = %q, want normalization_failed", updated.Status) + } +} + +func TestHandleAdminMediaByID_Normalize(t *testing.T) { + deps, gormDB, _ := setupTestDeps(t) + + tmpDir := t.TempDir() + os.Setenv("MEDIA_DIR", tmpDir) + defer os.Unsetenv("MEDIA_DIR") + + srcPath := filepath.Join(tmpDir, "source.mov") + if err := os.WriteFile(srcPath, []byte("fake video"), 0o644); err != nil { + t.Fatal(err) + } + + gormDB.Create(&db.Media{ + Filename: "source.mov", + OriginalName: "source.mov", + Path: srcPath, + SizeBytes: 10, + DurationSeconds: 0, + Status: "needs_normalization", + Title: "source", + }) + + origNormalize := media.NormalizeVideo + t.Cleanup(func() { media.NormalizeVideo = origNormalize }) + media.NormalizeVideo = func(src, dst string) (int, int64, error) { + if err := os.WriteFile(dst, []byte("normalized"), 0o644); err != nil { + return 0, 0, err + } + return 90, 9, nil + } + + handler := HandleAdminMediaByID(deps) + req := httptest.NewRequest("POST", "/api/admin/media/1/normalize", nil) + w := httptest.NewRecorder() + + handler(w, req) + + if w.Code != http.StatusAccepted { + t.Errorf("status = %d, want 202", w.Code) + } } \ No newline at end of file diff --git a/backend/chat/src/media/media.go b/backend/chat/src/media/media.go index a825f85..4a90f35 100644 --- a/backend/chat/src/media/media.go +++ b/backend/chat/src/media/media.go @@ -155,6 +155,71 @@ func ProbeDurationSeconds(mediaPath string) (int, error) { return seconds, nil } +// Media status values (ADR-006 media library states). +const ( + StatusReady = "ready" + StatusNeedsNormalization = "needs_normalization" + StatusNormalizing = "normalizing" + StatusNormalizationFailed = "normalization_failed" +) + +// runCommand executes an external command and returns its combined output on failure. +// It is a package variable so tests can substitute a stub. +var runCommand = func(ctx context.Context, name string, args ...string) error { + out, err := exec.CommandContext(ctx, name, args...).CombinedOutput() + if err != nil { + return fmt.Errorf("%s failed: %v (%s)", name, err, strings.TrimSpace(string(out))) + } + return nil +} + +// normalizationArgs builds the ffmpeg arguments for the Linear Void normalization profile. +func normalizationArgs(srcPath, dstPath string) []string { + return []string{ + "-y", + "-i", srcPath, + "-map", "0:v:0", "-map", "0:a:0?", + "-vf", "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2,fps=30,format=yuv420p", + "-c:v", "libx264", "-preset", "medium", "-crf", "21", + "-pix_fmt", "yuv420p", + "-r", "30", "-g", "60", "-keyint_min", "60", "-sc_threshold", "0", + "-c:a", "aac", "-b:a", "160k", "-ar", "48000", "-ac", "2", + "-avoid_negative_ts", "make_zero", "-max_muxing_queue_size", "1024", + "-movflags", "+faststart", + dstPath, + } +} + +// normalizeVideo transcodes srcPath to the Linear Void normalization profile, +// re-verifies the output with ffprobe, and returns its duration (seconds) and size (bytes). +func normalizeVideo(srcPath, dstPath string) (int, int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour) + defer cancel() + + if err := runCommand(ctx, "ffmpeg", normalizationArgs(srcPath, dstPath)...); err != nil { + return 0, 0, err + } + + if res := ValidateNormalization(dstPath); !res.Valid { + return 0, 0, fmt.Errorf("normalized output failed validation: %s", strings.Join(res.Errors, "; ")) + } + + duration, err := ProbeDurationSeconds(dstPath) + if err != nil { + return 0, 0, fmt.Errorf("probe duration failed: %w", err) + } + + info, err := os.Stat(dstPath) + if err != nil { + return 0, 0, fmt.Errorf("stat output failed: %w", err) + } + + return duration, info.Size(), nil +} + +// NormalizeVideo is the injectable normalization entry point. +var NormalizeVideo = normalizeVideo + // DetectMime reads the first 512 bytes of a file to detect its MIME type. func DetectMime(filePath string) (string, error) { f, err := os.Open(filePath) diff --git a/backend/chat/src/media/media_test.go b/backend/chat/src/media/media_test.go index a101bd9..6f24972 100644 --- a/backend/chat/src/media/media_test.go +++ b/backend/chat/src/media/media_test.go @@ -1,7 +1,10 @@ package media import ( + "context" + "fmt" "path/filepath" + "strings" "testing" ) @@ -201,4 +204,43 @@ func TestUniqueTargetPath(t *testing.T) { if path != expected { t.Errorf("UniqueTargetPath() = %q, want %q", path, expected) } +} + +func TestNormalizationArgs(t *testing.T) { + args := normalizationArgs("/in/movie.mov", "/out/movie.normalized.mp4") + joined := strings.Join(args, " ") + for _, want := range []string{ + "-map 0:v:0", + "-map 0:a:0?", + "-c:v libx264", + "-r 30", + "-g 60", + "-keyint_min 60", + "-pix_fmt yuv420p", + "-c:a aac", + "-ar 48000", + "-ac 2", + "-avoid_negative_ts make_zero", + "/out/movie.normalized.mp4", + } { + if !strings.Contains(joined, want) { + t.Errorf("normalization args missing %q: %v", want, args) + } + } +} + +func TestNormalizeVideo_FFmpegError(t *testing.T) { + origRun := runCommand + t.Cleanup(func() { runCommand = origRun }) + + runCommand = func(ctx context.Context, name string, args ...string) error { + if name != "ffmpeg" { + t.Errorf("runCommand name = %q, want ffmpeg", name) + } + return fmt.Errorf("simulated ffmpeg failure") + } + + if _, _, err := normalizeVideo("/in/a.mov", "/out/a.normalized.mp4"); err == nil { + t.Fatal("expected error when ffmpeg fails") + } } \ No newline at end of file diff --git a/backend/playout/src/playlist.go b/backend/playout/src/playlist.go index 2fd7cb5..a192f88 100644 --- a/backend/playout/src/playlist.go +++ b/backend/playout/src/playlist.go @@ -3,6 +3,8 @@ package main import ( "bufio" "context" + "crypto/sha256" + "encoding/hex" "encoding/json" "fmt" "net/http" @@ -16,6 +18,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/fsnotify/fsnotify" ) @@ -32,12 +35,20 @@ var ( putObject = func(client *s3.Client, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { return client.PutObject(context.TODO(), input) } - bucket = os.Getenv("R2_BUCKET_NAME") - playoutMode = "starting" // "live", "slate", "degraded" - restartCount = 0 - restartWindow = time.Now() - slateDir = "/app/slate" - hlsDir = "/app/hls" + listObjects = func(client *s3.Client, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + return client.ListObjectsV2(context.TODO(), input) + } + deleteObjects = func(client *s3.Client, input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) { + return client.DeleteObjects(context.TODO(), input) + } + bucket = os.Getenv("R2_BUCKET_NAME") + playoutMode = "starting" // "live", "slate", "degraded" + restartCount = 0 + restartWindow = time.Now() + slateDir = "/app/slate" + hlsDir = "/app/hls" + hlsSegmentDuration = envWithDefault("HLS_SEGMENT_DURATION", "6") + hlsWindowSize = envWithDefault("HLS_WINDOW_SIZE", "24") // Unique segment naming (STREAMING.md contract: r-{runID}/seg-{seq}.m4s) runID string @@ -51,6 +62,14 @@ var ( consecutiveFails int lastFFmpegStartTime time.Time lastFFmpegExitError error + + // R2 retention cleanup counter + r2CleanupObjectsTotal int64 + + // Continuous playout loop metrics + loopsTotal int64 + lastRunDuration time.Duration + lastRunClean bool ) const ( @@ -61,6 +80,14 @@ const ( manifestCacheControl = "public, max-age=1, s-maxage=2, must-revalidate" ) +// envWithDefault returns the value of the environment variable or the default if not set. +func envWithDefault(key, defaultVal string) string { + if v := os.Getenv(key); v != "" { + return v + } + return defaultVal +} + // isR2Configured returns true when the required R2 environment variables are set. func isR2Configured() bool { return os.Getenv("R2_ENDPOINT") != "" && @@ -568,8 +595,21 @@ func resetHLSOutput(hlsDir string) error { func main() { playlistPath := "/app/playlist/playlist.txt" - if len(os.Args) > 1 { - playlistPath = os.Args[1] + validateOnly := false + for _, arg := range os.Args[1:] { + switch arg { + case "-validate": + validateOnly = true + default: + if !strings.HasPrefix(arg, "-") { + playlistPath = arg + } + } + } + + if validateOnly { + runValidateOnly(playlistPath) + return } fmt.Println("=== Linear Void Playout Service Starting ===") @@ -591,6 +631,11 @@ func main() { fmt.Println("R2 sync disabled:", err) } + // Start R2 segment retention cleanup (hourly + once at startup) + if r2Client != nil { + go runR2CleanupLoop() + } + // Start HTTP health check server http.HandleFunc("/health", HealthHandler) http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) { @@ -607,6 +652,15 @@ func main() { modeVal = 2 } fmt.Fprintf(w, "linearvoid_playout_mode %d\n", modeVal) + fmt.Fprintln(w, "# HELP linearvoid_playout_r2_cleanup_objects_total Total R2 objects deleted by retention cleanup.") + fmt.Fprintln(w, "# TYPE linearvoid_playout_r2_cleanup_objects_total counter") + fmt.Fprintf(w, "linearvoid_playout_r2_cleanup_objects_total %d\n", r2CleanupObjectsTotal) + fmt.Fprintln(w, "# HELP linearvoid_playout_loops_total Total clean playlist completions (loops).") + fmt.Fprintln(w, "# TYPE linearvoid_playout_loops_total counter") + fmt.Fprintf(w, "linearvoid_playout_loops_total %d\n", loopsTotal) + fmt.Fprintln(w, "# HELP linearvoid_playout_run_duration_seconds Duration of the last FFmpeg run in seconds.") + fmt.Fprintln(w, "# TYPE linearvoid_playout_run_duration_seconds gauge") + fmt.Fprintf(w, "linearvoid_playout_run_duration_seconds %f\n", lastRunDuration.Seconds()) }) go func() { fmt.Println("Health check server listening on :8081/health") @@ -625,10 +679,34 @@ func main() { // New run ID on each restart for unique R2 segment keys (STREAMING.md) initRunID() - if err := resetHLSOutput(hlsDir); err != nil { - fmt.Println("Failed to reset HLS output:", err) - time.Sleep(5 * time.Second) - continue + // Reset HLS output only when the playlist changed, we are recovering from a + // crash/slate, or this is the first run. Otherwise continue the segment + // sequence so the loop is seamless for viewers (no gap on clean loops). + fingerprint, fpErr := playlistFingerprint(playlistPath) + if fpErr != nil { + fmt.Println("Warning: cannot fingerprint playlist:", fpErr) + fingerprint = "" + } + playlistChanged := fingerprint != playlistRevision + shouldReset := playlistChanged || playoutMode != "live" || !lastRunClean + + windowSize := 24 + if ws, err := strconv.Atoi(hlsWindowSize); err == nil && ws > 0 { + windowSize = ws + } + + startNumber := 0 + if shouldReset { + if err := resetHLSOutput(hlsDir); err != nil { + fmt.Println("Failed to reset HLS output:", err) + time.Sleep(5 * time.Second) + continue + } + playlistRevision = fingerprint + } else { + if n, err := hlsNextStartNumber(hlsDir, windowSize); err == nil { + startNumber = n + } } // Start filesystem watcher for real-time R2 segment upload (STREAMING.md). @@ -652,17 +730,20 @@ func main() { "-r", "30", "-g", "60", "-keyint_min", "60", "-sc_threshold", "0", "-c:a", "aac", "-ar", "48000", "-ac", "2", "-b:a", "128k", "-f", "hls", - "-hls_time", "6", - "-hls_list_size", "24", + "-hls_time", hlsSegmentDuration, + "-hls_list_size", hlsWindowSize, "-hls_segment_type", "fmp4", "-hls_fmp4_init_filename", "init.mp4", "-hls_flags", "delete_segments+independent_segments", "-hls_segment_filename", filepath.Join(hlsDir, "seg-%06d.m4s"), + "-start_number", strconv.Itoa(startNumber), filepath.Join(hlsDir, "playlist.m3u8")) runErr := cmd.Run() runDuration := time.Since(lastFFmpegStartTime) lastFFmpegExitError = runErr + lastRunDuration = runDuration + lastRunClean = runErr == nil if watchCancel != nil { watchCancel() // Stop the filesystem watcher @@ -685,6 +766,7 @@ func main() { // Differentiate clean playlist completion from FFmpeg crash. // A nil error means FFmpeg completed the playlist normally — not a crash. if runErr == nil { + loopsTotal++ fmt.Printf("[%s] Playlist completed normally after %s. Looping...\n", time.Now().UTC().Format(time.RFC3339), runDuration.Round(time.Second)) // If we were in slate mode, a successful run means recovery @@ -747,4 +829,171 @@ func syncLatestToR2(hlsDir string) error { fmt.Println("Synced init.mp4 to R2") return nil +} + +// r2SegmentRetentionHours returns the R2 segment retention window in hours. +func r2SegmentRetentionHours() time.Duration { + h, err := strconv.Atoi(envWithDefault("R2_SEGMENT_RETENTION_HOURS", "24")) + if err != nil || h <= 0 { + h = 24 + } + return time.Duration(h) * time.Hour +} + +// cleanupOldSegments deletes R2 objects under the hls/ prefix that are older than +// the retention window, excluding the live manifest and any non-run keys. +// It returns the number of objects deleted. +func cleanupOldSegments(ctx context.Context) (int, error) { + if r2Client == nil { + if err := initR2Client(); err != nil { + return 0, err + } + } + + cutoff := time.Now().UTC().Add(-r2SegmentRetentionHours()) + + var stale []string + var token *string + for { + output, err := listObjects(r2Client, &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String("hls/"), + ContinuationToken: token, + }) + if err != nil { + return 0, err + } + for _, obj := range output.Contents { + key := aws.ToString(obj.Key) + if key == "hls/playlist.m3u8" || key == "hls/playlist.m3u8.tmp" { + continue + } + // Only consider segment/init objects under a run directory: hls/r-.../... + if !strings.HasPrefix(key, "hls/r-") { + continue + } + if obj.LastModified != nil && obj.LastModified.Before(cutoff) { + stale = append(stale, key) + } + } + if output.IsTruncated == nil || !*output.IsTruncated { + break + } + token = output.NextContinuationToken + } + + if len(stale) == 0 { + return 0, nil + } + + deleted := 0 + for i := 0; i < len(stale); i += 1000 { + end := i + 1000 + if end > len(stale) { + end = len(stale) + } + batch := stale[i:end] + objects := make([]types.ObjectIdentifier, len(batch)) + for j, key := range batch { + objects[j] = types.ObjectIdentifier{Key: aws.String(key)} + } + if _, err := deleteObjects(r2Client, &s3.DeleteObjectsInput{ + Bucket: aws.String(bucket), + Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)}, + }); err != nil { + return deleted, err + } + for _, key := range batch { + fmt.Printf("R2 cleanup: deleted %s\n", key) + } + deleted += len(batch) + } + + r2CleanupObjectsTotal += int64(deleted) + return deleted, nil +} + +// runR2CleanupLoop runs retention cleanup once at startup, then hourly. +func runR2CleanupLoop() { + if _, err := cleanupOldSegments(context.Background()); err != nil { + fmt.Println("R2 cleanup error:", err) + } + ticker := time.NewTicker(time.Hour) + defer ticker.Stop() + for range ticker.C { + if _, err := cleanupOldSegments(context.Background()); err != nil { + fmt.Println("R2 cleanup error:", err) + } + } +} + +// playlistFingerprint returns a short hash of the playlist file content, used to +// detect playlist changes between loops. +func playlistFingerprint(playlistPath string) (string, error) { + data, err := os.ReadFile(playlistPath) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +// hlsNextStartNumber returns the next segment sequence number to use, and prunes +// stale segments outside the live window so they do not accumulate across loops. +func hlsNextStartNumber(hlsDir string, windowSize int) (int, error) { + files, err := filepath.Glob(filepath.Join(hlsDir, "seg-*.m4s")) + if err != nil { + return 0, err + } + + seqOf := func(f string) int { + base := filepath.Base(f) + num := strings.TrimSuffix(strings.TrimPrefix(base, "seg-"), ".m4s") + n, err := strconv.Atoi(num) + if err != nil { + return -1 + } + return n + } + + maxSeq := -1 + for _, f := range files { + if n := seqOf(f); n > maxSeq { + maxSeq = n + } + } + next := maxSeq + 1 + + // Keep the live window; delete older segments no longer referenced. + threshold := next - windowSize + for _, f := range files { + if n := seqOf(f); n >= 0 && n < threshold { + if err := os.Remove(f); err != nil && !os.IsNotExist(err) { + return next, err + } + } + } + return next, nil +} + +// runValidateOnly validates the playlist and exits 0 (valid) or 1 (invalid/missing). +func runValidateOnly(playlistPath string) { + videos, err := ValidatePlaylist(playlistPath) + if err != nil { + fmt.Println("ERROR: Playlist validation failed:", err) + os.Exit(1) + } + valid := true + for _, v := range videos { + if v.Valid { + fmt.Printf("OK: %s (%.1fs)\n", v.Path, v.Duration) + } else { + valid = false + fmt.Printf("INVALID: %s — %s\n", v.Path, v.Validation) + } + } + if !valid { + os.Exit(1) + } + fmt.Println("Playlist valid.") } \ No newline at end of file diff --git a/backend/playout/src/playlist_test.go b/backend/playout/src/playlist_test.go index 0707719..8b5ccc8 100644 --- a/backend/playout/src/playlist_test.go +++ b/backend/playout/src/playlist_test.go @@ -27,6 +27,8 @@ package main // See backend/playout/testdata/*.txt. import ( + "context" + "fmt" "io" "net/http" "net/http/httptest" @@ -39,6 +41,7 @@ import ( "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" ) func ensureFFProbeInPath(t *testing.T) { @@ -265,7 +268,7 @@ func TestHealthHandler(t *testing.T) { `"status":"healthy"`, `"service":"playout"`, `"mode":"live"`, - `"restarts":3`, + `"crash_count":3`, `"run_id":"r-20260810T120000Z"`, `"last_segment_seq"`, `"consecutive_failures"`, @@ -588,4 +591,146 @@ func TestSyncToR2_NoCacheHeadersForUnknownExt(t *testing.T) { if gotContentType != "" { t.Fatalf("ContentType = %q, want empty", gotContentType) } +} + +func TestR2SegmentRetentionHours(t *testing.T) { + os.Setenv("R2_SEGMENT_RETENTION_HOURS", "") + if got := r2SegmentRetentionHours(); got != 24*time.Hour { + t.Errorf("default = %v, want 24h", got) + } + + os.Setenv("R2_SEGMENT_RETENTION_HOURS", "48") + if got := r2SegmentRetentionHours(); got != 48*time.Hour { + t.Errorf("override = %v, want 48h", got) + } + + os.Setenv("R2_SEGMENT_RETENTION_HOURS", "invalid") + if got := r2SegmentRetentionHours(); got != 24*time.Hour { + t.Errorf("invalid = %v, want 24h fallback", got) + } + + os.Unsetenv("R2_SEGMENT_RETENTION_HOURS") +} + +func TestCleanupOldSegments(t *testing.T) { + prevClient := r2Client + prevBucket := bucket + prevList := listObjects + prevDelete := deleteObjects + prevTotal := r2CleanupObjectsTotal + defer func() { + r2Client = prevClient + bucket = prevBucket + listObjects = prevList + deleteObjects = prevDelete + r2CleanupObjectsTotal = prevTotal + }() + + bucket = "linearvoid-media" + r2Client = &s3.Client{} + + now := time.Now().UTC() + old := now.Add(-25 * time.Hour) + recent := now.Add(-1 * time.Hour) + + listObjects = func(_ *s3.Client, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) { + return &s3.ListObjectsV2Output{ + Contents: []types.Object{ + {Key: aws.String("hls/r-old/seg-1.m4s"), LastModified: &old}, + {Key: aws.String("hls/r-old/init.mp4"), LastModified: &old}, + {Key: aws.String("hls/r-new/seg-2.m4s"), LastModified: &recent}, + {Key: aws.String("hls/playlist.m3u8"), LastModified: &old}, + {Key: aws.String("hls/playlist.m3u8.tmp"), LastModified: &old}, + }, + IsTruncated: aws.Bool(false), + }, nil + } + + var deletedKeys []string + deleteObjects = func(_ *s3.Client, input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) { + for _, obj := range input.Delete.Objects { + deletedKeys = append(deletedKeys, aws.ToString(obj.Key)) + } + return &s3.DeleteObjectsOutput{}, nil + } + + n, err := cleanupOldSegments(context.Background()) + if err != nil { + t.Fatalf("cleanupOldSegments error: %v", err) + } + if n != 2 { + t.Fatalf("deleted = %d, want 2", n) + } + + joined := strings.Join(deletedKeys, ",") + if !strings.Contains(joined, "hls/r-old/seg-1.m4s") || !strings.Contains(joined, "hls/r-old/init.mp4") { + t.Fatalf("unexpected deleted keys: %v", deletedKeys) + } + if strings.Contains(joined, "playlist.m3u8") { + t.Fatalf("manifest must not be deleted: %v", deletedKeys) + } + if r2CleanupObjectsTotal != 2 { + t.Fatalf("r2CleanupObjectsTotal = %d, want 2", r2CleanupObjectsTotal) + } +} + +func TestPlaylistFingerprint(t *testing.T) { + tmpDir := t.TempDir() + p := filepath.Join(tmpDir, "playlist.txt") + if err := os.WriteFile(p, []byte("file '/app/media/a.mp4'\n"), 0o644); err != nil { + t.Fatal(err) + } + + fp1, err := playlistFingerprint(p) + if err != nil { + t.Fatalf("playlistFingerprint error: %v", err) + } + fp2, err := playlistFingerprint(p) + if err != nil { + t.Fatalf("playlistFingerprint error: %v", err) + } + if fp1 != fp2 { + t.Errorf("fingerprint not stable: %q vs %q", fp1, fp2) + } + + if err := os.WriteFile(p, []byte("file '/app/media/b.mp4'\n"), 0o644); err != nil { + t.Fatal(err) + } + fp3, err := playlistFingerprint(p) + if err != nil { + t.Fatalf("playlistFingerprint error: %v", err) + } + if fp1 == fp3 { + t.Errorf("fingerprint should change when playlist content changes") + } +} + +func TestHLSNextStartNumber(t *testing.T) { + tmpDir := t.TempDir() + // Create segments 0..29 with a live window of 10. + for i := 0; i < 30; i++ { + if err := os.WriteFile(filepath.Join(tmpDir, fmt.Sprintf("seg-%06d.m4s", i)), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + + next, err := hlsNextStartNumber(tmpDir, 10) + if err != nil { + t.Fatalf("hlsNextStartNumber error: %v", err) + } + if next != 30 { + t.Errorf("next = %d, want 30", next) + } + + // Segments < 20 should be pruned; 20..29 kept (the live window). + for i := 0; i < 20; i++ { + if _, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("seg-%06d.m4s", i))); !os.IsNotExist(err) { + t.Errorf("seg-%06d should have been pruned", i) + } + } + for i := 20; i < 30; i++ { + if _, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("seg-%06d.m4s", i))); err != nil { + t.Errorf("seg-%06d should be kept: %v", i, err) + } + } } \ No newline at end of file diff --git a/backend/playout/start-playout.sh b/backend/playout/start-playout.sh index b9977a1..2092e46 100644 --- a/backend/playout/start-playout.sh +++ b/backend/playout/start-playout.sh @@ -19,36 +19,13 @@ fi # Run normalization validation using the Go tool echo "Validating playlist and enforcing normalization (ADR-001/ADR-003)..." -if ! /app/playlist-manager "$PLAYLIST_PATH"; then +if ! /app/playlist-manager -validate "$PLAYLIST_PATH"; then echo "ERROR: Playlist validation failed. Check logs." exit 1 fi echo "Playlist validated successfully. Starting continuous HLS playout..." -mkdir -p /app/hls -rm -f /app/hls/playlist.m3u8 /app/hls/init.mp4 /app/hls/segment_*.ts /app/hls/segment_*.m4s - -# Run FFmpeg in a loop for continuous playback -while true; do - echo "[$(date)] Starting FFmpeg playout with browser-safe HLS encoding..." - ffmpeg -re -f concat -safe 0 -i "$PLAYLIST_PATH" \ - -map 0:v:0 -map 0:a:0? \ - -vf "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2,fps=30,format=yuv420p" \ - -c:v libx264 -preset veryfast -pix_fmt yuv420p \ - -profile:v high -level 4.0 \ - -r 30 -g 60 -keyint_min 60 -sc_threshold 0 \ - -c:a aac -ar 48000 -ac 2 -b:a 128k \ - -f hls \ - -hls_time "${HLS_SEGMENT_DURATION}" \ - -hls_list_size "${HLS_WINDOW_SIZE}" \ - -hls_segment_type fmp4 \ - -hls_fmp4_init_filename init.mp4 \ - -strftime 1 \ - -hls_flags delete_segments+independent_segments \ - -hls_segment_filename "/app/hls/segment_%Y%m%d_%H%M%S.m4s" \ - "/app/hls/playlist.m3u8" || echo "FFmpeg exited with error (will restart in 5s per ADR-001 resilience requirements)..." - - echo "Playout loop crashed or ended. Restarting in 5 seconds..." - sleep 5 -done \ No newline at end of file +# Single source of truth: the Go binary runs the full loop (HLS, R2 sync, +# slate fallback, health server). The shell no longer duplicates the FFmpeg loop. +exec /app/playlist-manager "$PLAYLIST_PATH" \ No newline at end of file diff --git a/docs/SETUP.md b/docs/SETUP.md index bb5ec9a..4160fbb 100644 --- a/docs/SETUP.md +++ b/docs/SETUP.md @@ -1,5 +1,7 @@ # Linear Void Setup Guide +**Last Updated**: 2026-08-03 + This guide covers prerequisites, environment setup, running the application locally and in production, and troubleshooting. It references the [architecture documentation](./docs/architecture/README.md)(`docs/architecture/README.md`). ## Prerequisites @@ -7,7 +9,7 @@ This guide covers prerequisites, environment setup, running the application loca - **Docker & Docker Compose**: Required for running the full stack (playout, chat, nginx, redis, db, prometheus, grafana). Install from [docker.com](https://www.docker.com). - **Node.js 22+**: Required for frontend development and production builds (`cd frontend && npm install && npm run build`). - **npm**: Installed with Node.js and used for the frontend dependency installation and Vite workflow. -- **Go 1.23+**: Recommended when working on the Go services outside Docker (`backend/chat`, `backend/playout`). +- **Go 1.25+**: Recommended when working on the Go services outside Docker (`backend/chat`, `backend/playout`). - **FFmpeg**: Used by the playout service for HLS generation. Install via `apt install ffmpeg` (Linux), Homebrew (macOS), or Winget (Windows). - **Git**: For cloning and version control. - **Cloudflare Account**: For R2 storage and CDN (free tier sufficient for most use cases). @@ -81,70 +83,7 @@ flowchart LR cp .env.example .env ``` -3. **Configure `.env`** (edit with your values): - - ```env - # Cloudflare R2 (required for video storage and $0 egress) - R2_ACCESS_KEY_ID=your-r2-access-key - R2_SECRET_ACCESS_KEY=your-r2-secret-key - R2_BUCKET_NAME=linearvoid-media - R2_ENDPOINT=https://your-account-id.r2.cloudflarestorage.com - R2_REGION=auto - - # Cloudflare (for custom domain and cache rules) - CLOUDFLARE_API_TOKEN=your-cloudflare-api-token - CLOUDFLARE_ACCOUNT_ID=your-account-id - MEDIA_DOMAIN=media.linearvoid.tv - - # Public frontend URLs - # Preferred local setup uses the real public domains through Cloudflare and Nginx Proxy Manager. - VITE_PUBLIC_APP_URL=https://linearvoid.tv - VITE_PUBLIC_WS_URL=wss://linearvoid.tv/ws - VITE_PUBLIC_HLS_URL=https://media.linearvoid.tv/hls/playlist.m3u8 - VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR=admin - VITE_PUBLIC_ADMIN_DEFAULT_ROOM=main - VITE_PUBLIC_ADMIN_MODERATION_LOG_LIMIT=50 - ADMIN_USERNAME=admin - ADMIN_PASSWORD=change-me - ADMIN_SESSION_TTL_HOURS=12 - # Optional direct-local fallback when bypassing the public proxy: - # VITE_PUBLIC_APP_URL=http://localhost - # VITE_PUBLIC_WS_URL=ws://localhost/ws - # VITE_PUBLIC_HLS_URL=http://localhost/hls/playlist.m3u8 - - # Gitea Container Registry (for self-hosted Docker images) - CI_REGISTRY_URL=git.allucanget.biz - CI_REGISTRY_USER=your-gitea-username - CI_REGISTRY_TOKEN=your-gitea-token - CI_REGISTRY_NAMESPACE=zwitschi/linearvoid - # Gitea API access (optional, for CI/release automation) - CI_SERVER_URL=https://git.allucanget.biz - - # Chat / Database - DATABASE_URL=postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable - # or for SQLite-only local development: - # DATABASE_URL=sqlite://./data/chat.db - REDIS_URL=redis://redis:6379/0 - - # Ports (for local development) - PLAYOUT_PORT=8080 - CHAT_PORT=3000 - NGINX_PORT=80 - - # Playout Configuration - PLAYLIST_PATH=/app/playlist.txt - HLS_SEGMENT_DURATION=6 - HLS_WINDOW_SIZE=24 - - # Logging / Monitoring - LOG_LEVEL=info - PROMETHEUS_PORT=9090 - GRAFANA_PORT=3001 - - # Development flags - NODE_ENV=development - GO_ENV=development - ``` +3. **Configure `.env`** (copy from `.env.example` and replace placeholders) 4. **Check local value overrides for Docker Compose**: - `docker-compose.yml` builds images locally and maps the frontend build output from `frontend/dist` into nginx. @@ -227,7 +166,23 @@ flowchart LR - Enable Smart Tiered Cache for the R2-backed hostname. - If you need header normalization for immutable segment caching, deploy `infrastructure/cloudflare/worker.js` to the same zone. -7. **Verify access**: +7. **Configure R2 CORS**: + - Add CORS rules to allow the app domain to access the R2 bucket for HLS delivery. + - Example CORS configuration: + + ```json + [ + { + "AllowedOrigins": ["https://linearvoid.tv"], + "AllowedMethods": ["GET"], + "AllowedHeaders": ["*"], + "ExposeHeaders": ["ETag", "Cache-Control"], + "MaxAgeSeconds": 3600 + } + ] + ``` + +8. **Verify access**: - Confirm uploads with an S3-compatible client such as `aws s3 ls --endpoint-url `. - After deployment, check Cloudflare analytics for cache hit ratio and origin request volume. @@ -249,7 +204,8 @@ Execution tracker: `docs/operations/cloudflare-uptimerobot-checklist.md`. 1. Create bucket `linearvoid-media`. 2. Set location hint to `enam` (US-East). 3. Attach custom domain `media.linearvoid.tv` to the bucket. -4. Verify object read works from the custom domain. +4. Configure CORS rules to allow `https://linearvoid.tv` for HLS delivery. +5. Verify object read works from the custom domain. ### 3. Cache Rules @@ -588,5 +544,3 @@ See `docs/architecture/README.md` for the full arc42 documentation and the ADRs - Project Structure: Root `README.md`. The system is ready for use. For production, review ADR-001 for cost estimates and scaling guidance. - -**Last Updated**: 2026-08-03 diff --git a/frontend/src/admin.ts b/frontend/src/admin.ts index 531a5d2..4c289fa 100644 --- a/frontend/src/admin.ts +++ b/frontend/src/admin.ts @@ -480,6 +480,37 @@ async function deleteOverride(id: number) { } } +async function renderServiceRow(service: any) { + const svcDetail = service.detail || ""; + + // playout service returns json detail, all others return ISO formatted time string + if (service.name === "playout" && svcDetail) { + try { + // create a list of key-value pairs for each property in the JSON object + const detailObj = JSON.parse(svcDetail); + service.detail = Object.entries(detailObj) + .map(([key, value]) => `${key}: ${value}`) + .join("\n"); + } catch { + // ignore JSON parse errors + } + } + + const row = document.createElement("article"); + row.className = "service-row"; + row.innerHTML = ` +
+

${service.name}

+

${service.checked}

+
+
+

${service.status}

+

${service.detail}

+
+ `; + return row; +} + async function refreshSystemSummary() { setStatus("system-status", "Loading system summary..."); try { @@ -499,19 +530,9 @@ async function refreshSystemSummary() { servicesEl.innerHTML = ""; const services = (payload.services || []) as ServiceHealth[]; services.forEach((service) => { - const row = document.createElement("article"); - row.className = "service-row"; - row.innerHTML = ` -
-

${service.name}

-

${service.checked}

-
-
-

${service.status}

-

${service.detail}

-
- `; - servicesEl.appendChild(row); + renderServiceRow(service).then((row) => { + servicesEl.appendChild(row); + }); }); } @@ -534,6 +555,7 @@ function renderMediaRows(files: MediaEntry[]) { files.forEach((file) => { const tr = document.createElement("tr"); + const canNormalize = file.status === "needs_normalization" || file.status === "normalization_failed"; tr.innerHTML = ` ${file.name} ${file.title || "-"} @@ -543,10 +565,14 @@ function renderMediaRows(files: MediaEntry[]) { ${file.modifiedAt || "-"} + ${canNormalize ? `` : ""} `; tr.querySelector('[data-action="edit"]')?.addEventListener("click", () => editMedia(file)); + if (canNormalize) { + tr.querySelector('[data-action="normalize"]')?.addEventListener("click", () => normalizeMedia(file)); + } tr.querySelector('[data-action="delete"]')?.addEventListener("click", () => deleteMedia(file)); tbody.appendChild(tr); }); @@ -699,6 +725,55 @@ async function editMedia(file: MediaEntry) { } } +async function normalizeMedia(file: MediaEntry) { + if (!file.id) { + setStatus("media-status", "Cannot normalize media item without id.", true); + return; + } + + setStatus("media-status", `Normalizing ${file.name}...`); + try { + const response = await fetch(`/api/admin/media/${file.id}/normalize`, { method: "POST" }); + if (!response.ok) throw new Error(await readErrorMessage(response)); + + setStatus("media-status", `Normalization started for ${file.name}.`); + await refreshMediaLibrary(); + pollNormalizationStatus(file.id, 0); + } catch (error) { + console.error("Failed to normalize media", error); + const details = error instanceof Error ? error.message : "Unknown error."; + setStatus("media-status", `Failed to normalize ${file.name}. (${details})`, true); + } +} + +function pollNormalizationStatus(id: number, attempt: number) { + if (attempt >= 60) { + setStatus("media-status", "Normalization is taking longer than expected. Reload the library to check.", true); + return; + } + window.setTimeout(async () => { + try { + const response = await fetch(`/api/admin/media/${id}`); + if (!response.ok) return; + const file = (await response.json()) as MediaEntry; + if (file.status === "normalizing" || file.status === "needs_normalization") { + pollNormalizationStatus(id, attempt + 1); + return; + } + await refreshMediaLibrary(); + await refreshOverview(); + await refreshSchedule(); + setStatus( + "media-status", + file.status === "ready" ? `Normalization complete for ${file.name}.` : `Normalization failed for ${file.name}.`, + file.status !== "ready", + ); + } catch { + // Stop polling on error; the user can reload the library manually. + } + }, 3000); +} + async function deleteMedia(file: MediaEntry) { if (!file.id) { setStatus("media-status", "Cannot delete media item without id.", true); diff --git a/monitoring/grafana/dashboard.json b/monitoring/grafana/dashboard.json deleted file mode 100644 index f6e781f..0000000 --- a/monitoring/grafana/dashboard.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "title": "Linear Void Monitoring", - "panels": [ - { - "title": "Concurrent Viewers", - "type": "timeseries", - "targets": [{ "expr": "linearvoid_chat_connected_clients" }] - }, - { - "title": "Stored Messages", - "type": "stat", - "targets": [ - { - "expr": "linearvoid_chat_messages_total" - } - ] - }, - { - "title": "Moderation Actions", - "type": "timeseries", - "targets": [{ "expr": "linearvoid_chat_moderation_actions_total" }] - }, - { - "title": "Redis Reachable", - "type": "stat", - "targets": [{ "expr": "linearvoid_chat_redis_up" }] - }, - { - "title": "Admin Sessions", - "type": "timeseries", - "targets": [{ "expr": "linearvoid_chat_admin_sessions_active" }] - }, - { - "title": "Chat Scrape Health", - "type": "stat", - "targets": [{ "expr": "up{job=\"chat\"}" }] - }, - { - "title": "Cache Hit Ratio", - "type": "timeseries", - "targets": [ - { - "expr": "linearvoid_cache_hit_ratio" - } - ] - }, - { - "title": "Redis Memory Usage", - "type": "timeseries", - "targets": [ - { - "expr": "linearvoid_chat_redis_memory_usage_ratio" - } - ] - }, - { - "title": "Chat Request Latency p95", - "type": "timeseries", - "targets": [ - { - "expr": "histogram_quantile(0.95, sum(rate(linearvoid_chat_request_duration_seconds_bucket[5m])) by (le))" - } - ] - } - ] -}