feat: add automatic media normalization, R2 retention cleanup, and seamless playout loops
This commit is contained in:
1 parent
e15d89edfe
commit
06ab5b4655
16 files changed
+1103
-408
No files matched your search
@@ -8,6 +8,7 @@ R2_SECRET_ACCESS_KEY=your-r2-secret-key
|
||||
R2_BUCKET_NAME=linearvoid-media
|
||||
R2_ENDPOINT=https://your-account-id.r2.cloudflarestorage.com
|
||||
R2_REGION=auto
|
||||
R2_SEGMENT_RETENTION_HOURS=24
|
||||
|
||||
# Cloudflare (for custom domain, cache rules, Workers if used)
|
||||
CLOUDFLARE_API_TOKEN=your-cloudflare-api-token
|
||||
|
||||
@@ -1,34 +1,14 @@
|
||||
# BACKLOG
|
||||
|
||||
## Remaining from 2026-08-11 (summary)
|
||||
|
||||
- **Test Fixtures**: media + playlist fixtures created; ffmpeg generation commands documented in test headers
|
||||
- **User Frontend Tests**: video player, chat integration, responsive, accessibility, coverage
|
||||
- **Admin Frontend Tests**: playlist UI, moderation UI, metrics dashboards, coverage
|
||||
- **Cloudflare R2 Integration Test**: upload + cache header validation done (SyncToR2 with CacheControl + ContentType); edge cache hit/miss validation remains manual
|
||||
- **Cloudflare Config**: Cloudflare Pages deploy workflow and setup docs created; manual dashboard setup (Tiered Cache, WAF, rate limiting, HTTPS/HSTS) remains pending; Pages project must be created in dashboard first.
|
||||
- **External Services**: UptimeRobot monitors + alerts
|
||||
- **Production Deployment**: Hetzner provisioning, CD pipeline, DNS cutover, smoke test
|
||||
- **Scaling & Performance**: TTFB baseline and live benchmark runs still pending; loadtest scaffolding now includes Artillery/k6 helpers with auto summaries plus worksheet field mapping; chat p95, Redis saturation, and cache-hit threshold alerting are implemented
|
||||
> All remaining work is planned in [TODO](./TODO.md) (2026-08-13). This file records completed backlog items for reference; new unplanned work is added here.
|
||||
|
||||
## Cloudflare Configuration
|
||||
|
||||
### DNS & Pages
|
||||
|
||||
- [x] Configure DNS for linearvoid.tv apex and www redirect (manual)
|
||||
- [x] Configure media.linearvoid.tv for split-domain HLS delivery (manual)
|
||||
- [x] Configure Cloudflare Pages deploy workflow (`.gitea/workflows/pages-deploy.yml`) and setup docs (`docs/operations/cloudflare-pages-setup.md`, `docs/operations/gitea-cloudflare-pages.md`)
|
||||
- [x] Create Cloudflare Pages project in dashboard (unavailable, created `linearvoid` via API)
|
||||
- [ ] Cloudflare Pages: link custom domain (manual)
|
||||
- [ ] Create Cloudflare Pages project and cut over frontend hosting (manual)
|
||||
|
||||
### Storage & Cache
|
||||
|
||||
- [x] Create R2 bucket (manual)
|
||||
- [x] Configure Cache Rules: .ts/.m4s + .m3u8 (manual)
|
||||
- [x] Implement Worker for immutable cache headers on segment paths
|
||||
- [ ] CORS headers on HLS responses from media.linearvoid.tv (MEDIUM) — needs Cloudflare R2 CORS config
|
||||
|
||||
### Security & TLS
|
||||
|
||||
- [ ] Enable Tiered Cache, WAF, rate limiting (manual)
|
||||
@@ -40,13 +20,6 @@
|
||||
|
||||
## Monitoring & Observability
|
||||
|
||||
### Prometheus & Grafana
|
||||
|
||||
- [x] Set up Prometheus with chat/playout/nginx/redis/postgres/node scrape targets
|
||||
- [x] Add nginx-exporter, redis-exporter, postgres-exporter, node-exporter
|
||||
- [x] Import Grafana app + operational dashboards
|
||||
- [x] Define Prometheus alerting rules (8 rules: ChatDown, PlayoutDown, RedisDown, etc.)
|
||||
|
||||
### External Monitoring
|
||||
|
||||
- [ ] Set up UptimeRobot external monitoring
|
||||
@@ -54,44 +27,16 @@
|
||||
- [ ] UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword: `#EXTM3U`, 5min)
|
||||
- [ ] Alert on 2 consecutive failures, notify via email + Discord/Slack webhook
|
||||
|
||||
## Security
|
||||
|
||||
- [x] Implement admin cookie-session auth (HttpOnly, SameSite=Strict, Secure when TLS)
|
||||
- [x] Implement server-side session expiration
|
||||
- [x] Implement rate limiting for chat messages (Redis windowing)
|
||||
- [x] Implement WebSocket connection rate limiting (5/IP/60s)
|
||||
- [x] Implement admin login rate limiting (5/IP/5min)
|
||||
- [ ] Enforce HTTPS/WSS at edge and origin
|
||||
|
||||
## Test Fixtures
|
||||
|
||||
- [x] Create test media fixtures in `backend/playout/testdata/`: valid H.264/AAC, invalid codec, invalid framerate
|
||||
- [x] Create test playlist fixtures: `valid_playlist.txt`, `empty_playlist.txt`, `comments_only.txt`, `mixed_valid_invalid.txt`
|
||||
- [x] Document ffmpeg fixture-generation commands in test file headers
|
||||
|
||||
## Frontend Tests
|
||||
|
||||
### User Frontend
|
||||
|
||||
- [x] Video player behavior (play, pause, seek, buffering, error handling) — HLS player startup, native/fallback, URL resolution, error recovery with retries/backoff/reconnecting overlay
|
||||
- [x] Chat functionality (send/receive messages, presence, rate limiting, moderation) — send, receive, presence join/leave, DM flow, reconnect covered
|
||||
- [x] State management and data flow
|
||||
- [x] Responsive design and layout across different screen sizes — chat-collapsed toggle, CSS grid, now-playing aria
|
||||
- [x] Accessibility testing (keyboard navigation, screen reader support) — video controls, aria-labels, aria-expanded, input labels, reconnecting overlay aria-live/role
|
||||
- [x] Integration with backend APIs (mocking API responses for testing)
|
||||
- [ ] Test coverage for all critical frontend components and interactions
|
||||
- [ ] Add code coverage reporting and enforce minimum coverage threshold in CI/CD pipeline
|
||||
- [ ] Document test cases and expected outcomes
|
||||
|
||||
### Admin Frontend
|
||||
|
||||
- [x] Playlist management UI (add/remove videos, reorder, normalize) — save/error handling, sortable rendering
|
||||
- [ ] User moderation tools (mute/ban/report users, view moderation logs)
|
||||
- [x] Metrics and monitoring dashboards (Prometheus/Grafana integration) — overview + analytics rendering
|
||||
- [x] State management and data flow
|
||||
- [x] Responsive design and layout across different screen sizes — tab content visibility, essential tabs
|
||||
- [x] Accessibility testing (keyboard navigation, screen reader support) — nav aria-label, active tab aria-current, nav button focusability
|
||||
- [x] Integration with backend APIs (mocking API responses for testing)
|
||||
- [ ] Test coverage for all critical admin UI components and interactions
|
||||
- [ ] Add code coverage reporting and enforce minimum coverage threshold in CI/CD pipeline
|
||||
- [ ] Document test cases and expected outcomes
|
||||
@@ -100,29 +45,9 @@
|
||||
|
||||
### Cloudflare R2
|
||||
|
||||
- [x] Upload test HLS segments and manifests to R2 bucket (SyncToR2 with mock, verified bucket/key/body)
|
||||
- [x] Verify correct cache headers are applied (immutable segments via segmentCacheControl, short-lived manifests via manifestCacheControl)
|
||||
- [ ] Test retrieval of segments and manifests from R2 via Cloudflare edge
|
||||
- [ ] Validate cache hit/miss behavior and performance metrics
|
||||
- [ ] Validate edge cache hit/miss behavior with repeat requests — script `scripts/validate-edge-cache.sh` created
|
||||
- [x] Ensure integration test can run in CI/CD pipeline with proper mocking/stubbing of R2 API (putObject mock, no network calls)
|
||||
- [x] Document test cases and expected outcomes
|
||||
- [x] Document required stubs or mocks (putObject variable injection pattern)
|
||||
|
||||
## Streaming Pipeline
|
||||
|
||||
- [x] **Continuous R2 sync during playout (HIGH)**: fsnotify watcher uploads segments in real-time (~7s delay)
|
||||
- [x] **Globally unique segment naming (HIGH)**: `r-{ISOTIME}/seg-{seq}.m4s` via run ID + ffmpeg monotonic sequence
|
||||
- [x] **Atomic manifest publication (MEDIUM)**: Upload to `playlist.m3u8.tmp`, then `CopyObject` to final key
|
||||
- [x] **Manifest Cache-Control alignment (LOW)**: `public, max-age=1, s-maxage=2, must-revalidate`
|
||||
- [ ] **CORS headers on HLS responses (MEDIUM)**: Needs Cloudflare R2 bucket CORS config (infrastructure, not code)
|
||||
- [x] **Player error recovery (MEDIUM)**: `hls.on(Hls.Events.ERROR)` handler with bounded retries (5x), exponential backoff, reconnecting overlay
|
||||
- [x] **Health endpoint detail (LOW)**: Added active item, run ID, playlist revision, last segment seq, last R2 pub, consecutive failures
|
||||
- [x] Source media contract + validation (H.264/AAC, 30fps, keyframe check)
|
||||
- [x] fMP4 HLS packaging (6s segments, 24 window)
|
||||
- [x] Slate fallback + recovery (`shouldEscalateToSlate`)
|
||||
- [x] R2 cache headers (immutable segments, short-lived manifest)
|
||||
- [x] Cloudflare Worker immutable `.m4s` headers
|
||||
|
||||
## External Services
|
||||
|
||||
@@ -162,9 +87,6 @@
|
||||
|
||||
### Load Testing
|
||||
|
||||
- [x] Load-test scaffolding: `ws-chat.yml`, `admin-api.js`, `run-loadtests.ps1`, summarizers, README
|
||||
- [x] Cache-hit ratio exporter (`scripts/cache-hit-ratio.js` + `.ps1`)
|
||||
- [x] Saturation alerts: chat p95 latency, Redis memory usage, cache-hit ratio
|
||||
- [ ] Execute WebSocket load tests with real infrastructure — ramp 10→100→500→1000 connections, measure p95 latency + errors
|
||||
- [ ] Execute admin API benchmarks with real infrastructure — `wrk`/`k6` against `/api/admin/*`, target p95 <1s
|
||||
|
||||
|
||||
+169
-80
@@ -4,74 +4,74 @@ Completed tasks from `.github/instructions/TODO.md`. Historical record for CHANG
|
||||
|
||||
## Session 2026-08-07 — Unit Tests
|
||||
|
||||
- [x] Chat service refactor: `main.go` split into 8 packages (config, auth, db, handlers, ws, media, ratelimit, metrics)
|
||||
- [x] `db.RedisClient` + `db.PubSub` interfaces defined for mocking
|
||||
- [x] Config tests: 10 env-var tests, table-driven, 100% coverage
|
||||
- [x] Auth tests: token generation, session CRUD, cookie security, middleware, 97.8% coverage
|
||||
- [x] DB tests: SQLite in-memory CRUD for all 4 models, 14.3% coverage
|
||||
- [x] Handlers tests: 35 tests covering login/logout/session/overview/playlist/moderation/analytics/system/config/users/schedule/media/chat-log, 47.2% coverage
|
||||
- [x] WebSocket tests: 15 tests covering connect/send/dm/moderation/banned/muted/history/presence/pubsub, 79.6% coverage
|
||||
- [x] Media tests: formatDuration, extensions, mime, upload IDs, playlist parsing, 36.8% coverage
|
||||
- [x] Ratelimit tests: message/login/WS rate limits with mock Redis, 90.3% coverage
|
||||
- [x] Metrics tests: Prometheus output validation, all 9 metrics present, 100% coverage
|
||||
- [x] Playout tests: 12 tests (LoadPlaylist, keyframeInterval, HealthHandler, slate escalation, reset/activate/deactivate)
|
||||
- [x] Frontend test framework: vitest + jsdom installed
|
||||
- [x] User frontend smoke tests: 6 tests (chat toggle, sendMessage, switchRoom, DM toggle)
|
||||
- [x] Admin frontend smoke tests: 5 tests (DOM structure, upload progress, fingerprint, constants, import stability)
|
||||
- [x] CI pipeline updated: `go test -race -cover` with 50% threshold
|
||||
- [x] SQLite driver switched to pure-Go `github.com/glebarez/sqlite` (CGO-free)
|
||||
- [x] Total: 78 tests across Go + frontend, all passing
|
||||
- Chat service refactor: `main.go` split into 8 packages (config, auth, db, handlers, ws, media, ratelimit, metrics)
|
||||
- `db.RedisClient` + `db.PubSub` interfaces defined for mocking
|
||||
- Config tests: 10 env-var tests, table-driven, 100% coverage
|
||||
- Auth tests: token generation, session CRUD, cookie security, middleware, 97.8% coverage
|
||||
- DB tests: SQLite in-memory CRUD for all 4 models, 14.3% coverage
|
||||
- Handlers tests: 35 tests covering login/logout/session/overview/playlist/moderation/analytics/system/config/users/schedule/media/chat-log, 47.2% coverage
|
||||
- WebSocket tests: 15 tests covering connect/send/dm/moderation/banned/muted/history/presence/pubsub, 79.6% coverage
|
||||
- Media tests: formatDuration, extensions, mime, upload IDs, playlist parsing, 36.8% coverage
|
||||
- Ratelimit tests: message/login/WS rate limits with mock Redis, 90.3% coverage
|
||||
- Metrics tests: Prometheus output validation, all 9 metrics present, 100% coverage
|
||||
- Playout tests: 12 tests (LoadPlaylist, keyframeInterval, HealthHandler, slate escalation, reset/activate/deactivate)
|
||||
- Frontend test framework: vitest + jsdom installed
|
||||
- User frontend smoke tests: 6 tests (chat toggle, sendMessage, switchRoom, DM toggle)
|
||||
- Admin frontend smoke tests: 5 tests (DOM structure, upload progress, fingerprint, constants, import stability)
|
||||
- CI pipeline updated: `go test -race -cover` with 50% threshold
|
||||
- SQLite driver switched to pure-Go `github.com/glebarez/sqlite` (CGO-free)
|
||||
- Total: 78 tests across Go + frontend, all passing
|
||||
|
||||
## Session 2026-08-07 — Runtime Validation
|
||||
|
||||
- [x] Check `monitoring/prometheus/prometheus.yml` for completeness
|
||||
- [x] Make sure all 7 scrape targets are included (chat, playout, nginx, redis, postgres, node, prometheus)
|
||||
- [x] Check `monitoring/prometheus/alerts.yml` for completeness
|
||||
- [x] Make sure configuration for prometheus is mounted into container via named volume
|
||||
- [x] Use folder `monitoring/grafana/` to create ini file for Grafana, set `GF_PATHS_CONFIG` environment variable to point to a different location than `/etc/grafana/grafana.ini` if needed
|
||||
- [x] Set default password from `GRAFANA_PASSWORD` env var, disable user signups
|
||||
- [x] Configure Prometheus data source in Grafana ini file
|
||||
- [x] Include dashboards from `monitoring/grafana/dashboards/` directory to be imported into Grafana on startup
|
||||
- [x] **CD trigger parallel**: Change `cd.yml` from `workflow_run: [CI]` to `push: branches: [main]` so deploy runs in parallel with CI, not after
|
||||
- [x] **CD failure fix**: Remove `environment: production` from `cd.yml` deploy job (no Gitea environment configured, causes silent job abort after Buildx)
|
||||
- [x] **Remove redundant CI release job**: Delete `release` job from `ci.yml` — fully handled by `release.yml`
|
||||
- [x] **Tag format**: Use `*.*.*` tags (e.g. `1.0.9`) — `release.yml` pattern is `*.*.*`, not `v*.*.*`. Fix existing `v1.0.x` tags to `1.0.x`
|
||||
- [x] **Update DEPLOYMENT.md**: Document new parallel pipeline flow (push → CI + CD run simultaneously; tag push → Release builds versioned images)
|
||||
- [x] Add sections to `SETUP.md` under `## Cloudflare` for detailed instructions:
|
||||
- [x] DNS: `linearvoid.tv` apex + `www` CNAME → Cloudflare Pages
|
||||
- [x] DNS: `media.linearvoid.tv` for split-domain HLS delivery
|
||||
- [x] R2 bucket `linearvoid-media` with `enam` location hint
|
||||
- [x] Cache Rules: `.ts/.m4s` immutable long TTL (31536000), `.m3u8` short TTL (2s, revalidate)
|
||||
- [x] Cloudflare Pages: link Gitea repo, build `cd frontend && npm run build`, output `frontend/dist`
|
||||
- [x] Tiered Cache: enable Smart Tiered Cache, validate `Cf-Cache-Status: HIT` from upper tier
|
||||
- [x] WAF: enable OWASP Managed Ruleset, set to Simulate → review 48h → Block, enable Bot Fight Mode
|
||||
- [x] Rate limiting: `/api/admin/login` + `/api/admin/*` → 10 req/10s per IP, Challenge action
|
||||
- [x] HTTPS/WSS: Always Use HTTPS, HSTS max-age=31536000 includeSubDomains preload, min TLS 1.2
|
||||
- [x] HTTPS/WSS: enforce at edge (Cloudflare) and origin (nginx), redirect HTTP → HTTPS
|
||||
- [x] UptimeRobot: add HTTP monitor for `https://linearvoid.tv/health` (5min interval)
|
||||
- [x] UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword: `#EXTM3U`, 5min)
|
||||
- [x] Alert on 2 consecutive failures, notify via email + Discord/Slack webhook
|
||||
- [x] Check all known URLs for metrics and their availability:
|
||||
- [x] Prometheus: `http://<host>:9090` (targets, graph, alerts)
|
||||
- [x] Grafana: `http://<host>:3001` (default: admin/admin, change via `GRAFANA_PASSWORD`)
|
||||
- [x] Chat metrics: `http://chat:3000/metrics` from Prometheus container
|
||||
- [x] Playout metrics: `http://playout:8081/metrics` from Prometheus container
|
||||
- [x] Nginx metrics: `http://nginx-exporter:9113/metrics` from Prometheus container
|
||||
- [x] Redis metrics: `http://redis-exporter:9121/metrics` from Prometheus container
|
||||
- [x] Postgres metrics: `http://postgres-exporter:9187/metrics` from Prometheus container
|
||||
- [x] Node metrics: `http://node-exporter:9100/metrics` from Prometheus container
|
||||
- [x] Make sure Prometheus config is mounted into container via named volume `prometheus-config` (not bind mount)
|
||||
- [x] Make sure Prometheus alerting rules are configured and working (8 rules: ChatDown, PlayoutDown, RedisDown, etc.)
|
||||
- [x] Verify that all Prometheus targets are up and reporting metrics
|
||||
- [x] Verify that Grafana dashboards are imported and displaying metrics correctly
|
||||
- [x] Video normalization: test with MOV, MKV, variable framerate, different audio layouts
|
||||
- Check `monitoring/prometheus/prometheus.yml` for completeness
|
||||
- Make sure all 7 scrape targets are included (chat, playout, nginx, redis, postgres, node, prometheus)
|
||||
- Check `monitoring/prometheus/alerts.yml` for completeness
|
||||
- Make sure configuration for prometheus is mounted into container via named volume
|
||||
- Use folder `monitoring/grafana/` to create ini file for Grafana, set `GF_PATHS_CONFIG` environment variable to point to a different location than `/etc/grafana/grafana.ini` if needed
|
||||
- Set default password from `GRAFANA_PASSWORD` env var, disable user signups
|
||||
- Configure Prometheus data source in Grafana ini file
|
||||
- Include dashboards from `monitoring/grafana/dashboards/` directory to be imported into Grafana on startup
|
||||
- **CD trigger parallel**: Change `cd.yml` from `workflow_run: [CI]` to `push: branches: [main]` so deploy runs in parallel with CI, not after
|
||||
- **CD failure fix**: Remove `environment: production` from `cd.yml` deploy job (no Gitea environment configured, causes silent job abort after Buildx)
|
||||
- **Remove redundant CI release job**: Delete `release` job from `ci.yml` — fully handled by `release.yml`
|
||||
- **Tag format**: Use `*.*.*` tags (e.g. `1.0.9`) — `release.yml` pattern is `*.*.*`, not `v*.*.*`. Fix existing `v1.0.x` tags to `1.0.x`
|
||||
- **Update DEPLOYMENT.md**: Document new parallel pipeline flow (push → CI + CD run simultaneously; tag push → Release builds versioned images)
|
||||
- Add sections to `SETUP.md` under `## Cloudflare` for detailed instructions:
|
||||
- DNS: `linearvoid.tv` apex + `www` CNAME → Cloudflare Pages
|
||||
- DNS: `media.linearvoid.tv` for split-domain HLS delivery
|
||||
- R2 bucket `linearvoid-media` with `enam` location hint
|
||||
- Cache Rules: `.ts/.m4s` immutable long TTL (31536000), `.m3u8` short TTL (2s, revalidate)
|
||||
- Cloudflare Pages: link Gitea repo, build `cd frontend && npm run build`, output `frontend/dist`
|
||||
- Tiered Cache: enable Smart Tiered Cache, validate `Cf-Cache-Status: HIT` from upper tier
|
||||
- WAF: enable OWASP Managed Ruleset, set to Simulate → review 48h → Block, enable Bot Fight Mode
|
||||
- Rate limiting: `/api/admin/login` + `/api/admin/*` → 10 req/10s per IP, Challenge action
|
||||
- HTTPS/WSS: Always Use HTTPS, HSTS max-age=31536000 includeSubDomains preload, min TLS 1.2
|
||||
- HTTPS/WSS: enforce at edge (Cloudflare) and origin (nginx), redirect HTTP → HTTPS
|
||||
- UptimeRobot: add HTTP monitor for `https://linearvoid.tv/health` (5min interval)
|
||||
- UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword: `#EXTM3U`, 5min)
|
||||
- Alert on 2 consecutive failures, notify via email + Discord/Slack webhook
|
||||
- Check all known URLs for metrics and their availability:
|
||||
- Prometheus: `http://<host>:9090` (targets, graph, alerts)
|
||||
- Grafana: `http://<host>:3001` (default: admin/admin, change via `GRAFANA_PASSWORD`)
|
||||
- Chat metrics: `http://chat:3000/metrics` from Prometheus container
|
||||
- Playout metrics: `http://playout:8081/metrics` from Prometheus container
|
||||
- Nginx metrics: `http://nginx-exporter:9113/metrics` from Prometheus container
|
||||
- Redis metrics: `http://redis-exporter:9121/metrics` from Prometheus container
|
||||
- Postgres metrics: `http://postgres-exporter:9187/metrics` from Prometheus container
|
||||
- Node metrics: `http://node-exporter:9100/metrics` from Prometheus container
|
||||
- Make sure Prometheus config is mounted into container via named volume `prometheus-config` (not bind mount)
|
||||
- Make sure Prometheus alerting rules are configured and working (8 rules: ChatDown, PlayoutDown, RedisDown, etc.)
|
||||
- Verify that all Prometheus targets are up and reporting metrics
|
||||
- Verify that Grafana dashboards are imported and displaying metrics correctly
|
||||
- Video normalization: test with MOV, MKV, variable framerate, different audio layouts
|
||||
- Evidence: all normalized outputs probed as `h264`, `yuv420p`, `30/1`, `aac`, `48000`, `2ch`.
|
||||
- [x] Video normalization: verify output plays back-to-back in continuous concat
|
||||
- Video normalization: verify output plays back-to-back in continuous concat
|
||||
- Evidence: concat remux reported `CONCAT_OK`; output duration `25.344667s`.
|
||||
- [x] Viewer SPA: test responsive layout on iOS Safari + Android Chrome
|
||||
- Viewer SPA: test responsive layout on iOS Safari + Android Chrome
|
||||
- Evidence: viewport emulation (`390x844`, `412x915`) showed video/chat/toggle present; fixed 8px horizontal overflow in `frontend/src/style.css` (`100vw` -> `100%`).
|
||||
- Limitation: no physical-device Safari/Chrome test in this run.
|
||||
- [x] Redis: test restart, verify sessions + presence data survive
|
||||
- Redis: test restart, verify sessions + presence data survive
|
||||
- Evidence: admin session HTTP `200` before/after Redis restart; seeded presence key visible before/after restart; Redis health returned to `healthy`.
|
||||
|
||||
### Runtime Tests (all 4)
|
||||
@@ -84,15 +84,15 @@ Completed tasks from `.github/instructions/TODO.md`. Historical record for CHANG
|
||||
|
||||
### Documentation Polish
|
||||
|
||||
- [x] API.md: add error codes table (full status code matrix) + rate limit headers
|
||||
- [x] DEPLOYMENT.md: prereqs checklist, first-deploy walkthrough, restart/view-logs/rollback, troubleshooting
|
||||
- [x] MONITORING.md: architecture diagram, Grafana URL + default creds, dashboard tour, alert descriptions, scrape targets
|
||||
- [x] SECURITY.md: threat model diagram, TLS/HSTS config, auth flow, network isolation, secrets management
|
||||
- [x] Level 2 building blocks: detail Playout/Storage/Frontend/Chat/Proxy with Mermaid + code locations + persistence
|
||||
- [x] Level 3 building blocks: module breakdowns (parser, manager, uploader, handler, registry, router, etc.)
|
||||
- [x] Recovery runbook: document quarterly disaster recovery drill
|
||||
- API.md: add error codes table (full status code matrix) + rate limit headers
|
||||
- DEPLOYMENT.md: prereqs checklist, first-deploy walkthrough, restart/view-logs/rollback, troubleshooting
|
||||
- MONITORING.md: architecture diagram, Grafana URL + default creds, dashboard tour, alert descriptions, scrape targets
|
||||
- SECURITY.md: threat model diagram, TLS/HSTS config, auth flow, network isolation, secrets management
|
||||
- Level 2 building blocks: detail Playout/Storage/Frontend/Chat/Proxy with Mermaid + code locations + persistence
|
||||
- Level 3 building blocks: module breakdowns (parser, manager, uploader, handler, registry, router, etc.)
|
||||
- Recovery runbook: document quarterly disaster recovery drill
|
||||
- Evidence: expanded `docs/incident-runbook.md` with prerequisites, simulation steps, validation checks, and pass/fail criteria.
|
||||
- [x] Recovery runbook: rehearse quarterly disaster recovery drill
|
||||
- Recovery runbook: rehearse quarterly disaster recovery drill
|
||||
- Manual step: execute full drill on target environment and capture RTO/RPO evidence.
|
||||
|
||||
## Session 2026-08-06 — Documentation Adjustments, Playout, Normalization, Chat, Viewer SPA, Admin, Docker, CI/CD
|
||||
@@ -975,12 +975,12 @@ Notes:
|
||||
|
||||
Completed 6 of 7 streaming pipeline gaps from STREAMING.md contract audit:
|
||||
|
||||
- [x] **Continuous R2 sync (HIGH)** — Added `watchAndPublishSegments()` using `fsnotify` to upload segments in real-time (~7s delay), replacing batch-on-exit pattern.
|
||||
- [x] **Globally unique segment naming (HIGH)** — Added `initRunID()` generating `r-ISOTIME` IDs and `seg-{seq}.m4s` monotonic naming. Replaced `strftime` collision-prone pattern.
|
||||
- [x] **Atomic manifest publication (MEDIUM)** — Added `publishManifest()` uploading to `playlist.m3u8.tmp` then `CopyObject` to final key.
|
||||
- [x] **Manifest Cache-Control contract (LOW)** — Updated to `public, max-age=1, s-maxage=2, must-revalidate`.
|
||||
- [x] **Player error recovery (MEDIUM)** — Added `hls.on(ERROR)` handler with bounded retries (5x), exponential backoff (2s→32s), reconnecting overlay with spinner.
|
||||
- [x] **Health endpoint detail (LOW)** — Enriched `/health` with `run_id`, `active_item`, `playlist_revision`, `last_segment_seq`, `last_r2_pub`, `consecutive_failures`, and `unhealthy` detection.
|
||||
- **Continuous R2 sync (HIGH)** — Added `watchAndPublishSegments()` using `fsnotify` to upload segments in real-time (~7s delay), replacing batch-on-exit pattern.
|
||||
- **Globally unique segment naming (HIGH)** — Added `initRunID()` generating `r-ISOTIME` IDs and `seg-{seq}.m4s` monotonic naming. Replaced `strftime` collision-prone pattern.
|
||||
- **Atomic manifest publication (MEDIUM)** — Added `publishManifest()` uploading to `playlist.m3u8.tmp` then `CopyObject` to final key.
|
||||
- **Manifest Cache-Control contract (LOW)** — Updated to `public, max-age=1, s-maxage=2, must-revalidate`.
|
||||
- **Player error recovery (MEDIUM)** — Added `hls.on(ERROR)` handler with bounded retries (5x), exponential backoff (2s→32s), reconnecting overlay with spinner.
|
||||
- **Health endpoint detail (LOW)** — Enriched `/health` with `run_id`, `active_item`, `playlist_revision`, `last_segment_seq`, `last_r2_pub`, `consecutive_failures`, and `unhealthy` detection.
|
||||
- [ ] CORS on `media.linearvoid.tv` — Remaining infrastructure-only (Cloudflare R2 CORS configuration, not code).
|
||||
|
||||
**Files changed:** `backend/playout/src/playlist.go` (+171/-35), `playlist_test.go` (+51/-7), `go.mod` (+2), `frontend/src/main.ts` (+76/-2), `main.test.ts` (+43/-1), `style.css` (+37), `index.html` (+5).
|
||||
@@ -990,11 +990,100 @@ Completed 6 of 7 streaming pipeline gaps from STREAMING.md contract audit:
|
||||
|
||||
Completed Phase 2 (viewer) and Phase 3 (admin) responsive/accessibility tests + coverage:
|
||||
|
||||
- [x] **Viewer accessibility (5 tests):** video controls/autoplay/playsinline attributes, chat panel aria-labels (live region + panel label), chat toggle aria-expanded state, message input label association, reconnecting overlay aria-live=assertive + role=alert.
|
||||
- [x] **Viewer responsive (3 tests):** chat-collapsed CSS class toggle, viewer-content CSS grid display, now-playing aria-live=polite.
|
||||
- [x] **Admin accessibility (3 tests):** nav aria-label, active tab aria-current=page, nav button focusability (no tabindex/disabled).
|
||||
- [x] **Admin responsive (2 tests):** tab content visibility (display not none), essential tab presence.
|
||||
- [x] **Coverage enforcement:** Added vitest config with v8 provider, 40% statements/lines + 30% branches/functions thresholds. Added `test:coverage` script and CI step.
|
||||
- **Viewer accessibility (5 tests):** video controls/autoplay/playsinline attributes, chat panel aria-labels (live region + panel label), chat toggle aria-expanded state, message input label association, reconnecting overlay aria-live=assertive + role=alert.
|
||||
- **Viewer responsive (3 tests):** chat-collapsed CSS class toggle, viewer-content CSS grid display, now-playing aria-live=polite.
|
||||
- **Admin accessibility (3 tests):** nav aria-label, active tab aria-current=page, nav button focusability (no tabindex/disabled).
|
||||
- **Admin responsive (2 tests):** tab content visibility (display not none), essential tab presence.
|
||||
- **Coverage enforcement:** Added vitest config with v8 provider, 40% statements/lines + 30% branches/functions thresholds. Added `test:coverage` script and CI step.
|
||||
|
||||
**Files:** `frontend/src/main.test.ts` (+65), `frontend/src/admin.test.ts` (+55), `frontend/vitest.config.ts` (+12), `frontend/package.json` (+1), `.gitea/workflows/ci.yml` (+3).
|
||||
**Total:** 44 frontend tests (31 existing + 13 new).
|
||||
|
||||
## Session 2026-08-13 — Plan consolidation (TODO → DONE)
|
||||
|
||||
Completed tasks moved out of `.github/instructions/TODO.md`. Remaining work replanned in TODO from `BACKLOG.md`.
|
||||
|
||||
### Phase 1 — playout test fixtures + source-media contract
|
||||
|
||||
- Create small compliant fixture media in `backend/playout/testdata/` for the validator and playout tests
|
||||
- Create invalid fixture media for codec, framerate, and audio rejection paths
|
||||
- Create playlist fixtures for empty, comment-only, mixed-validity, and valid playlists
|
||||
- Document the ffmpeg fixture-generation commands in the relevant test headers
|
||||
|
||||
### Phase 2 — viewer frontend coverage
|
||||
|
||||
- HLS player startup, manifest resolution, and cross-origin/local fallback tests
|
||||
- Chat send/receive, presence updates, private DM flow, and reconnect tests
|
||||
- Responsive-layout and accessibility checks for the viewer shell and chat sidebar
|
||||
- Coverage reporting added and minimum threshold enforced in CI
|
||||
|
||||
### Phase 3 — admin frontend coverage
|
||||
|
||||
- Playlist management, moderation flows, and dashboard rendering tests
|
||||
- API integration points with mocked backend responses
|
||||
- Responsive-layout and accessibility checks for admin screens
|
||||
- Coverage reporting added and minimum threshold enforced in CI
|
||||
|
||||
### Phase 4 — R2 and edge-delivery validation
|
||||
|
||||
- Integration test for upload and retrieval of HLS objects through the R2 path
|
||||
- Segment and manifest cache headers verified (immutable media, live manifest freshness)
|
||||
- Edge cache hit/miss validation scripted (`scripts/validate-edge-cache.sh`)
|
||||
- Stubs/mocks documented (putObject variable-injection pattern)
|
||||
|
||||
### Phase 7 (partial) — load-test scaffolding + saturation alerts
|
||||
|
||||
- WebSocket load-test scaffolding and summary artifacts (`ws-chat.yml`, `run-loadtests.ps1`, `summarize-artillery.ps1`)
|
||||
- Admin API benchmark scaffolding and endpoint summary artifacts (`admin-api.js`, `run-loadtests.ps1`, `summarize-k6.ps1`)
|
||||
- Saturation thresholds added to monitoring (chat p95 latency + Redis memory usage)
|
||||
- Cache-hit threshold alert operationalization added to monitoring
|
||||
|
||||
### Streaming pipeline gaps (audit → implemented)
|
||||
|
||||
- Continuous R2 sync during playout (fsnotify watcher, ~7s delay)
|
||||
- Globally unique segment naming with run ID (`r-{ISOTIME}/seg-{seq}.m4s`)
|
||||
- Atomic manifest publication (upload to `.tmp`, then CopyObject)
|
||||
- Manifest Cache-Control: `public, max-age=1, s-maxage=2, must-revalidate`
|
||||
- Player error recovery (bounded retries, exponential backoff, reconnecting overlay)
|
||||
- Health endpoint detail (run ID, active item, playlist revision, segment seq, R2 pub, consecutive failures)
|
||||
- CORS headers — documented as Cloudflare R2 infrastructure config
|
||||
- Phase 4 edge cache validation — `scripts/validate-edge-cache.sh` created
|
||||
|
||||
## Session 2026-08-13 — Playout Enhancements
|
||||
|
||||
Completed the three Playout enhancement workstreams from `TODO.md`. Remaining polish items kept in TODO.
|
||||
|
||||
### 1. Automatic normalization of source media on upload (H.264/AAC, 30fps, keyframe check)
|
||||
|
||||
- Implement `media.NormalizeVideo` in `backend/chat/src/media/media.go` (ffmpeg exec, Linear Void profile, bounded 2h timeout, stderr capture)
|
||||
- Write normalized output to `*.normalized.mp4` and re-verify with ffprobe (`ValidateNormalization` + `ProbeDurationSeconds`)
|
||||
- Extend media status states: `normalizing`, `normalization_failed` (alongside `ready` / `needs_normalization`)
|
||||
- Integrate into single-upload path — `HandleAdminMedia`: persist `needs_normalization` then `go normalizeMediaRecord(...)`
|
||||
- Integrate into chunked-upload completion — `HandleAdminMediaUploads` `"complete"`: same async flow after assembly + validation
|
||||
- Add `POST /api/admin/media/{id}/normalize` (behind `requireAdminAuth`, returns 202 + `normalizing` record)
|
||||
- Admin UI (`admin.ts`): "Normalize" action + status polling until `ready`/`normalization_failed`
|
||||
- Tests: `NormalizeVideo` args + ffmpeg-error path; handler `normalizeMediaRecord` success/failure; `/normalize` endpoint 202
|
||||
- Retire hardcoded `backend/playout/normalize-video.sh` (superseded by in-Go normalization) - file deleted
|
||||
|
||||
### 2. R2 data retention: delete old segments after 24h
|
||||
|
||||
- Add `cleanupOldSegments(ctx)` to `backend/playout/src/playlist.go`
|
||||
- Use `s3.ListObjectsV2` on prefix `hls/` to enumerate run directories
|
||||
- Delete `hls/r-*/*` older than `now - R2_SEGMENT_RETENTION_HOURS` (default 24)
|
||||
- Never delete `hls/playlist.m3u8` or `hls/playlist.m3u8.tmp`
|
||||
- Run cleanup as a ticker goroutine (hourly) plus once at startup
|
||||
- Add `R2_SEGMENT_RETENTION_HOURS` (default `24`) to `.env.example`
|
||||
- Expose `linearvoid_playout_r2_cleanup_objects_total` metric and log deleted keys
|
||||
- Tests: mock `ListObjectsV2`/`DeleteObjects` via injectable vars (mirror the existing `putObject` pattern)
|
||||
|
||||
### 3. Continuous playout: confirm ffmpeg loops without interruption
|
||||
|
||||
- Resolve the dual-loop ambiguity between `main()` and `start-playout.sh`
|
||||
- Add a `-validate` mode to `main()`: run `ValidatePlaylist` and exit 0/1 without the ffmpeg loop
|
||||
- Update `start-playout.sh` to invoke `-validate` and `exec` the Go binary (single source of truth)
|
||||
- Minimize the restart gap: skip full `resetHLSOutput` when the playlist is unchanged
|
||||
- Track `playlistRevision` (SHA-256 fingerprint); reset only when the playlist changed, on crash/slate recovery, or first run
|
||||
- Continue segment numbering via `-start_number` + prune stale segments (`hlsNextStartNumber`)
|
||||
- Verify `init.mp4` + manifest are correctly published on clean completion (existing `publishManifest` path runs after every run)
|
||||
- Add metrics `linearvoid_playout_loops_total` (clean completions) and `linearvoid_playout_run_duration_seconds`
|
||||
- Tests: `TestPlaylistFingerprint`, `TestHLSNextStartNumber`
|
||||
@@ -1,80 +1,90 @@
|
||||
# TODO
|
||||
|
||||
> Completed tasks moved to [DONE](./DONE.md). Full remaining task list in [BACKLOG](./BACKLOG.md).
|
||||
> Completed tasks moved to [DONE](./DONE.md). Remaining tasks in [BACKLOG](./BACKLOG.md).
|
||||
|
||||
## Plan
|
||||
## Enhancements
|
||||
|
||||
### Phase 1: unblock playout test fixtures and lock down the source-media contract
|
||||
### Application configuration
|
||||
|
||||
- [x] Create small compliant fixture media in `backend/playout/testdata/` for the validator and playout tests
|
||||
- [x] Create invalid fixture media for codec, framerate, and audio rejection paths
|
||||
- [x] Create playlist fixtures for empty, comment-only, mixed-validity, and valid playlists
|
||||
- [x] Document the ffmpeg fixture-generation commands in the relevant test headers
|
||||
- [ ] Configuration: create simple getter/setter for configuration values (e.g. `MATOMO_URL`, `MATOMO_SITE_ID`) in `backend/admin/config.go`
|
||||
- [ ] Configuration: expose configuration values to frontend via `/api/admin/config` endpoint
|
||||
- [ ] Configuration: make configuration values editable:
|
||||
- [ ] HLS_SEGMENT_DURATION
|
||||
- [ ] HLS_WINDOW_SIZE
|
||||
- [ ] MATOMO_URL
|
||||
- [ ] MATOMO_SITE_ID
|
||||
|
||||
### Phase 2: finish frontend coverage for the live viewer experience
|
||||
### Admin frontend
|
||||
|
||||
- [x] Add tests for HLS player startup, manifest resolution, and cross-origin/local fallback behavior
|
||||
- [x] Add tests for chat send/receive, presence updates, private DM flow, and reconnect behavior
|
||||
- [x] Add responsive-layout and accessibility checks for the viewer shell and chat sidebar
|
||||
- [x] Add coverage reporting and enforce a minimum threshold in CI
|
||||
- [ ] Playlist: show `needs_normalization` badge and block/queue playlist-add until `ready`
|
||||
- [ ] Playlist: show `ready` badge and allow playlist-add when `ready`
|
||||
- [ ] Extend `system.html` with a section for configuration settings on top of page (HLS segment duration, window size, Matomo URL/site ID)
|
||||
- [ ] Add functions to `admin.ts` to call `/api/admin/config` for getting and updating configuration values
|
||||
|
||||
### Phase 3: finish frontend coverage for the admin experience
|
||||
### User frontend
|
||||
|
||||
- [x] Add tests for playlist management, moderation flows, and dashboard rendering
|
||||
- [x] Add tests for API integration points with mocked backend responses
|
||||
- [x] Add responsive-layout and accessibility checks for admin screens
|
||||
- [x] Add coverage reporting and enforce a minimum threshold in CI
|
||||
- [ ] Player frontend: Embed JavaScript snippet in `index.html` to send pageview events to Matomo
|
||||
|
||||
### Phase 4: validate the R2 and edge-delivery path
|
||||
### Chat user features
|
||||
|
||||
- [x] Add an integration test for upload and retrieval of HLS objects through the R2 path
|
||||
- [x] Verify segment and manifest cache headers, including immutable media and live manifest freshness
|
||||
- [x] Validate edge cache hit/miss behavior with repeat requests
|
||||
- [x] Document any required stubs or mocks so the test can run in CI (putObject mock pattern documented in test)
|
||||
- [ ] Chat: add user registration and login flows
|
||||
- [ ] Chat: private DM history and search
|
||||
- [ ] Chat UI: add login/logout button, show username in chat, show/toggle user list, show/toggle online status, show typing indicator
|
||||
|
||||
### Monitoring
|
||||
|
||||
- [ ] Prometheus TSDB retention policy: delete metrics older than 30d
|
||||
|
||||
## Setup phases remaining
|
||||
|
||||
### Phase 5: complete manual Cloudflare and external-service setup
|
||||
|
||||
- [ ] Apply Cloudflare Pages, Tiered Cache, WAF, rate limiting, and HTTPS/HSTS settings in the dashboard
|
||||
- [ ] Create or verify UptimeRobot monitors for health and HLS manifest availability
|
||||
- [ ] Cloudflare Pages: link custom domain and cut over frontend hosting
|
||||
- [ ] Enable Smart Tiered Cache; validate `Cf-Cache-Status: HIT` from upper tier
|
||||
- [ ] WAF: enable OWASP Managed Ruleset (Simulate → review 48h → Block), enable Bot Fight Mode
|
||||
- [ ] Rate limiting: `/api/admin/login` + `/api/admin/*` → 10 req/10s per IP, Challenge action
|
||||
- [ ] HTTPS/WSS: Always Use HTTPS, HSTS max-age=31536000 includeSubDomains preload, min TLS 1.2; enforce at edge and origin
|
||||
- [ ] UptimeRobot: add HTTP monitor for `https://linearvoid.tv/health` (5min interval)
|
||||
- [ ] UptimeRobot: add keyword monitor for `https://linearvoid.tv/hls/playlist.m3u8` (keyword `#EXTM3U`, 5min)
|
||||
- [ ] UptimeRobot: alert on 2 consecutive failures via email + Discord/Slack webhook
|
||||
- [ ] Capture evidence in `docs/operations/cloudflare-uptimerobot-checklist.md`
|
||||
|
||||
### Phase 6: rehearse production deployment end to end
|
||||
|
||||
- [ ] Provision the Hetzner host and prepare the Docker/runtime baseline
|
||||
- [ ] Configure production secrets, remote deployment, and DNS cutover
|
||||
- [ ] Run health, HLS, chat, and admin smoke tests against production-like infrastructure
|
||||
- [ ] Verify backup creation, restore, and integrity checks on the production host
|
||||
- [ ] Provision Hetzner CX22 (2 vCPU, 4 GB, 40 GB) in Ashburn VA, Ubuntu 24.04
|
||||
- [ ] Install Docker Engine + Compose plugin; open firewall: 80/443 public, 2375 runner-only
|
||||
- [ ] Configure Docker daemon for remote TLS access (`-H tcp://0.0.0.0:2376` with certs)
|
||||
- [ ] Create `/srv/linearvoid/data/{playlist,media,hls,chat,slate}` directories
|
||||
- [ ] Add `CI_PROD_*` secrets to Gitea (DOCKER_HOST, R2 keys, admin password, DB password, Grafana password)
|
||||
- [ ] Trigger CD workflow with `DOCKER_HOST` pointing to Hetzner IP
|
||||
- [ ] Verify all 11 containers healthy: `/health`, `/hls/playlist.m3u8`, WebSocket
|
||||
- [ ] Update Cloudflare A record `linearvoid.tv` → Hetzner IP, SSL Full (strict); test HTTPS + WSS
|
||||
- [ ] Smoke test: viewer SPA loads, HLS plays, chat connects, admin login → upload → playlist → playout
|
||||
- [ ] Set up Prometheus alerting notifications + UptimeRobot on production URL
|
||||
- [ ] Run `scripts/backup-db.sh` + `scripts/verify-backups.sh` on production host
|
||||
- [ ] Document production IP, SSH access, rollback procedure, scaling thresholds
|
||||
|
||||
### Phase 7: baseline scaling and performance
|
||||
|
||||
- [ ] Measure manifest TTFB and cache-hit distribution from multiple US regions
|
||||
- [x] Prepare WebSocket load-test scaffolding and summary artifacts (`ws-chat.yml`, `run-loadtests.ps1`, `summarize-artillery.ps1`)
|
||||
- [ ] Execute WebSocket throughput and chat-latency benchmarks under growing connection counts
|
||||
- [x] Prepare Admin API benchmark scaffolding and endpoint summary artifacts (`admin-api.js`, `run-loadtests.ps1`, `summarize-k6.ps1`)
|
||||
- [ ] Execute Admin API benchmarks and identify slow paths
|
||||
- [x] Add saturation thresholds to monitoring (chat p95 latency + Redis memory usage)
|
||||
- [x] Add cache-hit threshold alert operationalization to monitoring
|
||||
- [ ] Execute WebSocket load tests — ramp 10→100→500→1000 connections; measure p95 latency + errors
|
||||
- [ ] Execute admin API benchmarks — `wrk`/`k6` against `/api/admin/*`; target p95 <1s
|
||||
- [ ] Measure manifest TTFB and cache-hit distribution from multiple US regions (WebPageTest 3+ regions, <4s warm cache)
|
||||
- [ ] Tune manifest TTL to 0.5× segment duration; validate `Cf-Cache-Status` distribution
|
||||
- [ ] Track cache-hit ratio and R2 ops in production (Cloudflare GraphQL → Grafana panel, alert <90%)
|
||||
- [ ] Capacity triggers: Prometheus rules for cache-miss >10%/15min, CPU >80%/10min
|
||||
- [ ] Plan Phase 2 service separation (playout CPU, chat latency, DB state, monitoring)
|
||||
- [ ] Plan Phase 3 multi-origin resilience (second Hetzner location, warm standby, DNS failover, DB/Redis replication)
|
||||
|
||||
## Status Update (2026-08-10)
|
||||
### Phase 8: close out remaining test and validation gaps
|
||||
|
||||
- [x] Phase 1 (test fixtures) complete.
|
||||
- [x] Phase 2 (viewer frontend): player startup, URL resolution, chat send/receive, reconnect, presence, DM flow, message rendering all covered (27 tests).
|
||||
- [x] Phase 3 (admin frontend): dashboard rendering, API integration, playlist management all covered (31 tests total).
|
||||
- [x] Phase 4 (R2 validation) partially complete: SyncToR2 cache headers + integration tests done; edge cache hit/miss remains manual.
|
||||
- [ ] Phase 5 (Cloudflare): Pages deploy workflow (`.gitea/workflows/pages-deploy.yml`) created; Pages/Workers setup docs written; manual dashboard tasks remain.
|
||||
- [ ] Phase 6 (production deployment) pending: Hetzner provisioning, secrets, DNS cutover, smoke tests, backup/restore.
|
||||
- [ ] Phase 7: Loadtest scaffolding complete; `run-loadtests.ps1` auto-generates ws-chat-summary.txt and admin-api-endpoint-summary.txt; live benchmark execution pending.
|
||||
- Version tagged 1.0.11; go vet clean.
|
||||
- [ ] Admin frontend: user moderation tools tests (mute/ban/report, view moderation logs)
|
||||
- [ ] Frontend: coverage for all critical user + admin components and interactions
|
||||
- [ ] Frontend: document test cases and expected outcomes
|
||||
- [ ] Backend R2: test retrieval of segments/manifests via Cloudflare edge
|
||||
- [ ] Backend R2: validate edge cache hit/miss behavior with repeat requests (`scripts/validate-edge-cache.sh`)
|
||||
|
||||
## Streaming Pipeline Gaps (2026-08-10 audit → 2026-08-10 implemented)
|
||||
### Playout
|
||||
|
||||
STREAMING.md contract vs playlist.go/main.ts implementation. See Backlog for full details.
|
||||
|
||||
- [x] Continuous R2 sync during playout (HIGH) — implemented: fsnotify watcher uploads segments within ~7s
|
||||
- [x] Globally unique segment naming with run ID (HIGH) — implemented: `r-{ISOTIME}/seg-{seq}.m4s`
|
||||
- [x] Atomic manifest publication with temp key (MEDIUM) — implemented: upload to .tmp, then CopyObject
|
||||
- [x] Manifest Cache-Control: add must-revalidate + s-maxage split (LOW) — implemented: `public, max-age=1, s-maxage=2, must-revalidate`
|
||||
- [ ] CORS headers on HLS responses from media.linearvoid.tv (MEDIUM) — infrastructure, needs Cloudflare R2 CORS config
|
||||
- [x] Player error recovery: retries, backoff, reconnecting state (MEDIUM) — implemented: hls.on(ERROR) handler with exponential backoff
|
||||
- [x] Health endpoint: add active item, revision, run ID, segment info (LOW) — implemented: full STREAMING.md contract fields
|
||||
- [x] Phase 4 edge cache validation — script `scripts/validate-edge-cache.sh` created
|
||||
- [ ] Configure an R2 lifecycle rule on the `hls/` prefix as defense-in-depth (manual)
|
||||
- [ ] Full-loop E2E (2-file playlist, assert `Looping...` with no `CRASHED` across the boundary) — manual via `scripts/test-playout-e2e.sh`
|
||||
+18
-1
@@ -4,6 +4,23 @@
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
### Added
|
||||
|
||||
- **Automatic media normalization on upload**: Chat backend auto-transcodes non-conforming uploads. New `media.NormalizeVideo` wraps ffmpeg with the Linear Void profile (H.264/AAC, 30fps, 2s keyframes, yuv420p, 48kHz stereo), re-verifies output with ffprobe, and writes `*.normalized.mp4`. Media records gain `normalizing` / `normalization_failed` statuses; `POST /api/admin/media/{id}/normalize` re-triggers manually. Admin UI shows a "Normalize" action with status polling.
|
||||
- **R2 segment retention**: Playout `cleanupOldSegments` deletes `hls/r-*/*` objects older than `R2_SEGMENT_RETENTION_HOURS` (default 24h), run hourly + at startup, never touching the live manifest. Exposes `linearvoid_playout_r2_cleanup_objects_total` metric. `.env.example` adds `R2_SEGMENT_RETENTION_HOURS`.
|
||||
- **Playout loop metrics**: `linearvoid_playout_loops_total` (clean completions) and `linearvoid_playout_run_duration_seconds` (last run duration).
|
||||
|
||||
### Changed
|
||||
|
||||
- **Playout seamless looping**: `main()` now only resets HLS output when the playlist changed (SHA-256 `playlistRevision`), on crash/slate recovery, or first run; otherwise it continues segment numbering via `-start_number` and prunes stale segments. Eliminates the viewer gap on clean playlist loops.
|
||||
- **Single playout entrypoint**: `start-playout.sh` now runs `/app/playlist-manager -validate` then `exec`s the Go binary, removing the duplicate shell ffmpeg loop. Added a `-validate` mode to `main()`.
|
||||
|
||||
### Tests
|
||||
|
||||
- Media: `TestNormalizationArgs`, `TestNormalizeVideo_FFmpegError`.
|
||||
- Handlers: `TestNormalizeMediaRecord_Success`, `TestNormalizeMediaRecord_Failure`, `TestHandleAdminMediaByID_Normalize`.
|
||||
- Playout: `TestCleanupOldSegments`, `TestR2SegmentRetentionHours`, `TestPlaylistFingerprint`, `TestHLSNextStartNumber`.
|
||||
|
||||
## [1.0.12] - 2026-08-11
|
||||
|
||||
### Changed
|
||||
@@ -13,7 +30,7 @@
|
||||
### Fixed
|
||||
|
||||
- **Playout R2 endpoint validation**: `initR2Client()` now validates all required R2 environment variables (`R2_ENDPOINT`, `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET_NAME`) before creating the S3 client. Previously, an empty `R2_ENDPOINT` caused cryptic `Custom endpoint '' was not a valid URI` errors on every segment upload attempt. Playout now gracefully degrades when R2 is not configured — local HLS via nginx continues to work.
|
||||
- **CD workflow missing R2 secrets**: `.gitea/workflows/cd.yml` "Prepare environment" step now injects all R2_ variables (`R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET_NAME`, `R2_ENDPOINT`, plus admin variants) from Gitea secrets. Previously the generated `.env` was missing R2 entirely, causing the playout container to start with an empty endpoint.
|
||||
- **CD workflow missing R2 secrets**: `.gitea/workflows/cd.yml` "Prepare environment" step now injects all R2\_ variables (`R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, `R2_BUCKET_NAME`, `R2_ENDPOINT`, plus admin variants) from Gitea secrets. Previously the generated `.env` was missing R2 entirely, causing the playout container to start with an empty endpoint.
|
||||
- **Deployment docs**: `docs/DEPLOYMENT.md` now includes explicit R2 and Cloudflare variable lists plus a "Gitea Secrets" table documenting all 15 required secrets.
|
||||
|
||||
## [1.0.11] - 2026-08-10
|
||||
|
||||
@@ -46,6 +46,8 @@ flowchart TD
|
||||
P -->|sync| R2
|
||||
```
|
||||
|
||||
A detailed description of the Streaming architecture, playout, and delivery is available in [`docs/STREAMING.md`](docs/STREAMING.md).
|
||||
|
||||
## Project Structure
|
||||
|
||||
This structure directly reflects the building blocks, runtime flows, deployment view, and cross-cutting concepts defined in the arc42 documentation.
|
||||
|
||||
@@ -976,9 +976,9 @@ func HandleAdminMedia(deps *Dependencies) http.HandlerFunc {
|
||||
}
|
||||
|
||||
normResult := media.ValidateNormalization(targetPath)
|
||||
status := "ready"
|
||||
status := media.StatusReady
|
||||
if !normResult.Valid {
|
||||
status = "needs_normalization"
|
||||
status = media.StatusNeedsNormalization
|
||||
}
|
||||
|
||||
filename := filepath.Base(targetPath)
|
||||
@@ -998,6 +998,10 @@ func HandleAdminMedia(deps *Dependencies) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
if !normResult.Valid {
|
||||
go normalizeMediaRecord(deps, mediaRecord)
|
||||
}
|
||||
|
||||
files, err := media.ListFiles(deps.DB)
|
||||
if err != nil {
|
||||
http.Error(w, fmt.Sprintf("failed to list media files: %v", err), http.StatusInternalServerError)
|
||||
@@ -1307,9 +1311,9 @@ func HandleAdminMediaUploads(deps *Dependencies) http.HandlerFunc {
|
||||
}
|
||||
|
||||
normResult := media.ValidateNormalization(targetPath)
|
||||
status := "ready"
|
||||
status := media.StatusReady
|
||||
if !normResult.Valid {
|
||||
status = "needs_normalization"
|
||||
status = media.StatusNeedsNormalization
|
||||
}
|
||||
|
||||
filename := filepath.Base(targetPath)
|
||||
@@ -1329,6 +1333,10 @@ func HandleAdminMediaUploads(deps *Dependencies) http.HandlerFunc {
|
||||
return
|
||||
}
|
||||
|
||||
if !normResult.Valid {
|
||||
go normalizeMediaRecord(deps, mediaRecord)
|
||||
}
|
||||
|
||||
_ = os.RemoveAll(media.UploadSessionDir(mediaDir, uploadID))
|
||||
|
||||
files, err := media.ListFiles(deps.DB)
|
||||
@@ -1348,6 +1356,11 @@ func HandleAdminMediaUploads(deps *Dependencies) http.HandlerFunc {
|
||||
func HandleAdminMediaByID(deps *Dependencies) http.HandlerFunc {
|
||||
mediaDir := config.MediaDir()
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if strings.HasSuffix(r.URL.Path, "/normalize") {
|
||||
handleNormalizeMedia(deps, w, r)
|
||||
return
|
||||
}
|
||||
|
||||
mediaID, err := media.GetMediaIDFromPath(r.URL.Path, "/api/admin/media/")
|
||||
if err != nil {
|
||||
http.Error(w, err.Error(), http.StatusBadRequest)
|
||||
@@ -1424,6 +1437,75 @@ func HandleAdminMediaByID(deps *Dependencies) http.HandlerFunc {
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeMediaRecord runs ffmpeg normalization for a media record and updates
|
||||
// its status, path, duration, and size. Safe to run asynchronously.
|
||||
func normalizeMediaRecord(deps *Dependencies, record db.Media) {
|
||||
src := record.Path
|
||||
dst := strings.TrimSuffix(src, filepath.Ext(src)) + ".normalized.mp4"
|
||||
|
||||
if err := deps.DB.Model(&record).Update("status", media.StatusNormalizing).Error; err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
duration, size, err := media.NormalizeVideo(src, dst)
|
||||
if err != nil {
|
||||
_ = deps.DB.Model(&record).Updates(map[string]any{
|
||||
"status": media.StatusNormalizationFailed,
|
||||
"tags": `"normalization":false`,
|
||||
}).Error
|
||||
return
|
||||
}
|
||||
|
||||
_ = deps.DB.Model(&record).Updates(map[string]any{
|
||||
"path": dst,
|
||||
"filename": filepath.Base(dst),
|
||||
"size_bytes": size,
|
||||
"duration_seconds": duration,
|
||||
"status": media.StatusReady,
|
||||
"tags": `"normalization":true`,
|
||||
}).Error
|
||||
}
|
||||
|
||||
// handleNormalizeMedia handles POST /api/admin/media/{id}/normalize.
|
||||
func handleNormalizeMedia(deps *Dependencies, w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)
|
||||
return
|
||||
}
|
||||
|
||||
idPart := strings.TrimSuffix(r.URL.Path, "/normalize")
|
||||
idPart = strings.TrimPrefix(idPart, "/api/admin/media/")
|
||||
idPart = strings.Trim(idPart, "/")
|
||||
mediaID, err := strconv.ParseUint(idPart, 10, 64)
|
||||
if err != nil || mediaID == 0 {
|
||||
http.Error(w, "invalid media id", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
|
||||
var record db.Media
|
||||
if err := deps.DB.First(&record, mediaID).Error; err != nil {
|
||||
http.Error(w, "media item not found", http.StatusNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
if record.Status == media.StatusNormalizing {
|
||||
http.Error(w, "normalization already in progress", http.StatusConflict)
|
||||
return
|
||||
}
|
||||
|
||||
record.Status = media.StatusNormalizing
|
||||
if err := deps.DB.Model(&record).Update("status", media.StatusNormalizing).Error; err != nil {
|
||||
http.Error(w, fmt.Sprintf("failed to update media status: %v", err), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
|
||||
go normalizeMediaRecord(deps, record)
|
||||
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(http.StatusAccepted)
|
||||
json.NewEncoder(w).Encode(mediaEntryFromRecord(record))
|
||||
}
|
||||
|
||||
// HandleAdminChatLog lists chat log entries.
|
||||
func HandleAdminChatLog(deps *Dependencies) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
@@ -19,6 +20,7 @@ import (
|
||||
"chat/src/auth"
|
||||
"chat/src/config"
|
||||
"chat/src/db"
|
||||
"chat/src/media"
|
||||
"chat/src/ws"
|
||||
)
|
||||
|
||||
@@ -945,3 +947,132 @@ func TestHandleAdminMediaUploads_InvalidFile(t *testing.T) {
|
||||
t.Errorf("status = %d, want 400", w.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMediaRecord_Success(t *testing.T) {
|
||||
deps, gormDB, _ := setupTestDeps(t)
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
os.Setenv("MEDIA_DIR", tmpDir)
|
||||
defer os.Unsetenv("MEDIA_DIR")
|
||||
|
||||
srcPath := filepath.Join(tmpDir, "source.mov")
|
||||
if err := os.WriteFile(srcPath, []byte("fake video"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gormDB.Create(&db.Media{
|
||||
Filename: "source.mov",
|
||||
OriginalName: "source.mov",
|
||||
Path: srcPath,
|
||||
SizeBytes: 10,
|
||||
DurationSeconds: 0,
|
||||
Status: "needs_normalization",
|
||||
Title: "source",
|
||||
})
|
||||
|
||||
origNormalize := media.NormalizeVideo
|
||||
t.Cleanup(func() { media.NormalizeVideo = origNormalize })
|
||||
media.NormalizeVideo = func(src, dst string) (int, int64, error) {
|
||||
if err := os.WriteFile(dst, []byte("normalized"), 0o644); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return 120, 10, nil
|
||||
}
|
||||
|
||||
var record db.Media
|
||||
gormDB.First(&record, 1)
|
||||
normalizeMediaRecord(deps, record)
|
||||
|
||||
var updated db.Media
|
||||
gormDB.First(&updated, 1)
|
||||
if updated.Status != "ready" {
|
||||
t.Errorf("Status = %q, want ready", updated.Status)
|
||||
}
|
||||
if updated.DurationSeconds != 120 {
|
||||
t.Errorf("DurationSeconds = %d, want 120", updated.DurationSeconds)
|
||||
}
|
||||
if !strings.HasSuffix(updated.Path, ".normalized.mp4") {
|
||||
t.Errorf("Path = %q, want .normalized.mp4 suffix", updated.Path)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeMediaRecord_Failure(t *testing.T) {
|
||||
deps, gormDB, _ := setupTestDeps(t)
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
os.Setenv("MEDIA_DIR", tmpDir)
|
||||
defer os.Unsetenv("MEDIA_DIR")
|
||||
|
||||
srcPath := filepath.Join(tmpDir, "source.mov")
|
||||
if err := os.WriteFile(srcPath, []byte("fake video"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gormDB.Create(&db.Media{
|
||||
Filename: "source.mov",
|
||||
OriginalName: "source.mov",
|
||||
Path: srcPath,
|
||||
SizeBytes: 10,
|
||||
DurationSeconds: 0,
|
||||
Status: "needs_normalization",
|
||||
Title: "source",
|
||||
})
|
||||
|
||||
origNormalize := media.NormalizeVideo
|
||||
t.Cleanup(func() { media.NormalizeVideo = origNormalize })
|
||||
media.NormalizeVideo = func(src, dst string) (int, int64, error) {
|
||||
return 0, 0, fmt.Errorf("ffmpeg failed")
|
||||
}
|
||||
|
||||
var record db.Media
|
||||
gormDB.First(&record, 1)
|
||||
normalizeMediaRecord(deps, record)
|
||||
|
||||
var updated db.Media
|
||||
gormDB.First(&updated, 1)
|
||||
if updated.Status != "normalization_failed" {
|
||||
t.Errorf("Status = %q, want normalization_failed", updated.Status)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandleAdminMediaByID_Normalize(t *testing.T) {
|
||||
deps, gormDB, _ := setupTestDeps(t)
|
||||
|
||||
tmpDir := t.TempDir()
|
||||
os.Setenv("MEDIA_DIR", tmpDir)
|
||||
defer os.Unsetenv("MEDIA_DIR")
|
||||
|
||||
srcPath := filepath.Join(tmpDir, "source.mov")
|
||||
if err := os.WriteFile(srcPath, []byte("fake video"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
gormDB.Create(&db.Media{
|
||||
Filename: "source.mov",
|
||||
OriginalName: "source.mov",
|
||||
Path: srcPath,
|
||||
SizeBytes: 10,
|
||||
DurationSeconds: 0,
|
||||
Status: "needs_normalization",
|
||||
Title: "source",
|
||||
})
|
||||
|
||||
origNormalize := media.NormalizeVideo
|
||||
t.Cleanup(func() { media.NormalizeVideo = origNormalize })
|
||||
media.NormalizeVideo = func(src, dst string) (int, int64, error) {
|
||||
if err := os.WriteFile(dst, []byte("normalized"), 0o644); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
return 90, 9, nil
|
||||
}
|
||||
|
||||
handler := HandleAdminMediaByID(deps)
|
||||
req := httptest.NewRequest("POST", "/api/admin/media/1/normalize", nil)
|
||||
w := httptest.NewRecorder()
|
||||
|
||||
handler(w, req)
|
||||
|
||||
if w.Code != http.StatusAccepted {
|
||||
t.Errorf("status = %d, want 202", w.Code)
|
||||
}
|
||||
}
|
||||
@@ -155,6 +155,71 @@ func ProbeDurationSeconds(mediaPath string) (int, error) {
|
||||
return seconds, nil
|
||||
}
|
||||
|
||||
// Media status values (ADR-006 media library states).
|
||||
const (
|
||||
StatusReady = "ready"
|
||||
StatusNeedsNormalization = "needs_normalization"
|
||||
StatusNormalizing = "normalizing"
|
||||
StatusNormalizationFailed = "normalization_failed"
|
||||
)
|
||||
|
||||
// runCommand executes an external command and returns its combined output on failure.
|
||||
// It is a package variable so tests can substitute a stub.
|
||||
var runCommand = func(ctx context.Context, name string, args ...string) error {
|
||||
out, err := exec.CommandContext(ctx, name, args...).CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s failed: %v (%s)", name, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// normalizationArgs builds the ffmpeg arguments for the Linear Void normalization profile.
|
||||
func normalizationArgs(srcPath, dstPath string) []string {
|
||||
return []string{
|
||||
"-y",
|
||||
"-i", srcPath,
|
||||
"-map", "0:v:0", "-map", "0:a:0?",
|
||||
"-vf", "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2,fps=30,format=yuv420p",
|
||||
"-c:v", "libx264", "-preset", "medium", "-crf", "21",
|
||||
"-pix_fmt", "yuv420p",
|
||||
"-r", "30", "-g", "60", "-keyint_min", "60", "-sc_threshold", "0",
|
||||
"-c:a", "aac", "-b:a", "160k", "-ar", "48000", "-ac", "2",
|
||||
"-avoid_negative_ts", "make_zero", "-max_muxing_queue_size", "1024",
|
||||
"-movflags", "+faststart",
|
||||
dstPath,
|
||||
}
|
||||
}
|
||||
|
||||
// normalizeVideo transcodes srcPath to the Linear Void normalization profile,
|
||||
// re-verifies the output with ffprobe, and returns its duration (seconds) and size (bytes).
|
||||
func normalizeVideo(srcPath, dstPath string) (int, int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Hour)
|
||||
defer cancel()
|
||||
|
||||
if err := runCommand(ctx, "ffmpeg", normalizationArgs(srcPath, dstPath)...); err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
if res := ValidateNormalization(dstPath); !res.Valid {
|
||||
return 0, 0, fmt.Errorf("normalized output failed validation: %s", strings.Join(res.Errors, "; "))
|
||||
}
|
||||
|
||||
duration, err := ProbeDurationSeconds(dstPath)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("probe duration failed: %w", err)
|
||||
}
|
||||
|
||||
info, err := os.Stat(dstPath)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("stat output failed: %w", err)
|
||||
}
|
||||
|
||||
return duration, info.Size(), nil
|
||||
}
|
||||
|
||||
// NormalizeVideo is the injectable normalization entry point.
|
||||
var NormalizeVideo = normalizeVideo
|
||||
|
||||
// DetectMime reads the first 512 bytes of a file to detect its MIME type.
|
||||
func DetectMime(filePath string) (string, error) {
|
||||
f, err := os.Open(filePath)
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
package media
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
@@ -202,3 +205,42 @@ func TestUniqueTargetPath(t *testing.T) {
|
||||
t.Errorf("UniqueTargetPath() = %q, want %q", path, expected)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationArgs(t *testing.T) {
|
||||
args := normalizationArgs("/in/movie.mov", "/out/movie.normalized.mp4")
|
||||
joined := strings.Join(args, " ")
|
||||
for _, want := range []string{
|
||||
"-map 0:v:0",
|
||||
"-map 0:a:0?",
|
||||
"-c:v libx264",
|
||||
"-r 30",
|
||||
"-g 60",
|
||||
"-keyint_min 60",
|
||||
"-pix_fmt yuv420p",
|
||||
"-c:a aac",
|
||||
"-ar 48000",
|
||||
"-ac 2",
|
||||
"-avoid_negative_ts make_zero",
|
||||
"/out/movie.normalized.mp4",
|
||||
} {
|
||||
if !strings.Contains(joined, want) {
|
||||
t.Errorf("normalization args missing %q: %v", want, args)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeVideo_FFmpegError(t *testing.T) {
|
||||
origRun := runCommand
|
||||
t.Cleanup(func() { runCommand = origRun })
|
||||
|
||||
runCommand = func(ctx context.Context, name string, args ...string) error {
|
||||
if name != "ffmpeg" {
|
||||
t.Errorf("runCommand name = %q, want ffmpeg", name)
|
||||
}
|
||||
return fmt.Errorf("simulated ffmpeg failure")
|
||||
}
|
||||
|
||||
if _, _, err := normalizeVideo("/in/a.mov", "/out/a.normalized.mp4"); err == nil {
|
||||
t.Fatal("expected error when ffmpeg fails")
|
||||
}
|
||||
}
|
||||
+263
-14
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
@@ -16,6 +18,7 @@ import (
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/config"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
"github.com/fsnotify/fsnotify"
|
||||
)
|
||||
|
||||
@@ -32,12 +35,20 @@ var (
|
||||
putObject = func(client *s3.Client, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) {
|
||||
return client.PutObject(context.TODO(), input)
|
||||
}
|
||||
bucket = os.Getenv("R2_BUCKET_NAME")
|
||||
playoutMode = "starting" // "live", "slate", "degraded"
|
||||
restartCount = 0
|
||||
restartWindow = time.Now()
|
||||
slateDir = "/app/slate"
|
||||
hlsDir = "/app/hls"
|
||||
listObjects = func(client *s3.Client, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return client.ListObjectsV2(context.TODO(), input)
|
||||
}
|
||||
deleteObjects = func(client *s3.Client, input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) {
|
||||
return client.DeleteObjects(context.TODO(), input)
|
||||
}
|
||||
bucket = os.Getenv("R2_BUCKET_NAME")
|
||||
playoutMode = "starting" // "live", "slate", "degraded"
|
||||
restartCount = 0
|
||||
restartWindow = time.Now()
|
||||
slateDir = "/app/slate"
|
||||
hlsDir = "/app/hls"
|
||||
hlsSegmentDuration = envWithDefault("HLS_SEGMENT_DURATION", "6")
|
||||
hlsWindowSize = envWithDefault("HLS_WINDOW_SIZE", "24")
|
||||
|
||||
// Unique segment naming (STREAMING.md contract: r-{runID}/seg-{seq}.m4s)
|
||||
runID string
|
||||
@@ -51,6 +62,14 @@ var (
|
||||
consecutiveFails int
|
||||
lastFFmpegStartTime time.Time
|
||||
lastFFmpegExitError error
|
||||
|
||||
// R2 retention cleanup counter
|
||||
r2CleanupObjectsTotal int64
|
||||
|
||||
// Continuous playout loop metrics
|
||||
loopsTotal int64
|
||||
lastRunDuration time.Duration
|
||||
lastRunClean bool
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -61,6 +80,14 @@ const (
|
||||
manifestCacheControl = "public, max-age=1, s-maxage=2, must-revalidate"
|
||||
)
|
||||
|
||||
// envWithDefault returns the value of the environment variable or the default if not set.
|
||||
func envWithDefault(key, defaultVal string) string {
|
||||
if v := os.Getenv(key); v != "" {
|
||||
return v
|
||||
}
|
||||
return defaultVal
|
||||
}
|
||||
|
||||
// isR2Configured returns true when the required R2 environment variables are set.
|
||||
func isR2Configured() bool {
|
||||
return os.Getenv("R2_ENDPOINT") != "" &&
|
||||
@@ -568,8 +595,21 @@ func resetHLSOutput(hlsDir string) error {
|
||||
|
||||
func main() {
|
||||
playlistPath := "/app/playlist/playlist.txt"
|
||||
if len(os.Args) > 1 {
|
||||
playlistPath = os.Args[1]
|
||||
validateOnly := false
|
||||
for _, arg := range os.Args[1:] {
|
||||
switch arg {
|
||||
case "-validate":
|
||||
validateOnly = true
|
||||
default:
|
||||
if !strings.HasPrefix(arg, "-") {
|
||||
playlistPath = arg
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if validateOnly {
|
||||
runValidateOnly(playlistPath)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println("=== Linear Void Playout Service Starting ===")
|
||||
@@ -591,6 +631,11 @@ func main() {
|
||||
fmt.Println("R2 sync disabled:", err)
|
||||
}
|
||||
|
||||
// Start R2 segment retention cleanup (hourly + once at startup)
|
||||
if r2Client != nil {
|
||||
go runR2CleanupLoop()
|
||||
}
|
||||
|
||||
// Start HTTP health check server
|
||||
http.HandleFunc("/health", HealthHandler)
|
||||
http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -607,6 +652,15 @@ func main() {
|
||||
modeVal = 2
|
||||
}
|
||||
fmt.Fprintf(w, "linearvoid_playout_mode %d\n", modeVal)
|
||||
fmt.Fprintln(w, "# HELP linearvoid_playout_r2_cleanup_objects_total Total R2 objects deleted by retention cleanup.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_playout_r2_cleanup_objects_total counter")
|
||||
fmt.Fprintf(w, "linearvoid_playout_r2_cleanup_objects_total %d\n", r2CleanupObjectsTotal)
|
||||
fmt.Fprintln(w, "# HELP linearvoid_playout_loops_total Total clean playlist completions (loops).")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_playout_loops_total counter")
|
||||
fmt.Fprintf(w, "linearvoid_playout_loops_total %d\n", loopsTotal)
|
||||
fmt.Fprintln(w, "# HELP linearvoid_playout_run_duration_seconds Duration of the last FFmpeg run in seconds.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_playout_run_duration_seconds gauge")
|
||||
fmt.Fprintf(w, "linearvoid_playout_run_duration_seconds %f\n", lastRunDuration.Seconds())
|
||||
})
|
||||
go func() {
|
||||
fmt.Println("Health check server listening on :8081/health")
|
||||
@@ -625,10 +679,34 @@ func main() {
|
||||
// New run ID on each restart for unique R2 segment keys (STREAMING.md)
|
||||
initRunID()
|
||||
|
||||
if err := resetHLSOutput(hlsDir); err != nil {
|
||||
fmt.Println("Failed to reset HLS output:", err)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
// Reset HLS output only when the playlist changed, we are recovering from a
|
||||
// crash/slate, or this is the first run. Otherwise continue the segment
|
||||
// sequence so the loop is seamless for viewers (no gap on clean loops).
|
||||
fingerprint, fpErr := playlistFingerprint(playlistPath)
|
||||
if fpErr != nil {
|
||||
fmt.Println("Warning: cannot fingerprint playlist:", fpErr)
|
||||
fingerprint = ""
|
||||
}
|
||||
playlistChanged := fingerprint != playlistRevision
|
||||
shouldReset := playlistChanged || playoutMode != "live" || !lastRunClean
|
||||
|
||||
windowSize := 24
|
||||
if ws, err := strconv.Atoi(hlsWindowSize); err == nil && ws > 0 {
|
||||
windowSize = ws
|
||||
}
|
||||
|
||||
startNumber := 0
|
||||
if shouldReset {
|
||||
if err := resetHLSOutput(hlsDir); err != nil {
|
||||
fmt.Println("Failed to reset HLS output:", err)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
playlistRevision = fingerprint
|
||||
} else {
|
||||
if n, err := hlsNextStartNumber(hlsDir, windowSize); err == nil {
|
||||
startNumber = n
|
||||
}
|
||||
}
|
||||
|
||||
// Start filesystem watcher for real-time R2 segment upload (STREAMING.md).
|
||||
@@ -652,17 +730,20 @@ func main() {
|
||||
"-r", "30", "-g", "60", "-keyint_min", "60", "-sc_threshold", "0",
|
||||
"-c:a", "aac", "-ar", "48000", "-ac", "2", "-b:a", "128k",
|
||||
"-f", "hls",
|
||||
"-hls_time", "6",
|
||||
"-hls_list_size", "24",
|
||||
"-hls_time", hlsSegmentDuration,
|
||||
"-hls_list_size", hlsWindowSize,
|
||||
"-hls_segment_type", "fmp4",
|
||||
"-hls_fmp4_init_filename", "init.mp4",
|
||||
"-hls_flags", "delete_segments+independent_segments",
|
||||
"-hls_segment_filename", filepath.Join(hlsDir, "seg-%06d.m4s"),
|
||||
"-start_number", strconv.Itoa(startNumber),
|
||||
filepath.Join(hlsDir, "playlist.m3u8"))
|
||||
|
||||
runErr := cmd.Run()
|
||||
runDuration := time.Since(lastFFmpegStartTime)
|
||||
lastFFmpegExitError = runErr
|
||||
lastRunDuration = runDuration
|
||||
lastRunClean = runErr == nil
|
||||
|
||||
if watchCancel != nil {
|
||||
watchCancel() // Stop the filesystem watcher
|
||||
@@ -685,6 +766,7 @@ func main() {
|
||||
// Differentiate clean playlist completion from FFmpeg crash.
|
||||
// A nil error means FFmpeg completed the playlist normally — not a crash.
|
||||
if runErr == nil {
|
||||
loopsTotal++
|
||||
fmt.Printf("[%s] Playlist completed normally after %s. Looping...\n",
|
||||
time.Now().UTC().Format(time.RFC3339), runDuration.Round(time.Second))
|
||||
// If we were in slate mode, a successful run means recovery
|
||||
@@ -748,3 +830,170 @@ func syncLatestToR2(hlsDir string) error {
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// r2SegmentRetentionHours returns the R2 segment retention window in hours.
|
||||
func r2SegmentRetentionHours() time.Duration {
|
||||
h, err := strconv.Atoi(envWithDefault("R2_SEGMENT_RETENTION_HOURS", "24"))
|
||||
if err != nil || h <= 0 {
|
||||
h = 24
|
||||
}
|
||||
return time.Duration(h) * time.Hour
|
||||
}
|
||||
|
||||
// cleanupOldSegments deletes R2 objects under the hls/ prefix that are older than
|
||||
// the retention window, excluding the live manifest and any non-run keys.
|
||||
// It returns the number of objects deleted.
|
||||
func cleanupOldSegments(ctx context.Context) (int, error) {
|
||||
if r2Client == nil {
|
||||
if err := initR2Client(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
|
||||
cutoff := time.Now().UTC().Add(-r2SegmentRetentionHours())
|
||||
|
||||
var stale []string
|
||||
var token *string
|
||||
for {
|
||||
output, err := listObjects(r2Client, &s3.ListObjectsV2Input{
|
||||
Bucket: aws.String(bucket),
|
||||
Prefix: aws.String("hls/"),
|
||||
ContinuationToken: token,
|
||||
})
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
for _, obj := range output.Contents {
|
||||
key := aws.ToString(obj.Key)
|
||||
if key == "hls/playlist.m3u8" || key == "hls/playlist.m3u8.tmp" {
|
||||
continue
|
||||
}
|
||||
// Only consider segment/init objects under a run directory: hls/r-.../...
|
||||
if !strings.HasPrefix(key, "hls/r-") {
|
||||
continue
|
||||
}
|
||||
if obj.LastModified != nil && obj.LastModified.Before(cutoff) {
|
||||
stale = append(stale, key)
|
||||
}
|
||||
}
|
||||
if output.IsTruncated == nil || !*output.IsTruncated {
|
||||
break
|
||||
}
|
||||
token = output.NextContinuationToken
|
||||
}
|
||||
|
||||
if len(stale) == 0 {
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
deleted := 0
|
||||
for i := 0; i < len(stale); i += 1000 {
|
||||
end := i + 1000
|
||||
if end > len(stale) {
|
||||
end = len(stale)
|
||||
}
|
||||
batch := stale[i:end]
|
||||
objects := make([]types.ObjectIdentifier, len(batch))
|
||||
for j, key := range batch {
|
||||
objects[j] = types.ObjectIdentifier{Key: aws.String(key)}
|
||||
}
|
||||
if _, err := deleteObjects(r2Client, &s3.DeleteObjectsInput{
|
||||
Bucket: aws.String(bucket),
|
||||
Delete: &types.Delete{Objects: objects, Quiet: aws.Bool(true)},
|
||||
}); err != nil {
|
||||
return deleted, err
|
||||
}
|
||||
for _, key := range batch {
|
||||
fmt.Printf("R2 cleanup: deleted %s\n", key)
|
||||
}
|
||||
deleted += len(batch)
|
||||
}
|
||||
|
||||
r2CleanupObjectsTotal += int64(deleted)
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// runR2CleanupLoop runs retention cleanup once at startup, then hourly.
|
||||
func runR2CleanupLoop() {
|
||||
if _, err := cleanupOldSegments(context.Background()); err != nil {
|
||||
fmt.Println("R2 cleanup error:", err)
|
||||
}
|
||||
ticker := time.NewTicker(time.Hour)
|
||||
defer ticker.Stop()
|
||||
for range ticker.C {
|
||||
if _, err := cleanupOldSegments(context.Background()); err != nil {
|
||||
fmt.Println("R2 cleanup error:", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// playlistFingerprint returns a short hash of the playlist file content, used to
|
||||
// detect playlist changes between loops.
|
||||
func playlistFingerprint(playlistPath string) (string, error) {
|
||||
data, err := os.ReadFile(playlistPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
sum := sha256.Sum256(data)
|
||||
return hex.EncodeToString(sum[:]), nil
|
||||
}
|
||||
|
||||
// hlsNextStartNumber returns the next segment sequence number to use, and prunes
|
||||
// stale segments outside the live window so they do not accumulate across loops.
|
||||
func hlsNextStartNumber(hlsDir string, windowSize int) (int, error) {
|
||||
files, err := filepath.Glob(filepath.Join(hlsDir, "seg-*.m4s"))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
seqOf := func(f string) int {
|
||||
base := filepath.Base(f)
|
||||
num := strings.TrimSuffix(strings.TrimPrefix(base, "seg-"), ".m4s")
|
||||
n, err := strconv.Atoi(num)
|
||||
if err != nil {
|
||||
return -1
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
maxSeq := -1
|
||||
for _, f := range files {
|
||||
if n := seqOf(f); n > maxSeq {
|
||||
maxSeq = n
|
||||
}
|
||||
}
|
||||
next := maxSeq + 1
|
||||
|
||||
// Keep the live window; delete older segments no longer referenced.
|
||||
threshold := next - windowSize
|
||||
for _, f := range files {
|
||||
if n := seqOf(f); n >= 0 && n < threshold {
|
||||
if err := os.Remove(f); err != nil && !os.IsNotExist(err) {
|
||||
return next, err
|
||||
}
|
||||
}
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
// runValidateOnly validates the playlist and exits 0 (valid) or 1 (invalid/missing).
|
||||
func runValidateOnly(playlistPath string) {
|
||||
videos, err := ValidatePlaylist(playlistPath)
|
||||
if err != nil {
|
||||
fmt.Println("ERROR: Playlist validation failed:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
valid := true
|
||||
for _, v := range videos {
|
||||
if v.Valid {
|
||||
fmt.Printf("OK: %s (%.1fs)\n", v.Path, v.Duration)
|
||||
} else {
|
||||
valid = false
|
||||
fmt.Printf("INVALID: %s — %s\n", v.Path, v.Validation)
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
os.Exit(1)
|
||||
}
|
||||
fmt.Println("Playlist valid.")
|
||||
}
|
||||
@@ -27,6 +27,8 @@ package main
|
||||
// See backend/playout/testdata/*.txt.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
@@ -39,6 +41,7 @@ import (
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3"
|
||||
"github.com/aws/aws-sdk-go-v2/service/s3/types"
|
||||
)
|
||||
|
||||
func ensureFFProbeInPath(t *testing.T) {
|
||||
@@ -265,7 +268,7 @@ func TestHealthHandler(t *testing.T) {
|
||||
`"status":"healthy"`,
|
||||
`"service":"playout"`,
|
||||
`"mode":"live"`,
|
||||
`"restarts":3`,
|
||||
`"crash_count":3`,
|
||||
`"run_id":"r-20260810T120000Z"`,
|
||||
`"last_segment_seq"`,
|
||||
`"consecutive_failures"`,
|
||||
@@ -589,3 +592,145 @@ func TestSyncToR2_NoCacheHeadersForUnknownExt(t *testing.T) {
|
||||
t.Fatalf("ContentType = %q, want empty", gotContentType)
|
||||
}
|
||||
}
|
||||
|
||||
func TestR2SegmentRetentionHours(t *testing.T) {
|
||||
os.Setenv("R2_SEGMENT_RETENTION_HOURS", "")
|
||||
if got := r2SegmentRetentionHours(); got != 24*time.Hour {
|
||||
t.Errorf("default = %v, want 24h", got)
|
||||
}
|
||||
|
||||
os.Setenv("R2_SEGMENT_RETENTION_HOURS", "48")
|
||||
if got := r2SegmentRetentionHours(); got != 48*time.Hour {
|
||||
t.Errorf("override = %v, want 48h", got)
|
||||
}
|
||||
|
||||
os.Setenv("R2_SEGMENT_RETENTION_HOURS", "invalid")
|
||||
if got := r2SegmentRetentionHours(); got != 24*time.Hour {
|
||||
t.Errorf("invalid = %v, want 24h fallback", got)
|
||||
}
|
||||
|
||||
os.Unsetenv("R2_SEGMENT_RETENTION_HOURS")
|
||||
}
|
||||
|
||||
func TestCleanupOldSegments(t *testing.T) {
|
||||
prevClient := r2Client
|
||||
prevBucket := bucket
|
||||
prevList := listObjects
|
||||
prevDelete := deleteObjects
|
||||
prevTotal := r2CleanupObjectsTotal
|
||||
defer func() {
|
||||
r2Client = prevClient
|
||||
bucket = prevBucket
|
||||
listObjects = prevList
|
||||
deleteObjects = prevDelete
|
||||
r2CleanupObjectsTotal = prevTotal
|
||||
}()
|
||||
|
||||
bucket = "linearvoid-media"
|
||||
r2Client = &s3.Client{}
|
||||
|
||||
now := time.Now().UTC()
|
||||
old := now.Add(-25 * time.Hour)
|
||||
recent := now.Add(-1 * time.Hour)
|
||||
|
||||
listObjects = func(_ *s3.Client, input *s3.ListObjectsV2Input) (*s3.ListObjectsV2Output, error) {
|
||||
return &s3.ListObjectsV2Output{
|
||||
Contents: []types.Object{
|
||||
{Key: aws.String("hls/r-old/seg-1.m4s"), LastModified: &old},
|
||||
{Key: aws.String("hls/r-old/init.mp4"), LastModified: &old},
|
||||
{Key: aws.String("hls/r-new/seg-2.m4s"), LastModified: &recent},
|
||||
{Key: aws.String("hls/playlist.m3u8"), LastModified: &old},
|
||||
{Key: aws.String("hls/playlist.m3u8.tmp"), LastModified: &old},
|
||||
},
|
||||
IsTruncated: aws.Bool(false),
|
||||
}, nil
|
||||
}
|
||||
|
||||
var deletedKeys []string
|
||||
deleteObjects = func(_ *s3.Client, input *s3.DeleteObjectsInput) (*s3.DeleteObjectsOutput, error) {
|
||||
for _, obj := range input.Delete.Objects {
|
||||
deletedKeys = append(deletedKeys, aws.ToString(obj.Key))
|
||||
}
|
||||
return &s3.DeleteObjectsOutput{}, nil
|
||||
}
|
||||
|
||||
n, err := cleanupOldSegments(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("cleanupOldSegments error: %v", err)
|
||||
}
|
||||
if n != 2 {
|
||||
t.Fatalf("deleted = %d, want 2", n)
|
||||
}
|
||||
|
||||
joined := strings.Join(deletedKeys, ",")
|
||||
if !strings.Contains(joined, "hls/r-old/seg-1.m4s") || !strings.Contains(joined, "hls/r-old/init.mp4") {
|
||||
t.Fatalf("unexpected deleted keys: %v", deletedKeys)
|
||||
}
|
||||
if strings.Contains(joined, "playlist.m3u8") {
|
||||
t.Fatalf("manifest must not be deleted: %v", deletedKeys)
|
||||
}
|
||||
if r2CleanupObjectsTotal != 2 {
|
||||
t.Fatalf("r2CleanupObjectsTotal = %d, want 2", r2CleanupObjectsTotal)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlaylistFingerprint(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
p := filepath.Join(tmpDir, "playlist.txt")
|
||||
if err := os.WriteFile(p, []byte("file '/app/media/a.mp4'\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fp1, err := playlistFingerprint(p)
|
||||
if err != nil {
|
||||
t.Fatalf("playlistFingerprint error: %v", err)
|
||||
}
|
||||
fp2, err := playlistFingerprint(p)
|
||||
if err != nil {
|
||||
t.Fatalf("playlistFingerprint error: %v", err)
|
||||
}
|
||||
if fp1 != fp2 {
|
||||
t.Errorf("fingerprint not stable: %q vs %q", fp1, fp2)
|
||||
}
|
||||
|
||||
if err := os.WriteFile(p, []byte("file '/app/media/b.mp4'\n"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fp3, err := playlistFingerprint(p)
|
||||
if err != nil {
|
||||
t.Fatalf("playlistFingerprint error: %v", err)
|
||||
}
|
||||
if fp1 == fp3 {
|
||||
t.Errorf("fingerprint should change when playlist content changes")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHLSNextStartNumber(t *testing.T) {
|
||||
tmpDir := t.TempDir()
|
||||
// Create segments 0..29 with a live window of 10.
|
||||
for i := 0; i < 30; i++ {
|
||||
if err := os.WriteFile(filepath.Join(tmpDir, fmt.Sprintf("seg-%06d.m4s", i)), []byte("x"), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
next, err := hlsNextStartNumber(tmpDir, 10)
|
||||
if err != nil {
|
||||
t.Fatalf("hlsNextStartNumber error: %v", err)
|
||||
}
|
||||
if next != 30 {
|
||||
t.Errorf("next = %d, want 30", next)
|
||||
}
|
||||
|
||||
// Segments < 20 should be pruned; 20..29 kept (the live window).
|
||||
for i := 0; i < 20; i++ {
|
||||
if _, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("seg-%06d.m4s", i))); !os.IsNotExist(err) {
|
||||
t.Errorf("seg-%06d should have been pruned", i)
|
||||
}
|
||||
}
|
||||
for i := 20; i < 30; i++ {
|
||||
if _, err := os.Stat(filepath.Join(tmpDir, fmt.Sprintf("seg-%06d.m4s", i))); err != nil {
|
||||
t.Errorf("seg-%06d should be kept: %v", i, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -19,36 +19,13 @@ fi
|
||||
|
||||
# Run normalization validation using the Go tool
|
||||
echo "Validating playlist and enforcing normalization (ADR-001/ADR-003)..."
|
||||
if ! /app/playlist-manager "$PLAYLIST_PATH"; then
|
||||
if ! /app/playlist-manager -validate "$PLAYLIST_PATH"; then
|
||||
echo "ERROR: Playlist validation failed. Check logs."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Playlist validated successfully. Starting continuous HLS playout..."
|
||||
|
||||
mkdir -p /app/hls
|
||||
rm -f /app/hls/playlist.m3u8 /app/hls/init.mp4 /app/hls/segment_*.ts /app/hls/segment_*.m4s
|
||||
|
||||
# Run FFmpeg in a loop for continuous playback
|
||||
while true; do
|
||||
echo "[$(date)] Starting FFmpeg playout with browser-safe HLS encoding..."
|
||||
ffmpeg -re -f concat -safe 0 -i "$PLAYLIST_PATH" \
|
||||
-map 0:v:0 -map 0:a:0? \
|
||||
-vf "scale=1280:720:force_original_aspect_ratio=decrease,pad=1280:720:(ow-iw)/2:(oh-ih)/2,fps=30,format=yuv420p" \
|
||||
-c:v libx264 -preset veryfast -pix_fmt yuv420p \
|
||||
-profile:v high -level 4.0 \
|
||||
-r 30 -g 60 -keyint_min 60 -sc_threshold 0 \
|
||||
-c:a aac -ar 48000 -ac 2 -b:a 128k \
|
||||
-f hls \
|
||||
-hls_time "${HLS_SEGMENT_DURATION}" \
|
||||
-hls_list_size "${HLS_WINDOW_SIZE}" \
|
||||
-hls_segment_type fmp4 \
|
||||
-hls_fmp4_init_filename init.mp4 \
|
||||
-strftime 1 \
|
||||
-hls_flags delete_segments+independent_segments \
|
||||
-hls_segment_filename "/app/hls/segment_%Y%m%d_%H%M%S.m4s" \
|
||||
"/app/hls/playlist.m3u8" || echo "FFmpeg exited with error (will restart in 5s per ADR-001 resilience requirements)..."
|
||||
|
||||
echo "Playout loop crashed or ended. Restarting in 5 seconds..."
|
||||
sleep 5
|
||||
done
|
||||
# Single source of truth: the Go binary runs the full loop (HLS, R2 sync,
|
||||
# slate fallback, health server). The shell no longer duplicates the FFmpeg loop.
|
||||
exec /app/playlist-manager "$PLAYLIST_PATH"
|
||||
+23
-69
@@ -1,5 +1,7 @@
|
||||
# Linear Void Setup Guide
|
||||
|
||||
**Last Updated**: 2026-08-03
|
||||
|
||||
This guide covers prerequisites, environment setup, running the application locally and in production, and troubleshooting. It references the [architecture documentation](./docs/architecture/README.md)(`docs/architecture/README.md`).
|
||||
|
||||
## Prerequisites
|
||||
@@ -7,7 +9,7 @@ This guide covers prerequisites, environment setup, running the application loca
|
||||
- **Docker & Docker Compose**: Required for running the full stack (playout, chat, nginx, redis, db, prometheus, grafana). Install from [docker.com](https://www.docker.com).
|
||||
- **Node.js 22+**: Required for frontend development and production builds (`cd frontend && npm install && npm run build`).
|
||||
- **npm**: Installed with Node.js and used for the frontend dependency installation and Vite workflow.
|
||||
- **Go 1.23+**: Recommended when working on the Go services outside Docker (`backend/chat`, `backend/playout`).
|
||||
- **Go 1.25+**: Recommended when working on the Go services outside Docker (`backend/chat`, `backend/playout`).
|
||||
- **FFmpeg**: Used by the playout service for HLS generation. Install via `apt install ffmpeg` (Linux), Homebrew (macOS), or Winget (Windows).
|
||||
- **Git**: For cloning and version control.
|
||||
- **Cloudflare Account**: For R2 storage and CDN (free tier sufficient for most use cases).
|
||||
@@ -81,70 +83,7 @@ flowchart LR
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
3. **Configure `.env`** (edit with your values):
|
||||
|
||||
```env
|
||||
# Cloudflare R2 (required for video storage and $0 egress)
|
||||
R2_ACCESS_KEY_ID=your-r2-access-key
|
||||
R2_SECRET_ACCESS_KEY=your-r2-secret-key
|
||||
R2_BUCKET_NAME=linearvoid-media
|
||||
R2_ENDPOINT=https://your-account-id.r2.cloudflarestorage.com
|
||||
R2_REGION=auto
|
||||
|
||||
# Cloudflare (for custom domain and cache rules)
|
||||
CLOUDFLARE_API_TOKEN=your-cloudflare-api-token
|
||||
CLOUDFLARE_ACCOUNT_ID=your-account-id
|
||||
MEDIA_DOMAIN=media.linearvoid.tv
|
||||
|
||||
# Public frontend URLs
|
||||
# Preferred local setup uses the real public domains through Cloudflare and Nginx Proxy Manager.
|
||||
VITE_PUBLIC_APP_URL=https://linearvoid.tv
|
||||
VITE_PUBLIC_WS_URL=wss://linearvoid.tv/ws
|
||||
VITE_PUBLIC_HLS_URL=https://media.linearvoid.tv/hls/playlist.m3u8
|
||||
VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR=admin
|
||||
VITE_PUBLIC_ADMIN_DEFAULT_ROOM=main
|
||||
VITE_PUBLIC_ADMIN_MODERATION_LOG_LIMIT=50
|
||||
ADMIN_USERNAME=admin
|
||||
ADMIN_PASSWORD=change-me
|
||||
ADMIN_SESSION_TTL_HOURS=12
|
||||
# Optional direct-local fallback when bypassing the public proxy:
|
||||
# VITE_PUBLIC_APP_URL=http://localhost
|
||||
# VITE_PUBLIC_WS_URL=ws://localhost/ws
|
||||
# VITE_PUBLIC_HLS_URL=http://localhost/hls/playlist.m3u8
|
||||
|
||||
# Gitea Container Registry (for self-hosted Docker images)
|
||||
CI_REGISTRY_URL=git.allucanget.biz
|
||||
CI_REGISTRY_USER=your-gitea-username
|
||||
CI_REGISTRY_TOKEN=your-gitea-token
|
||||
CI_REGISTRY_NAMESPACE=zwitschi/linearvoid
|
||||
# Gitea API access (optional, for CI/release automation)
|
||||
CI_SERVER_URL=https://git.allucanget.biz
|
||||
|
||||
# Chat / Database
|
||||
DATABASE_URL=postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable
|
||||
# or for SQLite-only local development:
|
||||
# DATABASE_URL=sqlite://./data/chat.db
|
||||
REDIS_URL=redis://redis:6379/0
|
||||
|
||||
# Ports (for local development)
|
||||
PLAYOUT_PORT=8080
|
||||
CHAT_PORT=3000
|
||||
NGINX_PORT=80
|
||||
|
||||
# Playout Configuration
|
||||
PLAYLIST_PATH=/app/playlist.txt
|
||||
HLS_SEGMENT_DURATION=6
|
||||
HLS_WINDOW_SIZE=24
|
||||
|
||||
# Logging / Monitoring
|
||||
LOG_LEVEL=info
|
||||
PROMETHEUS_PORT=9090
|
||||
GRAFANA_PORT=3001
|
||||
|
||||
# Development flags
|
||||
NODE_ENV=development
|
||||
GO_ENV=development
|
||||
```
|
||||
3. **Configure `.env`** (copy from `.env.example` and replace placeholders)
|
||||
|
||||
4. **Check local value overrides for Docker Compose**:
|
||||
- `docker-compose.yml` builds images locally and maps the frontend build output from `frontend/dist` into nginx.
|
||||
@@ -227,7 +166,23 @@ flowchart LR
|
||||
- Enable Smart Tiered Cache for the R2-backed hostname.
|
||||
- If you need header normalization for immutable segment caching, deploy `infrastructure/cloudflare/worker.js` to the same zone.
|
||||
|
||||
7. **Verify access**:
|
||||
7. **Configure R2 CORS**:
|
||||
- Add CORS rules to allow the app domain to access the R2 bucket for HLS delivery.
|
||||
- Example CORS configuration:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"AllowedOrigins": ["https://linearvoid.tv"],
|
||||
"AllowedMethods": ["GET"],
|
||||
"AllowedHeaders": ["*"],
|
||||
"ExposeHeaders": ["ETag", "Cache-Control"],
|
||||
"MaxAgeSeconds": 3600
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
8. **Verify access**:
|
||||
- Confirm uploads with an S3-compatible client such as `aws s3 ls --endpoint-url <R2_ENDPOINT>`.
|
||||
- After deployment, check Cloudflare analytics for cache hit ratio and origin request volume.
|
||||
|
||||
@@ -249,7 +204,8 @@ Execution tracker: `docs/operations/cloudflare-uptimerobot-checklist.md`.
|
||||
1. Create bucket `linearvoid-media`.
|
||||
2. Set location hint to `enam` (US-East).
|
||||
3. Attach custom domain `media.linearvoid.tv` to the bucket.
|
||||
4. Verify object read works from the custom domain.
|
||||
4. Configure CORS rules to allow `https://linearvoid.tv` for HLS delivery.
|
||||
5. Verify object read works from the custom domain.
|
||||
|
||||
### 3. Cache Rules
|
||||
|
||||
@@ -588,5 +544,3 @@ See `docs/architecture/README.md` for the full arc42 documentation and the ADRs
|
||||
- Project Structure: Root `README.md`.
|
||||
|
||||
The system is ready for use. For production, review ADR-001 for cost estimates and scaling guidance.
|
||||
|
||||
**Last Updated**: 2026-08-03
|
||||
+88
-13
@@ -480,6 +480,37 @@ async function deleteOverride(id: number) {
|
||||
}
|
||||
}
|
||||
|
||||
async function renderServiceRow(service: any) {
|
||||
const svcDetail = service.detail || "";
|
||||
|
||||
// playout service returns json detail, all others return ISO formatted time string
|
||||
if (service.name === "playout" && svcDetail) {
|
||||
try {
|
||||
// create a list of key-value pairs for each property in the JSON object
|
||||
const detailObj = JSON.parse(svcDetail);
|
||||
service.detail = Object.entries(detailObj)
|
||||
.map(([key, value]) => `${key}: ${value}`)
|
||||
.join("\n");
|
||||
} catch {
|
||||
// ignore JSON parse errors
|
||||
}
|
||||
}
|
||||
|
||||
const row = document.createElement("article");
|
||||
row.className = "service-row";
|
||||
row.innerHTML = `
|
||||
<div>
|
||||
<h3>${service.name}</h3>
|
||||
<p class="small-muted">${service.checked}</p>
|
||||
</div>
|
||||
<div class="service-row-right">
|
||||
<p class="service-status service-status-${service.status}">${service.status}</p>
|
||||
<p>${service.detail}</p>
|
||||
</div>
|
||||
`;
|
||||
return row;
|
||||
}
|
||||
|
||||
async function refreshSystemSummary() {
|
||||
setStatus("system-status", "Loading system summary...");
|
||||
try {
|
||||
@@ -499,19 +530,9 @@ async function refreshSystemSummary() {
|
||||
servicesEl.innerHTML = "";
|
||||
const services = (payload.services || []) as ServiceHealth[];
|
||||
services.forEach((service) => {
|
||||
const row = document.createElement("article");
|
||||
row.className = "service-row";
|
||||
row.innerHTML = `
|
||||
<div>
|
||||
<h3>${service.name}</h3>
|
||||
<p class="small-muted">${service.checked}</p>
|
||||
</div>
|
||||
<div class="service-row-right">
|
||||
<p class="service-status service-status-${service.status}">${service.status}</p>
|
||||
<p>${service.detail}</p>
|
||||
</div>
|
||||
`;
|
||||
servicesEl.appendChild(row);
|
||||
renderServiceRow(service).then((row) => {
|
||||
servicesEl.appendChild(row);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -534,6 +555,7 @@ function renderMediaRows(files: MediaEntry[]) {
|
||||
|
||||
files.forEach((file) => {
|
||||
const tr = document.createElement("tr");
|
||||
const canNormalize = file.status === "needs_normalization" || file.status === "normalization_failed";
|
||||
tr.innerHTML = `
|
||||
<td>${file.name}</td>
|
||||
<td>${file.title || "-"}</td>
|
||||
@@ -543,10 +565,14 @@ function renderMediaRows(files: MediaEntry[]) {
|
||||
<td>${file.modifiedAt || "-"}</td>
|
||||
<td class="action-cell">
|
||||
<button type="button" class="btn btn-secondary btn-small" data-action="edit" data-id="${file.id || ""}">Edit</button>
|
||||
${canNormalize ? `<button type="button" class="btn btn-secondary btn-small" data-action="normalize" data-id="${file.id || ""}">Normalize</button>` : ""}
|
||||
<button type="button" class="btn btn-danger btn-small" data-action="delete" data-id="${file.id || ""}">Delete</button>
|
||||
</td>
|
||||
`;
|
||||
tr.querySelector('[data-action="edit"]')?.addEventListener("click", () => editMedia(file));
|
||||
if (canNormalize) {
|
||||
tr.querySelector('[data-action="normalize"]')?.addEventListener("click", () => normalizeMedia(file));
|
||||
}
|
||||
tr.querySelector('[data-action="delete"]')?.addEventListener("click", () => deleteMedia(file));
|
||||
tbody.appendChild(tr);
|
||||
});
|
||||
@@ -699,6 +725,55 @@ async function editMedia(file: MediaEntry) {
|
||||
}
|
||||
}
|
||||
|
||||
async function normalizeMedia(file: MediaEntry) {
|
||||
if (!file.id) {
|
||||
setStatus("media-status", "Cannot normalize media item without id.", true);
|
||||
return;
|
||||
}
|
||||
|
||||
setStatus("media-status", `Normalizing ${file.name}...`);
|
||||
try {
|
||||
const response = await fetch(`/api/admin/media/${file.id}/normalize`, { method: "POST" });
|
||||
if (!response.ok) throw new Error(await readErrorMessage(response));
|
||||
|
||||
setStatus("media-status", `Normalization started for ${file.name}.`);
|
||||
await refreshMediaLibrary();
|
||||
pollNormalizationStatus(file.id, 0);
|
||||
} catch (error) {
|
||||
console.error("Failed to normalize media", error);
|
||||
const details = error instanceof Error ? error.message : "Unknown error.";
|
||||
setStatus("media-status", `Failed to normalize ${file.name}. (${details})`, true);
|
||||
}
|
||||
}
|
||||
|
||||
function pollNormalizationStatus(id: number, attempt: number) {
|
||||
if (attempt >= 60) {
|
||||
setStatus("media-status", "Normalization is taking longer than expected. Reload the library to check.", true);
|
||||
return;
|
||||
}
|
||||
window.setTimeout(async () => {
|
||||
try {
|
||||
const response = await fetch(`/api/admin/media/${id}`);
|
||||
if (!response.ok) return;
|
||||
const file = (await response.json()) as MediaEntry;
|
||||
if (file.status === "normalizing" || file.status === "needs_normalization") {
|
||||
pollNormalizationStatus(id, attempt + 1);
|
||||
return;
|
||||
}
|
||||
await refreshMediaLibrary();
|
||||
await refreshOverview();
|
||||
await refreshSchedule();
|
||||
setStatus(
|
||||
"media-status",
|
||||
file.status === "ready" ? `Normalization complete for ${file.name}.` : `Normalization failed for ${file.name}.`,
|
||||
file.status !== "ready",
|
||||
);
|
||||
} catch {
|
||||
// Stop polling on error; the user can reload the library manually.
|
||||
}
|
||||
}, 3000);
|
||||
}
|
||||
|
||||
async function deleteMedia(file: MediaEntry) {
|
||||
if (!file.id) {
|
||||
setStatus("media-status", "Cannot delete media item without id.", true);
|
||||
|
||||
@@ -1,66 +0,0 @@
|
||||
{
|
||||
"title": "Linear Void Monitoring",
|
||||
"panels": [
|
||||
{
|
||||
"title": "Concurrent Viewers",
|
||||
"type": "timeseries",
|
||||
"targets": [{ "expr": "linearvoid_chat_connected_clients" }]
|
||||
},
|
||||
{
|
||||
"title": "Stored Messages",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "linearvoid_chat_messages_total"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Moderation Actions",
|
||||
"type": "timeseries",
|
||||
"targets": [{ "expr": "linearvoid_chat_moderation_actions_total" }]
|
||||
},
|
||||
{
|
||||
"title": "Redis Reachable",
|
||||
"type": "stat",
|
||||
"targets": [{ "expr": "linearvoid_chat_redis_up" }]
|
||||
},
|
||||
{
|
||||
"title": "Admin Sessions",
|
||||
"type": "timeseries",
|
||||
"targets": [{ "expr": "linearvoid_chat_admin_sessions_active" }]
|
||||
},
|
||||
{
|
||||
"title": "Chat Scrape Health",
|
||||
"type": "stat",
|
||||
"targets": [{ "expr": "up{job=\"chat\"}" }]
|
||||
},
|
||||
{
|
||||
"title": "Cache Hit Ratio",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "linearvoid_cache_hit_ratio"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Redis Memory Usage",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "linearvoid_chat_redis_memory_usage_ratio"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"title": "Chat Request Latency p95",
|
||||
"type": "timeseries",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, sum(rate(linearvoid_chat_request_duration_seconds_bucket[5m])) by (le))"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in new issue
Block a user