perf(playout): parallelize playlist validation with dedup and worker pool

.gitea/workflows/cd.yml

@@ -270,8 +270,8 @@ jobs:
               echo "Playout is healthy"
               break
             fi
-            echo "Playout not ready yet (attempt $i/6), waiting 10s..."
+            echo "Playout not ready yet (attempt $i/6), waiting 20s..."
-            sleep 10
+            sleep 20
           done
           if ! docker exec linearvoid-playout curl -sf http://localhost:8081/health; then
             echo "Playout failed to become healthy after 6 attempts"

.github/instructions/DONE.md

@@ -2,6 +2,14 @@
 
 > Completed tasks from [TODO](./TODO.md). Product, feature, and release history lives in [CHANGELOG](./CHANGELOG.md); this file keeps only engineering and ops notes not captured there.
 
+## Playout startup performance (2026-08-16)
+
+- Parallelized playlist validation: dedup repeated paths, probe unique files concurrently (`PLAYOUT_VALIDATE_WORKERS` default `runtime.NumCPU()`), run the two `ffprobe` probes concurrently, and validate once in `main()` (removed the redundant `-validate` pre-step in `start-playout.sh`). See CHANGELOG.
+
+## Cloudflare Workers frontend URLs (2026-08-16)
+
+- Viewer `main.ts` uses `VITE_PUBLIC_APP_URL` for all API requests and `VITE_PUBLIC_WS_URL` for WebSocket connections so the Workers-served SPA reaches the app origin cross-origin. See CHANGELOG.
+
 ## Test infrastructure & refactors (2026-08-07)
 
 - Chat service `main.go` split into 8 packages (config, auth, db, handlers, ws, media, ratelimit, metrics); `db.RedisClient` + `db.PubSub` interfaces added for mocking.

.github/instructions/TODO.md

@@ -2,11 +2,6 @@
 
 > Completed tasks moved to [DONE](./DONE.md). Remaining tasks in [BACKLOG](./BACKLOG.md).
 
-## Cloudflare Workers
-
-- [x] Adjust `frontend/src/main.ts` to use `VITE_PUBLIC_APP_URL` for API requests instead of hardcoded `/api/...` paths.
-- [x] Adjust `frontend/src/main.ts` to use `VITE_PUBLIC_WS_URL` for WebSocket connections instead of hardcoded `wss://...` paths.
-
 ## Frontend Tests
 
 ### User Frontend

CHANGELOG.md

@@ -17,6 +17,7 @@
 - **R2 retention by run count + size**: Playout now keeps the newest `R2_KEEP_RUNS` run folders (default 2) and enforces an `R2_MAX_RETENTION_BYTES` cap (default 10 GiB), replacing the old 24h time-based `R2_SEGMENT_RETENTION_HOURS` cleanup. Cleanup runs after each new run plus an hourly backstop, and never deletes the live manifest. Adds `linearvoid_playout_r2_runs_retained` and `linearvoid_playout_r2_bytes_retained` gauges.
 - **Cloudflare Workers frontend hosting**: Replaced Cloudflare Pages with a Cloudflare Worker (`linearvoid-frontend`) that serves the static SPA via Workers Static Assets. Added `frontend/wrangler.toml` and renamed the deploy workflow to `.gitea/workflows/workers-deploy.yml` (builds the SPA and runs `npx wrangler deploy`). Architecture, setup/deployment, and ops documentation updated from Pages to Workers.
 - **Viewer session cookie cross-site**: The `linearvoid_user_session` cookie now uses `SameSite=None; Secure` over HTTPS (so it is sent cross-site from the Workers-served SPA) and `SameSite=Lax` over plain HTTP (local dev). The admin session cookie remains `SameSite=Strict`.
+- **Parallel playout startup validation**: `ValidatePlaylist` now deduplicates repeated paths and probes unique files concurrently through a bounded worker pool (`PLAYOUT_VALIDATE_WORKERS`, default `runtime.NumCPU()`); `ValidateVideo` runs its stream/format and keyframe `ffprobe` probes concurrently; and `main()` performs the strict validation once (removing the redundant `-validate` pre-step in `start-playout.sh`). This sharply cuts startup time on large or duplicated playlists.
 
 ### Fixed
 
@@ -28,6 +29,7 @@
 - Handlers: `TestHandleAdminChatLog_ExcludesSystem`.
 - Frontend: `presence flash` test.
 - Playout: `TestCleanupOldRuns`, `TestCleanupOldRuns_SizeCap`, `TestR2KeepRuns`, `TestR2MaxRetentionBytes`, `TestR2RunFromKey`.
+- Playout: `TestValidateWorkersConfig`, `TestValidatePlaylist_Dedup`.
 
 ## [1.0.13] - 2026-08-13
 

backend/playout/src/playlist.go

@@ -11,9 +11,11 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
+	"sync"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
@@ -238,16 +240,32 @@ func LoadPlaylist(playlistPath string) ([]VideoFile, error) {
 // ValidateVideo uses ffprobe to enforce codec, fps, and keyframe interval (ADR-001)
 func ValidateVideo(videoPath string) (VideoFile, error) {
 	v := VideoFile{Path: videoPath}
-	var errors []string
 
-	// Run ffprobe
+	// Run the stream/format probe and the keyframe probe concurrently: they are
+	// independent, and the keyframe probe (-show_frames) is the expensive one.
+	var streamOut []byte
+	var streamErr error
+	var keyframeErr error
+	var wg sync.WaitGroup
+	wg.Add(2)
+	go func() {
+		defer wg.Done()
 		cmd := exec.Command("ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", videoPath)
-	output, err := cmd.Output()
+		streamOut, streamErr = cmd.Output()
-	if err != nil {
+	}()
-		v.Validation = "ffprobe failed: " + err.Error()
+	go func() {
-		return v, fmt.Errorf("ffprobe: %w", err)
+		defer wg.Done()
+		keyframeErr = validateKeyframeInterval(videoPath)
+	}()
+	wg.Wait()
+
+	if streamErr != nil {
+		v.Validation = "ffprobe failed: " + streamErr.Error()
+		return v, fmt.Errorf("ffprobe: %w", streamErr)
 	}
 
+	var errors []string
+
 	// Parse JSON
 	var probe struct {
 		Streams []struct {
@@ -265,7 +283,7 @@ func ValidateVideo(videoPath string) (VideoFile, error) {
 			Duration string `json:"duration"`
 		} `json:"format"`
 	}
-	if err := json.Unmarshal(output, &probe); err != nil {
+	if err := json.Unmarshal(streamOut, &probe); err != nil {
 		v.Validation = "ffprobe JSON parse failed: " + err.Error()
 		return v, fmt.Errorf("ffprobe json: %w", err)
 	}
@@ -326,8 +344,7 @@ func ValidateVideo(videoPath string) (VideoFile, error) {
 		return v, fmt.Errorf("validation failed: %s", v.Validation)
 	}
 
-	// Validate keyframe interval <=2s (60 frames at 30fps)
+	// Keyframe interval check (probe already completed concurrently above).
-	keyframeErr := validateKeyframeInterval(videoPath)
 	if keyframeErr != nil {
 		errors = append(errors, keyframeErr.Error())
 	}
@@ -421,21 +438,62 @@ func ValidateVideoStrict(videoPath string) map[string]any {
 	return result
 }
 
-// ValidatePlaylist runs normalization validation on all entries
+// validateWorkers returns the number of concurrent validation workers.
+func validateWorkers() int {
+	n := runtime.NumCPU()
+	if v := os.Getenv("PLAYOUT_VALIDATE_WORKERS"); v != "" {
+		if parsed, err := strconv.Atoi(v); err == nil && parsed > 0 {
+			n = parsed
+		}
+	}
+	return n
+}
+
+// ValidatePlaylist runs normalization validation on all entries in parallel,
+// deduplicating repeated paths so each unique file is probed once.
 func ValidatePlaylist(playlistPath string) ([]VideoFile, error) {
 	videos, err := LoadPlaylist(playlistPath)
 	if err != nil {
 		return nil, err
 	}
 
-	for i := range videos {
+	// Collect unique paths in order of first appearance.
-		validated, err := ValidateVideo(videos[i].Path)
+	var order []string
-		videos[i] = validated
+	seen := make(map[string]bool, len(videos))
-		if err != nil {
+	for _, v := range videos {
-			fmt.Printf("Warning: %s - %s\n", videos[i].Path, err)
+		if !seen[v.Path] {
+			seen[v.Path] = true
+			order = append(order, v.Path)
 		}
 	}
 
+	// Validate unique paths concurrently with a bounded worker pool.
+	results := make(map[string]VideoFile, len(order))
+	var mu sync.Mutex
+	var wg sync.WaitGroup
+	sem := make(chan struct{}, validateWorkers())
+	for _, path := range order {
+		wg.Add(1)
+		sem <- struct{}{}
+		go func(p string) {
+			defer wg.Done()
+			defer func() { <-sem }()
+			vf, err := ValidateVideo(p)
+			if err != nil {
+				fmt.Printf("Warning: %s - %s\n", p, err)
+			}
+			mu.Lock()
+			results[p] = vf
+			mu.Unlock()
+		}(path)
+	}
+	wg.Wait()
+
+	// Map results back onto the playlist (preserving order and duplicates).
+	for i := range videos {
+		videos[i] = results[videos[i].Path]
+	}
+
 	return videos, nil
 }
 
@@ -632,11 +690,24 @@ func main() {
 	initRunID()
 	fmt.Println("Run ID:", runID)
 
-	// Validate playlist before starting (ADR-001 normalization enforcement)
+	// Validate playlist before starting (ADR-001 normalization enforcement).
-	if _, err := ValidatePlaylist(playlistPath); err != nil {
+	// Runs once here; start-playout.sh no longer pre-validates separately.
+	videos, err := ValidatePlaylist(playlistPath)
+	if err != nil {
 		fmt.Println("ERROR: Playlist validation failed:", err)
 		os.Exit(1)
 	}
+	invalid := 0
+	for _, v := range videos {
+		if !v.Valid {
+			fmt.Printf("ERROR: invalid media: %s — %s\n", v.Path, v.Validation)
+			invalid++
+		}
+	}
+	if invalid > 0 {
+		fmt.Printf("ERROR: Playlist validation failed: %d invalid entries\n", invalid)
+		os.Exit(1)
+	}
 
 	// Initialize R2 client (skip if not configured — local HLS via nginx still works)
 	if err := initR2Client(); err != nil {

backend/playout/src/playlist_test.go

@@ -465,6 +465,54 @@ func TestValidatePlaylist(t *testing.T) {
 	t.Skip("requires ffprobe mock or test media fixtures")
 }
 
+func TestValidateWorkersConfig(t *testing.T) {
+	os.Unsetenv("PLAYOUT_VALIDATE_WORKERS")
+	if got := validateWorkers(); got < 1 {
+		t.Errorf("validateWorkers() = %d, want >= 1", got)
+	}
+
+	os.Setenv("PLAYOUT_VALIDATE_WORKERS", "3")
+	if got := validateWorkers(); got != 3 {
+		t.Errorf("validateWorkers() = %d, want 3", got)
+	}
+
+	os.Setenv("PLAYOUT_VALIDATE_WORKERS", "invalid")
+	if got := validateWorkers(); got < 1 {
+		t.Errorf("validateWorkers() = %d, want fallback >= 1", got)
+	}
+
+	os.Unsetenv("PLAYOUT_VALIDATE_WORKERS")
+}
+
+func TestValidatePlaylist_Dedup(t *testing.T) {
+	ensureFFProbeInPath(t)
+
+	valid := filepath.Join("..", "testdata", "valid_720p30.mp4")
+	playlistDir := t.TempDir()
+	p := filepath.Join(playlistDir, "playlist.txt")
+	// Same file listed three times to exercise dedup + order preservation.
+	content := fmt.Sprintf("file '%s'\nfile '%s'\nfile '%s'\n", valid, valid, valid)
+	if err := os.WriteFile(p, []byte(content), 0o644); err != nil {
+		t.Fatalf("write playlist: %v", err)
+	}
+
+	videos, err := ValidatePlaylist(p)
+	if err != nil {
+		t.Fatalf("ValidatePlaylist error: %v", err)
+	}
+	if len(videos) != 3 {
+		t.Fatalf("len = %d, want 3", len(videos))
+	}
+	for i, v := range videos {
+		if !v.Valid {
+			t.Errorf("entry %d invalid: %s", i, v.Validation)
+		}
+		if v.Path != valid {
+			t.Errorf("entry %d path = %q, want %q", i, v.Path, valid)
+		}
+	}
+}
+
 func TestSyncToR2(t *testing.T) {
 	prevClient := r2Client
 	prevBucket := bucket

backend/playout/start-playout.sh

@@ -17,15 +17,7 @@ file '/app/media/demo2.mp4'
 EOF
 fi
 
-# Run normalization validation using the Go tool
+# Single source of truth: the Go binary validates the playlist (ADR-001/ADR-003
-echo "Validating playlist and enforcing normalization (ADR-001/ADR-003)..."
+# normalization enforcement) and runs the full loop (HLS, R2 sync, slate
-if ! /app/playlist-manager -validate "$PLAYLIST_PATH"; then
+# fallback, health server). Validation runs once inside the binary.
-  echo "ERROR: Playlist validation failed. Check logs."
-  exit 1
-fi
-
-echo "Playlist validated successfully. Starting continuous HLS playout..."
-
-# Single source of truth: the Go binary runs the full loop (HLS, R2 sync,
-# slate fallback, health server). The shell no longer duplicates the FFmpeg loop.
 exec /app/playlist-manager "$PLAYLIST_PATH"