chore: release v1.0.7
Playout: slate fallback, strict ffprobe validation, Prometheus metrics Chat: history on join, DM offline delivery, env-config rate limits, Redis auto-reconnect, heartbeat, ban/mute enforcement, DM metrics Viewer: presence indicators, schedule overlay, public /api/schedule Admin: playlist drag-reorder, schedule overrides CRUD, user actions, inline mod buttons, CSV export, config editor, restart buttons, upload normalization gate, analytics time range nginx: CORS preflight, JSON health, stub_status Docs: test-lab vs production distinction across all configs/docs Infra: Redis maxmemory, CD health IP-agnostic, slate volume mount
This commit is contained in:
zwitschi committed
1 parent
17ac04856b
commit
5b5cdc4cd8
26 files changed
+1723
-110
No files matched your search
+282
-15
@@ -3,11 +3,13 @@ package main
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@@ -25,8 +27,18 @@ type VideoFile struct {
|
||||
}
|
||||
|
||||
var (
|
||||
r2Client *s3.Client
|
||||
bucket = os.Getenv("R2_BUCKET_NAME")
|
||||
r2Client *s3.Client
|
||||
bucket = os.Getenv("R2_BUCKET_NAME")
|
||||
playoutMode = "starting" // "live", "slate", "degraded"
|
||||
restartCount = 0
|
||||
restartWindow = time.Now()
|
||||
slateDir = "/app/slate"
|
||||
hlsDir = "/app/hls"
|
||||
)
|
||||
|
||||
const (
|
||||
maxRestartsPerMinute = 5
|
||||
slateManifestName = "playlist.m3u8"
|
||||
)
|
||||
|
||||
// initR2Client initializes the R2 (S3-compatible) client
|
||||
@@ -79,23 +91,181 @@ func LoadPlaylist(playlistPath string) ([]VideoFile, error) {
|
||||
// ValidateVideo uses ffprobe to enforce codec, fps, and keyframe interval (ADR-001)
|
||||
func ValidateVideo(videoPath string) (VideoFile, error) {
|
||||
v := VideoFile{Path: videoPath}
|
||||
var errors []string
|
||||
|
||||
// Run ffprobe
|
||||
cmd := exec.Command("ffprobe", "-v", "quiet", "-print_format", "json", "-show_format", "-show_streams", videoPath)
|
||||
_, err := cmd.Output()
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
v.Validation = "ffprobe failed: " + err.Error()
|
||||
return v, err
|
||||
return v, fmt.Errorf("ffprobe: %w", err)
|
||||
}
|
||||
|
||||
// TODO: Parse JSON for strict checks (codec_name=h264, r_frame_rate=30/1, key_frame interval)
|
||||
// For MVP we assume valid if ffprobe succeeds
|
||||
v.Valid = true
|
||||
v.Validation = "Validation passed (H.264/AVC, AAC, 30fps constant, 2s keyframes - full JSON parse TODO per ADR-001)"
|
||||
v.Duration = 60.0 // placeholder from ffprobe
|
||||
// Parse JSON
|
||||
var probe struct {
|
||||
Streams []struct {
|
||||
CodecName string `json:"codec_name"`
|
||||
CodecType string `json:"codec_type"`
|
||||
Width int `json:"width"`
|
||||
Height int `json:"height"`
|
||||
RFrameRate string `json:"r_frame_rate"`
|
||||
PixFmt string `json:"pix_fmt"`
|
||||
SampleRate string `json:"sample_rate"`
|
||||
Channels int `json:"channels"`
|
||||
Duration string `json:"duration"`
|
||||
} `json:"streams"`
|
||||
Format struct {
|
||||
Duration string `json:"duration"`
|
||||
} `json:"format"`
|
||||
}
|
||||
if err := json.Unmarshal(output, &probe); err != nil {
|
||||
v.Validation = "ffprobe JSON parse failed: " + err.Error()
|
||||
return v, fmt.Errorf("ffprobe json: %w", err)
|
||||
}
|
||||
|
||||
if len(probe.Streams) == 0 {
|
||||
errors = append(errors, "no streams detected")
|
||||
}
|
||||
|
||||
var hasVideo, hasAudio bool
|
||||
for _, s := range probe.Streams {
|
||||
switch s.CodecType {
|
||||
case "video":
|
||||
hasVideo = true
|
||||
if s.CodecName != "h264" {
|
||||
errors = append(errors, fmt.Sprintf("video codec must be h264, got %s", s.CodecName))
|
||||
}
|
||||
if s.RFrameRate != "30/1" {
|
||||
errors = append(errors, fmt.Sprintf("framerate must be 30/1, got %s", s.RFrameRate))
|
||||
}
|
||||
if s.Width != 1280 && s.Width != 1920 {
|
||||
errors = append(errors, fmt.Sprintf("width must be 1280 or 1920, got %d", s.Width))
|
||||
}
|
||||
if s.Height != 720 && s.Height != 1080 {
|
||||
errors = append(errors, fmt.Sprintf("height must be 720 or 1080, got %d", s.Height))
|
||||
}
|
||||
if s.PixFmt != "yuv420p" {
|
||||
errors = append(errors, fmt.Sprintf("pixel format must be yuv420p, got %s", s.PixFmt))
|
||||
}
|
||||
case "audio":
|
||||
hasAudio = true
|
||||
if s.CodecName != "aac" {
|
||||
errors = append(errors, fmt.Sprintf("audio codec must be aac, got %s", s.CodecName))
|
||||
}
|
||||
if s.SampleRate != "48000" {
|
||||
errors = append(errors, fmt.Sprintf("audio sample rate must be 48000, got %s", s.SampleRate))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !hasVideo {
|
||||
errors = append(errors, "no video stream found")
|
||||
}
|
||||
if !hasAudio {
|
||||
errors = append(errors, "no audio stream found")
|
||||
}
|
||||
|
||||
// Parse duration from format or stream
|
||||
durationSec := 0.0
|
||||
if probe.Format.Duration != "" {
|
||||
if d, err := strconv.ParseFloat(probe.Format.Duration, 64); err == nil {
|
||||
durationSec = d
|
||||
}
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
v.Valid = false
|
||||
v.Validation = strings.Join(errors, "; ")
|
||||
return v, fmt.Errorf("validation failed: %s", v.Validation)
|
||||
}
|
||||
|
||||
// Validate keyframe interval <=2s (60 frames at 30fps)
|
||||
keyframeErr := validateKeyframeInterval(videoPath)
|
||||
if keyframeErr != nil {
|
||||
errors = append(errors, keyframeErr.Error())
|
||||
}
|
||||
|
||||
if len(errors) > 0 {
|
||||
v.Valid = false
|
||||
v.Validation = strings.Join(errors, "; ")
|
||||
return v, fmt.Errorf("validation failed: %s", v.Validation)
|
||||
}
|
||||
|
||||
v.Valid = true
|
||||
v.Duration = durationSec
|
||||
v.Validation = fmt.Sprintf("OK: H.264/AAC, 30fps, yuv420p, 48kHz, %.1fs, keyframes OK", durationSec)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// validateKeyframeInterval checks that max gap between keyframes <=2s (60 frames at 30fps).
|
||||
func validateKeyframeInterval(videoPath string) error {
|
||||
cmd := exec.Command("ffprobe",
|
||||
"-v", "quiet",
|
||||
"-select_streams", "v:0",
|
||||
"-show_frames",
|
||||
"-show_entries", "frame=key_frame,pkt_pts_time",
|
||||
"-of", "csv=p=0",
|
||||
videoPath)
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("keyframe probe failed: %w", err)
|
||||
}
|
||||
|
||||
type kfEntry struct{ pts float64 }
|
||||
var keyframes []kfEntry
|
||||
lines := strings.Split(strings.TrimSpace(string(output)), "\n")
|
||||
for _, line := range lines {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(line, ",")
|
||||
if len(parts) < 2 || parts[0] != "1" {
|
||||
continue
|
||||
}
|
||||
pts, err := strconv.ParseFloat(strings.TrimSpace(parts[1]), 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
keyframes = append(keyframes, kfEntry{pts})
|
||||
}
|
||||
|
||||
if len(keyframes) < 2 {
|
||||
return fmt.Errorf("insufficient keyframes for interval analysis (need >=2, got %d)", len(keyframes))
|
||||
}
|
||||
|
||||
maxGap := 0.0
|
||||
for i := 1; i < len(keyframes); i++ {
|
||||
gap := keyframes[i].pts - keyframes[i-1].pts
|
||||
if gap > maxGap {
|
||||
maxGap = gap
|
||||
}
|
||||
}
|
||||
|
||||
if maxGap > 2.1 { // small tolerance for floating point
|
||||
return fmt.Errorf("keyframe interval too large: max %.2fs (must be <=2s)", maxGap)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// ValidateVideoStrict runs ValidateVideo and returns a structured JSON result.
|
||||
func ValidateVideoStrict(videoPath string) map[string]any {
|
||||
v, err := ValidateVideo(videoPath)
|
||||
result := map[string]any{
|
||||
"path": videoPath,
|
||||
"valid": v.Valid,
|
||||
"errors": []string{},
|
||||
}
|
||||
if err != nil {
|
||||
result["errors"] = strings.Split(v.Validation, "; ")
|
||||
}
|
||||
if v.Valid {
|
||||
result["duration_seconds"] = v.Duration
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// ValidatePlaylist runs normalization validation on all entries
|
||||
func ValidatePlaylist(playlistPath string) ([]VideoFile, error) {
|
||||
videos, err := LoadPlaylist(playlistPath)
|
||||
@@ -139,8 +309,65 @@ func SyncToR2(localPath, r2Key string) error {
|
||||
// HealthHandler provides a simple health check endpoint
|
||||
func HealthHandler(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
status := "healthy"
|
||||
if playoutMode == "slate" {
|
||||
status = "degraded"
|
||||
}
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprint(w, `{"status":"healthy","service":"playout","timestamp":"`+time.Now().UTC().Format(time.RFC3339)+`"}`)
|
||||
fmt.Fprintf(w, `{"status":"%s","service":"playout","mode":"%s","restarts":%d,"timestamp":"%s"}`,
|
||||
status, playoutMode, restartCount, time.Now().UTC().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// activateSlate copies pre-rendered slate HLS into the live output directory.
|
||||
// Writes a static manifest referencing slate segments. Returns error if slate not found.
|
||||
func activateSlate() error {
|
||||
slateHLS := filepath.Join(slateDir, "hls")
|
||||
if _, err := os.Stat(filepath.Join(slateHLS, "slate.m3u8")); os.IsNotExist(err) {
|
||||
return fmt.Errorf("slate HLS not found at %s — run scripts/generate-slate.sh first", slateHLS)
|
||||
}
|
||||
|
||||
// Clear existing live segments
|
||||
if err := resetHLSOutput(hlsDir); err != nil {
|
||||
return fmt.Errorf("failed to clear HLS output for slate: %w", err)
|
||||
}
|
||||
|
||||
// Copy slate segments into live HLS dir
|
||||
slateFiles, err := filepath.Glob(filepath.Join(slateHLS, "*"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, src := range slateFiles {
|
||||
dst := filepath.Join(hlsDir, filepath.Base(src))
|
||||
data, err := os.ReadFile(src)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(dst, data, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
playoutMode = "slate"
|
||||
fmt.Printf("[%s] EMERGENCY: Switched to slate fallback. %d restarts in window.\n",
|
||||
time.Now().Format(time.RFC3339), restartCount)
|
||||
return nil
|
||||
}
|
||||
|
||||
// deactivateSlate clears slate state and prepares for live playout resumption.
|
||||
func deactivateSlate() {
|
||||
playoutMode = "live"
|
||||
fmt.Printf("[%s] RECOVERY: Slate deactivated, resuming live playout.\n",
|
||||
time.Now().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
// shouldEscalateToSlate checks if restart count exceeds threshold within the window.
|
||||
func shouldEscalateToSlate() bool {
|
||||
if time.Since(restartWindow) > 60*time.Second {
|
||||
restartCount = 0
|
||||
restartWindow = time.Now()
|
||||
return false
|
||||
}
|
||||
return restartCount >= maxRestartsPerMinute
|
||||
}
|
||||
|
||||
func resetHLSOutput(hlsDir string) error {
|
||||
@@ -207,6 +434,21 @@ func main() {
|
||||
|
||||
// Start HTTP health check server
|
||||
http.HandleFunc("/health", HealthHandler)
|
||||
http.HandleFunc("/metrics", func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
fmt.Fprintln(w, "# HELP linearvoid_playout_restarts_total Total FFmpeg restarts.")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_playout_restarts_total counter")
|
||||
fmt.Fprintf(w, "linearvoid_playout_restarts_total %d\n", restartCount)
|
||||
fmt.Fprintln(w, "# HELP linearvoid_playout_mode Current playout mode (0=live, 1=slate, 2=degraded).")
|
||||
fmt.Fprintln(w, "# TYPE linearvoid_playout_mode gauge")
|
||||
modeVal := 0
|
||||
if playoutMode == "slate" {
|
||||
modeVal = 1
|
||||
} else if playoutMode == "degraded" {
|
||||
modeVal = 2
|
||||
}
|
||||
fmt.Fprintf(w, "linearvoid_playout_mode %d\n", modeVal)
|
||||
})
|
||||
go func() {
|
||||
fmt.Println("Health check server listening on :8081/health")
|
||||
if err := http.ListenAndServe(":8081", nil); err != nil {
|
||||
@@ -214,9 +456,14 @@ func main() {
|
||||
}
|
||||
}()
|
||||
|
||||
// Run FFmpeg in a loop for continuous playback with R2 sync hook
|
||||
// Run FFmpeg in a loop for continuous playback with slate fallback
|
||||
for {
|
||||
if err := resetHLSOutput("/app/hls"); err != nil {
|
||||
if playoutMode == "slate" {
|
||||
// During slate: attempt recovery by restarting FFmpeg
|
||||
fmt.Printf("[%s] Slate active — attempting live playout recovery...\n", time.Now().Format(time.RFC3339))
|
||||
}
|
||||
|
||||
if err := resetHLSOutput(hlsDir); err != nil {
|
||||
fmt.Println("Failed to reset HLS output:", err)
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
@@ -241,12 +488,32 @@ func main() {
|
||||
"-hls_segment_filename", "/app/hls/segment_%Y%m%d_%H%M%S.m4s",
|
||||
"/app/hls/playlist.m3u8")
|
||||
|
||||
if err := cmd.Run(); err != nil {
|
||||
fmt.Println("FFmpeg exited with error. Restarting in 5s (per ADR-001 resilience)...")
|
||||
runErr := cmd.Run()
|
||||
|
||||
// Track restart and check for slate escalation
|
||||
restartCount++
|
||||
fmt.Printf("[%s] FFmpeg exited (restart #%d in window): %v\n",
|
||||
time.Now().Format(time.RFC3339), restartCount, runErr)
|
||||
|
||||
if shouldEscalateToSlate() {
|
||||
fmt.Printf("[%s] CRITICAL: %d restarts in 60s — escalating to slate fallback\n",
|
||||
time.Now().Format(time.RFC3339), restartCount)
|
||||
if err := activateSlate(); err != nil {
|
||||
fmt.Printf("[%s] Slate activation failed: %v — will retry\n",
|
||||
time.Now().Format(time.RFC3339), err)
|
||||
}
|
||||
// Stay in slate mode, retry recovery after delay
|
||||
time.Sleep(30 * time.Second)
|
||||
continue
|
||||
}
|
||||
|
||||
// If we were in slate but FFmpeg ran successfully, we recovered
|
||||
if playoutMode == "slate" {
|
||||
deactivateSlate()
|
||||
}
|
||||
|
||||
// After FFmpeg run, sync latest segments and manifest to R2
|
||||
if err := syncLatestToR2("/app/hls"); err != nil {
|
||||
if err := syncLatestToR2(hlsDir); err != nil {
|
||||
fmt.Println("R2 sync warning:", err)
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user