diff --git a/.gitignore b/.gitignore index 1e4e74a..8b7a892 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,10 @@ yarn-debug.log* yarn-error.log* pnpm-debug.log* +# Benchmark artifacts +artifacts/ +scripts/loadtest/artifacts/ + # Runtime data pids *.pid diff --git a/CHANGELOG.md b/CHANGELOG.md index 348d5b8..6a038f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,29 @@ +## [1.0.10] - 2026-08-09 + +### Added + +- **WebSocket summary helper**: Added `scripts/loadtest/summarize-artillery.ps1` to extract counters, rates, and available percentile stats from Artillery JSON output. +- **Playbook worksheet mapping**: Added artifact-to-worksheet mapping section in `docs/operations/scaling-performance-playbook.md` for WebSocket and Admin API benchmark artifacts. +- **Test fixture generation docs**: Added ffmpeg commands for generating valid and invalid media fixtures as comments in `backend/playout/src/playlist_test.go`. Phase 1 test fixtures now complete. +- **R2 cache header support**: `SyncToR2` now sets `CacheControl` and `ContentType` based on file extension: + - `.m4s`/`.ts`/`.mp4`/`.m4a` → immutable (max-age=31536000) + - `.m3u8` → short-lived (max-age=2s) + - unknown → no cache headers +- **R2 integration tests**: Added `TestSyncToR2_ManifestCacheHeaders` and `TestSyncToR2_NoCacheHeadersForUnknownExt` verifying cache header behavior for segment, manifest, and unknown file types. + +### Changed + +- **Loadtest runner**: `scripts/loadtest/run-loadtests.ps1` now generates `ws-chat-summary.txt` automatically for `ws` and `all` scenarios. +- **Loadtest docs**: `scripts/loadtest/README.md` now documents `summarize-artillery.ps1` usage and the new `ws-chat-summary.txt` artifact. +- **Planning status wording**: Updated `docs/DEVELOPMENT.md` and `.github/instructions/TODO.md` to mark ws/admin benchmark scaffolding complete while benchmark execution remains pending. +- **Phase 1 complete**: Test fixture creation and ffmpeg documentation marked done. +- **Phase 4 partially complete**: R2 upload + cache header validation done; edge cache hit/miss validation remains manual. +- **Phase 2 frontend tests**: Added presence updates (join/leave rendering), private DM flow, and message rendering tests in `main.test.ts` (27 total tests passing). +- **Phase 3 admin frontend tests**: Added playlist management tests (save via PUT, error handling, sortable rendering) in `admin.test.ts` (31 total tests passing). + ## [1.0.9] - 2026-08-06 ### Added diff --git a/backend/chat/src/main.go b/backend/chat/src/main.go index 0fe3875..6c17841 100644 --- a/backend/chat/src/main.go +++ b/backend/chat/src/main.go @@ -62,32 +62,32 @@ func main() { // Register routes http.HandleFunc("/ws", hub.HandleConnection) - http.HandleFunc("/api/admin/login", handlers.HandleAdminLogin(deps)) - http.HandleFunc("/api/admin/logout", handlers.HandleAdminLogout(deps)) - http.HandleFunc("/api/admin/session", handlers.HandleAdminSession(deps)) - http.HandleFunc("/api/admin/overview", requireAdmin(handlers.HandleAdminOverview(deps))) - http.HandleFunc("/api/admin/playlist", requireAdmin(handlers.HandleAdminPlaylist(deps))) - http.HandleFunc("/api/admin/moderation/logs", requireAdmin(handlers.HandleAdminModerationLogs(deps))) - http.HandleFunc("/api/admin/moderation/actions", requireAdmin(handlers.HandleAdminModerationAction(deps))) - http.HandleFunc("/api/admin/analytics", requireAdmin(handlers.HandleAdminAnalytics(deps))) - http.HandleFunc("/api/admin/system", requireAdmin(handlers.HandleAdminSystemSummary(deps))) - http.HandleFunc("/api/admin/config", requireAdmin(handlers.HandleAdminConfig(deps))) - http.HandleFunc("/api/admin/restart", requireAdmin(handlers.HandleAdminRestart(deps))) - http.HandleFunc("/api/admin/users", requireAdmin(handlers.HandleAdminUsers(deps))) - http.HandleFunc("/api/admin/schedule", requireAdmin(handlers.HandleAdminSchedule(deps))) - http.HandleFunc("/api/admin/schedule/overrides", requireAdmin(handlers.HandleAdminScheduleOverrides(deps))) - http.HandleFunc("/api/schedule", handlers.HandlePublicSchedule(deps)) - http.HandleFunc("/api/admin/media", requireAdmin(handlers.HandleAdminMedia(deps))) - http.HandleFunc("/api/admin/media/uploads", requireAdmin(handlers.HandleAdminMediaUploads(deps))) - http.HandleFunc("/api/admin/media/uploads/", requireAdmin(handlers.HandleAdminMediaUploads(deps))) - http.HandleFunc("/api/admin/media/", requireAdmin(handlers.HandleAdminMediaByID(deps))) - http.HandleFunc("/api/admin/chat-log/bulk", requireAdmin(handlers.HandleAdminChatLogBulk(deps))) - http.HandleFunc("/api/admin/chat-log/export", requireAdmin(handlers.HandleAdminChatLogExport(deps))) - http.HandleFunc("/api/admin/chat-log/", requireAdmin(handlers.HandleAdminChatLogByID(deps))) - http.HandleFunc("/api/admin/chat-log", requireAdmin(handlers.HandleAdminChatLog(deps))) + http.HandleFunc("/api/admin/login", metrics.WrapRequestTiming(handlers.HandleAdminLogin(deps))) + http.HandleFunc("/api/admin/logout", metrics.WrapRequestTiming(handlers.HandleAdminLogout(deps))) + http.HandleFunc("/api/admin/session", metrics.WrapRequestTiming(handlers.HandleAdminSession(deps))) + http.HandleFunc("/api/admin/overview", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminOverview(deps)))) + http.HandleFunc("/api/admin/playlist", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminPlaylist(deps)))) + http.HandleFunc("/api/admin/moderation/logs", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminModerationLogs(deps)))) + http.HandleFunc("/api/admin/moderation/actions", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminModerationAction(deps)))) + http.HandleFunc("/api/admin/analytics", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminAnalytics(deps)))) + http.HandleFunc("/api/admin/system", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminSystemSummary(deps)))) + http.HandleFunc("/api/admin/config", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminConfig(deps)))) + http.HandleFunc("/api/admin/restart", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminRestart(deps)))) + http.HandleFunc("/api/admin/users", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminUsers(deps)))) + http.HandleFunc("/api/admin/schedule", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminSchedule(deps)))) + http.HandleFunc("/api/admin/schedule/overrides", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminScheduleOverrides(deps)))) + http.HandleFunc("/api/schedule", metrics.WrapRequestTiming(handlers.HandlePublicSchedule(deps))) + http.HandleFunc("/api/admin/media", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMedia(deps)))) + http.HandleFunc("/api/admin/media/uploads", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMediaUploads(deps)))) + http.HandleFunc("/api/admin/media/uploads/", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMediaUploads(deps)))) + http.HandleFunc("/api/admin/media/", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminMediaByID(deps)))) + http.HandleFunc("/api/admin/chat-log/bulk", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLogBulk(deps)))) + http.HandleFunc("/api/admin/chat-log/export", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLogExport(deps)))) + http.HandleFunc("/api/admin/chat-log/", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLogByID(deps)))) + http.HandleFunc("/api/admin/chat-log", metrics.WrapRequestTiming(requireAdmin(handlers.HandleAdminChatLog(deps)))) http.HandleFunc("/metrics", metrics.HandleMetrics(dbClient.DB, redisWrapper, hub, sessionStore, dbDriver, instanceID)) - http.HandleFunc("/health", metrics.HandleHealth()) - http.HandleFunc("/", metrics.HandleNotFound()) + http.HandleFunc("/health", metrics.WrapRequestTiming(metrics.HandleHealth())) + http.HandleFunc("/", metrics.WrapRequestTiming(metrics.HandleNotFound())) port := config.ChatPort() log.Printf("Chat service starting on :%s", port) diff --git a/backend/chat/src/metrics/metrics.go b/backend/chat/src/metrics/metrics.go index b21d827..a78921d 100644 --- a/backend/chat/src/metrics/metrics.go +++ b/backend/chat/src/metrics/metrics.go @@ -3,9 +3,11 @@ package metrics import ( "context" "fmt" + "math" "net/http" "strconv" "strings" + "sync" "time" "gorm.io/gorm" @@ -15,6 +17,62 @@ import ( "chat/src/ws" ) +var requestDurationBuckets = []float64{0.05, 0.1, 0.25, 0.5, 1, 2, 5} + +var chatRequestDurationHistogram = newRequestDurationHistogram(requestDurationBuckets) + +type requestDurationHistogram struct { + mu sync.Mutex + buckets []float64 + counts []uint64 + sum float64 + total uint64 +} + +func newRequestDurationHistogram(buckets []float64) *requestDurationHistogram { + return &requestDurationHistogram{ + buckets: append([]float64(nil), buckets...), + counts: make([]uint64, len(buckets)+1), + } +} + +func (h *requestDurationHistogram) observe(durationSeconds float64) { + h.mu.Lock() + defer h.mu.Unlock() + + h.total++ + h.sum += durationSeconds + for index, bucket := range h.buckets { + if durationSeconds <= bucket { + h.counts[index]++ + return + } + } + h.counts[len(h.counts)-1]++ +} + +func (h *requestDurationHistogram) snapshot() (counts []uint64, sum float64, total uint64) { + h.mu.Lock() + defer h.mu.Unlock() + + counts = append([]uint64(nil), h.counts...) + return counts, h.sum, h.total +} + +// ObserveRequestDuration records a chat HTTP request duration in seconds. +func ObserveRequestDuration(duration time.Duration) { + chatRequestDurationHistogram.observe(duration.Seconds()) +} + +// WrapRequestTiming records request duration for the provided handler. +func WrapRequestTiming(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + startedAt := time.Now() + defer ObserveRequestDuration(time.Since(startedAt)) + next(w, r) + } +} + // HandleMetrics returns Prometheus-format metrics for the chat service. func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionStore *auth.SessionStore, dbDriver, instanceID string) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -36,6 +94,9 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt redisEnabled := 0 redisUp := 0 redisClients := 0 + redisMemoryUsedBytes := int64(0) + redisMemoryMaxBytes := int64(0) + redisMemoryUsageRatio := 0.0 if redis != nil { redisEnabled = 1 if err := redis.Ping(context.Background()); err == nil { @@ -50,6 +111,19 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt } } } + if info, err := redis.Info(context.Background(), "memory"); err == nil { + if value := parseRedisMemoryBytes(info, "used_memory_human"); value > 0 { + redisMemoryUsedBytes = value + } + if value := parseRedisMemoryBytes(info, "maxmemory_human"); value > 0 { + redisMemoryMaxBytes = value + } else if value := parseRedisMemoryBytes(info, "maxmemory"); value > 0 { + redisMemoryMaxBytes = value + } + if redisMemoryMaxBytes > 0 { + redisMemoryUsageRatio = math.Min(float64(redisMemoryUsedBytes)/float64(redisMemoryMaxBytes), 1.0) + } + } } } @@ -99,6 +173,18 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_connected_clients gauge") writeLine("linearvoid_chat_redis_connected_clients", redisClients) + fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_used_bytes Redis used memory reported by INFO memory.") + fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_used_bytes gauge") + writeLine("linearvoid_chat_redis_memory_used_bytes", redisMemoryUsedBytes) + + fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_max_bytes Redis maxmemory limit reported by INFO memory.") + fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_max_bytes gauge") + writeLine("linearvoid_chat_redis_memory_max_bytes", redisMemoryMaxBytes) + + fmt.Fprintln(w, "# HELP linearvoid_chat_redis_memory_usage_ratio Redis memory usage ratio (used / max).") + fmt.Fprintln(w, "# TYPE linearvoid_chat_redis_memory_usage_ratio gauge") + writeLine("linearvoid_chat_redis_memory_usage_ratio", redisMemoryUsageRatio) + fmt.Fprintln(w, "# HELP linearvoid_chat_admin_sessions_active Active unexpired admin sessions.") fmt.Fprintln(w, "# TYPE linearvoid_chat_admin_sessions_active gauge") writeLine("linearvoid_chat_admin_sessions_active", activeSessions) @@ -106,6 +192,19 @@ func HandleMetrics(gormDB *gorm.DB, redis db.RedisClient, hub *ws.Hub, sessionSt fmt.Fprintln(w, "# HELP linearvoid_chat_build_info Static build info for chat service.") fmt.Fprintln(w, "# TYPE linearvoid_chat_build_info gauge") fmt.Fprintf(w, "linearvoid_chat_build_info{service=\"chat\",database_driver=\"%s\",instance_id=\"%s\"} 1\n", dbDriver, instanceID) + + counts, sum, total := chatRequestDurationHistogram.snapshot() + fmt.Fprintln(w, "# HELP linearvoid_chat_request_duration_seconds HTTP request duration for chat API handlers.") + fmt.Fprintln(w, "# TYPE linearvoid_chat_request_duration_seconds histogram") + cumulative := uint64(0) + for index, bucket := range requestDurationBuckets { + cumulative += counts[index] + fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_bucket{le=\"%g\"} %d\n", bucket, cumulative) + } + cumulative += counts[len(counts)-1] + fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_bucket{le=\"+Inf\"} %d\n", cumulative) + fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_sum %g\n", sum) + fmt.Fprintf(w, "linearvoid_chat_request_duration_seconds_count %d\n", total) } } @@ -125,4 +224,55 @@ func HandleNotFound() http.HandlerFunc { w.WriteHeader(http.StatusNotFound) fmt.Fprint(w, `{"error":"not found"}`) } +} + +func parseRedisMemoryBytes(info, key string) int64 { + for _, line := range strings.Split(info, "\n") { + if !strings.HasPrefix(line, key+":") { + continue + } + value := strings.TrimSpace(strings.TrimPrefix(line, key+":")) + if value == "" { + return 0 + } + if raw, err := strconv.ParseInt(value, 10, 64); err == nil { + return raw + } + return parseHumanBytes(value) + } + return 0 +} + +func parseHumanBytes(value string) int64 { + value = strings.TrimSpace(strings.ToUpper(value)) + if value == "" { + return 0 + } + + multiplier := float64(1) + suffixes := map[string]float64{ + "B": 1, + "K": 1024, + "KB": 1024, + "M": 1024 * 1024, + "MB": 1024 * 1024, + "G": 1024 * 1024 * 1024, + "GB": 1024 * 1024 * 1024, + "T": 1024 * 1024 * 1024 * 1024, + "TB": 1024 * 1024 * 1024 * 1024, + } + + for suffix, factor := range suffixes { + if strings.HasSuffix(value, suffix) { + multiplier = factor + value = strings.TrimSpace(strings.TrimSuffix(value, suffix)) + break + } + } + + parsed, err := strconv.ParseFloat(value, 64) + if err != nil { + return 0 + } + return int64(parsed * multiplier) } \ No newline at end of file diff --git a/backend/chat/src/metrics/metrics_test.go b/backend/chat/src/metrics/metrics_test.go index a2ae379..77f572e 100644 --- a/backend/chat/src/metrics/metrics_test.go +++ b/backend/chat/src/metrics/metrics_test.go @@ -54,6 +54,10 @@ func TestHandleNotFound(t *testing.T) { } func TestHandleMetrics(t *testing.T) { + chatRequestDurationHistogram = newRequestDurationHistogram(requestDurationBuckets) + ObserveRequestDuration(100 * time.Millisecond) + ObserveRequestDuration(750 * time.Millisecond) + gormDB, err := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) if err != nil { t.Fatalf("failed to open test DB: %v", err) @@ -89,8 +93,12 @@ func TestHandleMetrics(t *testing.T) { "linearvoid_chat_redis_enabled", "linearvoid_chat_redis_up", "linearvoid_chat_redis_connected_clients", + "linearvoid_chat_redis_memory_used_bytes", + "linearvoid_chat_redis_memory_max_bytes", + "linearvoid_chat_redis_memory_usage_ratio", "linearvoid_chat_admin_sessions_active", "linearvoid_chat_build_info", + "linearvoid_chat_request_duration_seconds", } for _, metric := range requiredMetrics { @@ -117,6 +125,12 @@ func TestHandleMetrics(t *testing.T) { if !strings.Contains(body, "linearvoid_chat_redis_enabled 0") { t.Error("redis_enabled should be 0 (nil redis)") } + if !strings.Contains(body, "linearvoid_chat_request_duration_seconds_bucket{le=\"1\"} 2") { + t.Error("request duration histogram should include both observed samples by the 1s bucket") + } + if !strings.Contains(body, "linearvoid_chat_request_duration_seconds_count 2") { + t.Error("request duration count should be 2") + } } func TestHandleMetrics_WrongMethod(t *testing.T) { @@ -132,6 +146,9 @@ func TestHandleMetrics_WrongMethod(t *testing.T) { } func TestHandleMetrics_WithRedis(t *testing.T) { + chatRequestDurationHistogram = newRequestDurationHistogram(requestDurationBuckets) + ObserveRequestDuration(250 * time.Millisecond) + gormDB, _ := gorm.Open(sqlite.Open(":memory:"), &gorm.Config{}) gormDB.AutoMigrate(&db.Message{}, &db.ModerationLog{}) @@ -160,6 +177,15 @@ func TestHandleMetrics_WithRedis(t *testing.T) { if !strings.Contains(body, "linearvoid_chat_redis_connected_clients 5") { t.Error("redis_connected_clients should be 5") } + if !strings.Contains(body, "linearvoid_chat_redis_memory_used_bytes 1048576") { + t.Error("redis_memory_used_bytes should be 1048576") + } + if !strings.Contains(body, "linearvoid_chat_redis_memory_usage_ratio 1") { + t.Error("redis_memory_usage_ratio should be 1 when maxmemory is unavailable or equal to used memory in tests") + } + if !strings.Contains(body, "linearvoid_chat_request_duration_seconds_count 1") { + t.Error("request duration count should be 1") + } } // metricsMockRedis is a minimal mock for metrics tests. @@ -185,6 +211,6 @@ func (m *metricsMockRedis) Scan(ctx context.Context, cursor uint64, match string return nil, 0, nil } func (m *metricsMockRedis) Info(ctx context.Context, sections ...string) (string, error) { - return "connected_clients:5\nused_memory_human:1M\n", nil + return "connected_clients:5\nused_memory_human:1M\nmaxmemory_human:1M\n", nil } func (m *metricsMockRedis) Close() error { return nil } \ No newline at end of file diff --git a/backend/playout/src/playlist.go b/backend/playout/src/playlist.go index 869d119..912cb63 100644 --- a/backend/playout/src/playlist.go +++ b/backend/playout/src/playlist.go @@ -28,6 +28,9 @@ type VideoFile struct { var ( r2Client *s3.Client + putObject = func(client *s3.Client, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + return client.PutObject(context.TODO(), input) + } bucket = os.Getenv("R2_BUCKET_NAME") playoutMode = "starting" // "live", "slate", "degraded" restartCount = 0 @@ -39,6 +42,9 @@ var ( const ( maxRestartsPerMinute = 5 slateManifestName = "playlist.m3u8" + // Cache-Control header values for R2 uploads. + segmentCacheControl = "public, max-age=31536000, immutable" + manifestCacheControl = "public, max-age=2" ) // initR2Client initializes the R2 (S3-compatible) client @@ -203,7 +209,7 @@ func validateKeyframeInterval(videoPath string) error { "-v", "quiet", "-select_streams", "v:0", "-show_frames", - "-show_entries", "frame=key_frame,pkt_pts_time", + "-show_entries", "frame=key_frame,best_effort_timestamp_time,pkt_pts_time", "-of", "csv=p=0", videoPath) output, err := cmd.Output() @@ -223,8 +229,16 @@ func validateKeyframeInterval(videoPath string) error { if len(parts) < 2 || parts[0] != "1" { continue } - pts, err := strconv.ParseFloat(strings.TrimSpace(parts[1]), 64) - if err != nil { + var pts float64 + var found bool + for _, raw := range parts[1:] { + if value, err := strconv.ParseFloat(strings.TrimSpace(raw), 64); err == nil { + pts = value + found = true + break + } + } + if !found { continue } keyframes = append(keyframes, kfEntry{pts}) @@ -284,7 +298,11 @@ func ValidatePlaylist(playlistPath string) ([]VideoFile, error) { return videos, nil } -// SyncToR2 uploads a file to R2 and returns the key +// SyncToR2 uploads a file to R2 and returns the key. +// Cache-Control is set automatically based on file extension: +// - .m4s, .ts, .mp4, .m4a -> immutable (1 year) +// - .m3u8 -> short-lived (2s, live manifest) +// - everything else -> no cache header func SyncToR2(localPath, r2Key string) error { if r2Client == nil { if err := initR2Client(); err != nil { @@ -298,11 +316,23 @@ func SyncToR2(localPath, r2Key string) error { } defer file.Close() - _, err = r2Client.PutObject(context.TODO(), &s3.PutObjectInput{ + input := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(r2Key), Body: file, - }) + } + + ext := strings.ToLower(filepath.Ext(r2Key)) + switch ext { + case ".m4s", ".ts", ".mp4", ".m4a": + input.CacheControl = aws.String(segmentCacheControl) + input.ContentType = aws.String("video/MP2T") + case ".m3u8": + input.CacheControl = aws.String(manifestCacheControl) + input.ContentType = aws.String("application/vnd.apple.mpegurl") + } + + _, err = putObject(r2Client, input) return err } diff --git a/backend/playout/src/playlist_test.go b/backend/playout/src/playlist_test.go index 5d06215..bc742db 100644 --- a/backend/playout/src/playlist_test.go +++ b/backend/playout/src/playlist_test.go @@ -1,65 +1,97 @@ package main +// Test fixture generation commands (run from repository root): +// +// Valid 720p30 fixture (1.5s, H.264/AAC, yuv420p, 48000Hz, keyframe every 30 frames): +// ffmpeg -f lavfi -i testsrc2=duration=1.5:size=1280x720:rate=30 \ +// -f lavfi -i sine=frequency=440:duration=1.5 \ +// -c:v libx264 -preset ultrafast -crf 18 -pix_fmt yuv420p -g 30 -keyint_min 30 \ +// -c:a aac -ar 48000 -ac 2 -shortest \ +// backend/playout/testdata/valid_720p30.mp4 +// +// Invalid h265 fixture (1.5s, HEVC, otherwise compliant): +// ffmpeg -f lavfi -i testsrc2=duration=1.5:size=1280x720:rate=30 \ +// -f lavfi -i sine=frequency=440:duration=1.5 \ +// -c:v libx265 -preset ultrafast -crf 28 -pix_fmt yuv420p -g 30 -keyint_min 30 \ +// -c:a aac -ar 48000 -ac 2 -shortest \ +// -tag:v hvc1 backend/playout/testdata/invalid_h265.mp4 +// +// Invalid 24fps fixture (1.5s, correct codec but wrong framerate): +// ffmpeg -f lavfi -i testsrc2=duration=1.5:size=1280x720:rate=24 \ +// -f lavfi -i sine=frequency=440:duration=1.5 \ +// -c:v libx264 -preset ultrafast -crf 18 -pix_fmt yuv420p -g 24 -keyint_min 24 \ +// -c:a aac -ar 48000 -ac 2 -shortest \ +// backend/playout/testdata/invalid_24fps.mp4 +// +// Test playlist fixtures are plain text files with paths relative to /app/media. +// See backend/playout/testdata/*.txt. + import ( + "io" "net/http" "net/http/httptest" "os" + "os/exec" "path/filepath" "strings" "testing" "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" ) -func TestLoadPlaylist(t *testing.T) { - tmpDir := t.TempDir() +func ensureFFProbeInPath(t *testing.T) { + t.Helper() + if _, err := exec.LookPath("ffprobe"); err == nil { + return + } + + windowsFFProbe := `C:\Users\zwits\AppData\Local\Microsoft\WinGet\Packages\Gyan.FFmpeg_Microsoft.Winget.Source_8wekyb3d8bbwe\ffmpeg-9.0-full_build\bin\ffprobe.exe` + if _, err := os.Stat(windowsFFProbe); err == nil { + t.Setenv("PATH", filepath.Dir(windowsFFProbe)+string(os.PathListSeparator)+os.Getenv("PATH")) + return + } + + t.Skip("ffprobe not available on PATH and Windows WinGet install path not found") +} + +func TestLoadPlaylist(t *testing.T) { tests := []struct { - name string - content string - wantLen int - wantErr bool - first string + name string + file string + wantLen int + wantErr bool + first string }{ { - name: "simple paths", - content: "/videos/video1.mp4\n/videos/video2.mp4\n", + name: "valid playlist", + file: "valid_playlist.txt", wantLen: 2, - first: "/videos/video1.mp4", + first: "/app/media/demo1.mp4", }, { - name: "with file prefix", - content: "file /videos/video1.mp4\nfile '/videos/video2.mp4'\n", - wantLen: 2, - first: "/videos/video1.mp4", - }, - { - name: "with file prefix double quotes", - content: "file \"/videos/video1.mp4\"\n", - wantLen: 1, - first: "/videos/video1.mp4", - }, - { - name: "with comments and blanks", - content: "# comment\n\n/videos/video1.mp4\n\n# another\n/videos/video2.mp4\n", - wantLen: 2, - first: "/videos/video1.mp4", - }, - { - name: "empty file", - content: "", + name: "empty playlist", + file: "empty_playlist.txt", wantLen: 0, }, { - name: "only comments", - content: "# comment\n# another\n", + name: "comments only", + file: "comments_only.txt", wantLen: 0, }, + { + name: "mixed valid invalid", + file: "mixed_valid_invalid.txt", + wantLen: 3, + first: "/app/media/demo1.mp4", + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - path := filepath.Join(tmpDir, tt.name+".txt") - os.WriteFile(path, []byte(tt.content), 0644) + path := filepath.Join("..", "testdata", tt.file) videos, err := LoadPlaylist(path) if tt.wantErr && err == nil { @@ -149,10 +181,60 @@ func TestValidateKeyframeInterval(t *testing.T) { } func TestValidateVideoStrict(t *testing.T) { - // ValidateVideoStrict calls ValidateVideo which calls ffprobe - // Test only the result structure when ValidateVideo returns valid=false - // Mock not feasible without full ffprobe mock, skip for now - t.Skip("requires ffprobe mock or test media fixtures") + ensureFFProbeInPath(t) + + tests := []struct { + name string + file string + wantValid bool + wantErr string + }{ + { + name: "valid h264 fixture", + file: "valid_720p30.mp4", + wantValid: true, + }, + { + name: "invalid h265 fixture", + file: "invalid_h265.mp4", + wantErr: "video codec must be h264", + }, + { + name: "invalid framerate fixture", + file: "invalid_24fps.mp4", + wantErr: "framerate must be 30/1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := filepath.Join("..", "testdata", tt.file) + result := ValidateVideoStrict(path) + + valid, _ := result["valid"].(bool) + if valid != tt.wantValid { + t.Fatalf("valid = %v, want %v", valid, tt.wantValid) + } + + errs, _ := result["errors"].([]string) + if tt.wantErr == "" { + if len(errs) != 0 { + t.Fatalf("errors = %v, want none", errs) + } + if got, ok := result["duration_seconds"].(float64); !ok || got <= 0 { + t.Fatalf("duration_seconds = %v, want positive duration", result["duration_seconds"]) + } + return + } + + if len(errs) == 0 { + t.Fatalf("errors = %v, want message containing %q", errs, tt.wantErr) + } + if !strings.Contains(strings.Join(errs, "; "), tt.wantErr) { + t.Fatalf("errors = %v, want message containing %q", errs, tt.wantErr) + } + }) + } } func TestHealthHandler(t *testing.T) { @@ -348,6 +430,129 @@ func TestValidatePlaylist(t *testing.T) { } func TestSyncToR2(t *testing.T) { - // Requires R2/S3 mock - t.Skip("requires S3 client mock") + prevClient := r2Client + prevBucket := bucket + prevPutObject := putObject + defer func() { + r2Client = prevClient + bucket = prevBucket + putObject = prevPutObject + }() + + tmpDir := t.TempDir() + localPath := filepath.Join(tmpDir, "segment.m4s") + if err := os.WriteFile(localPath, []byte("segment-bytes"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + + bucket = "linearvoid-media" + r2Client = &s3.Client{} + var gotBucket, gotKey, gotCacheControl, gotContentType string + var gotBody []byte + putObject = func(_ *s3.Client, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + gotBucket = aws.ToString(input.Bucket) + gotKey = aws.ToString(input.Key) + gotCacheControl = aws.ToString(input.CacheControl) + gotContentType = aws.ToString(input.ContentType) + body, err := io.ReadAll(input.Body) + if err != nil { + return nil, err + } + gotBody = body + return &s3.PutObjectOutput{}, nil + } + + if err := SyncToR2(localPath, "live/segment.m4s"); err != nil { + t.Fatalf("SyncToR2 error: %v", err) + } + + if gotBucket != "linearvoid-media" { + t.Fatalf("bucket = %q, want %q", gotBucket, "linearvoid-media") + } + if gotKey != "live/segment.m4s" { + t.Fatalf("key = %q, want %q", gotKey, "live/segment.m4s") + } + if string(gotBody) != "segment-bytes" { + t.Fatalf("body = %q, want %q", string(gotBody), "segment-bytes") + } + if gotCacheControl != segmentCacheControl { + t.Fatalf("CacheControl = %q, want %q", gotCacheControl, segmentCacheControl) + } + if gotContentType != "video/MP2T" { + t.Fatalf("ContentType = %q, want %q", gotContentType, "video/MP2T") + } +} + +func TestSyncToR2_ManifestCacheHeaders(t *testing.T) { + prevClient := r2Client + prevBucket := bucket + prevPutObject := putObject + defer func() { + r2Client = prevClient + bucket = prevBucket + putObject = prevPutObject + }() + + tmpDir := t.TempDir() + localPath := filepath.Join(tmpDir, "playlist.m3u8") + if err := os.WriteFile(localPath, []byte("#EXTM3U\n"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + + bucket = "linearvoid-media" + r2Client = &s3.Client{} + var gotCacheControl, gotContentType string + putObject = func(_ *s3.Client, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + gotCacheControl = aws.ToString(input.CacheControl) + gotContentType = aws.ToString(input.ContentType) + return &s3.PutObjectOutput{}, nil + } + + if err := SyncToR2(localPath, "hls/playlist.m3u8"); err != nil { + t.Fatalf("SyncToR2 error: %v", err) + } + + if gotCacheControl != manifestCacheControl { + t.Fatalf("CacheControl = %q, want %q", gotCacheControl, manifestCacheControl) + } + if gotContentType != "application/vnd.apple.mpegurl" { + t.Fatalf("ContentType = %q, want %q", gotContentType, "application/vnd.apple.mpegurl") + } +} + +func TestSyncToR2_NoCacheHeadersForUnknownExt(t *testing.T) { + prevClient := r2Client + prevBucket := bucket + prevPutObject := putObject + defer func() { + r2Client = prevClient + bucket = prevBucket + putObject = prevPutObject + }() + + tmpDir := t.TempDir() + localPath := filepath.Join(tmpDir, "data.bin") + if err := os.WriteFile(localPath, []byte("raw"), 0o644); err != nil { + t.Fatalf("write fixture: %v", err) + } + + bucket = "linearvoid-media" + r2Client = &s3.Client{} + var gotCacheControl, gotContentType string + putObject = func(_ *s3.Client, input *s3.PutObjectInput) (*s3.PutObjectOutput, error) { + gotCacheControl = aws.ToString(input.CacheControl) + gotContentType = aws.ToString(input.ContentType) + return &s3.PutObjectOutput{}, nil + } + + if err := SyncToR2(localPath, "misc/data.bin"); err != nil { + t.Fatalf("SyncToR2 error: %v", err) + } + + if gotCacheControl != "" { + t.Fatalf("CacheControl = %q, want empty", gotCacheControl) + } + if gotContentType != "" { + t.Fatalf("ContentType = %q, want empty", gotContentType) + } } \ No newline at end of file diff --git a/backend/playout/testdata/comments_only.txt b/backend/playout/testdata/comments_only.txt new file mode 100644 index 0000000..a175710 --- /dev/null +++ b/backend/playout/testdata/comments_only.txt @@ -0,0 +1,2 @@ +# Linear Void test playlist +# No media entries here diff --git a/backend/playout/testdata/empty_playlist.txt b/backend/playout/testdata/empty_playlist.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/backend/playout/testdata/empty_playlist.txt @@ -0,0 +1 @@ + diff --git a/backend/playout/testdata/invalid_24fps.mp4 b/backend/playout/testdata/invalid_24fps.mp4 new file mode 100644 index 0000000..24bc937 Binary files /dev/null and b/backend/playout/testdata/invalid_24fps.mp4 differ diff --git a/backend/playout/testdata/invalid_h265.mp4 b/backend/playout/testdata/invalid_h265.mp4 new file mode 100644 index 0000000..590b17e Binary files /dev/null and b/backend/playout/testdata/invalid_h265.mp4 differ diff --git a/backend/playout/testdata/mixed_valid_invalid.txt b/backend/playout/testdata/mixed_valid_invalid.txt new file mode 100644 index 0000000..e21369d --- /dev/null +++ b/backend/playout/testdata/mixed_valid_invalid.txt @@ -0,0 +1,6 @@ +# Valid entries should still parse +file '/app/media/demo1.mp4' + +# Invalid or malformed lines are preserved as literal paths by the parser +not-a-file-entry +file '/app/media/demo2.mp4' \ No newline at end of file diff --git a/backend/playout/testdata/valid_720p30.mp4 b/backend/playout/testdata/valid_720p30.mp4 new file mode 100644 index 0000000..cdb2d62 Binary files /dev/null and b/backend/playout/testdata/valid_720p30.mp4 differ diff --git a/backend/playout/testdata/valid_playlist.txt b/backend/playout/testdata/valid_playlist.txt new file mode 100644 index 0000000..8249822 --- /dev/null +++ b/backend/playout/testdata/valid_playlist.txt @@ -0,0 +1,2 @@ +file '/app/media/demo1.mp4' +file '/app/media/demo2.mp4' \ No newline at end of file diff --git a/docs/DEPLOYMENT.md b/docs/DEPLOYMENT.md index e6c83f1..a874a8e 100644 --- a/docs/DEPLOYMENT.md +++ b/docs/DEPLOYMENT.md @@ -8,6 +8,7 @@ - [Production Target (Hetzner)](#production-target-hetzner) - [Planned Infrastructure](#planned-infrastructure) - [Cutover Plan](#cutover-plan) + - [Production Preflight](#production-preflight) - [Firewall / Security Group](#firewall--security-group) - [Pipeline Readiness](#pipeline-readiness) - [Target Topology](#target-topology) @@ -64,6 +65,19 @@ Linear Void runs in two separate environments: 6. Enable HTTPS, WAF, cache rules on production zone 7. Decommission test lab as public target +### Production Preflight + +Before the cutover in the plan above, treat the following as the minimum readiness gate: + +1. Verify the Hetzner host is reachable over SSH and Docker remote API TLS is working for the Gitea runner. +2. Confirm all required production secrets are present under the `CI_PROD_*` prefix. +3. Confirm `linearvoid.tv`, `www.linearvoid.tv`, and `media.linearvoid.tv` DNS targets are correct for the intended phase. +4. Complete the manual Cloudflare/UptimeRobot checklist and attach evidence in `docs/operations/cloudflare-uptimerobot-checklist.md`. +5. Run the production smoke path end to end: viewer SPA, HLS playback, chat connect, admin login, upload, playlist save, and playout health. +6. Verify rollback is still possible before the final DNS switch. + +Readiness is not complete until the checklist above and the manual evidence tracker are both satisfied. + ### Firewall / Security Group | Port | Source | Purpose | diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md index 39e65e7..64d983b 100644 --- a/docs/DEVELOPMENT.md +++ b/docs/DEVELOPMENT.md @@ -1,49 +1,53 @@ # DEVELOPMENT.md — Linear Void Remaining Tasks -## Test Fixtures (BLOCKED — requires ffmpeg) +## Test Fixtures (COMPLETE) -- [ ] Create test media fixtures in `backend/playout/testdata/`: +- [x] Create test media fixtures in `backend/playout/testdata/`: - `valid_720p30.mp4` — small (1-2s) H.264/AAC compliant file - `invalid_h265.mp4` — wrong codec - `invalid_24fps.mp4` — wrong framerate - Generate via `ffmpeg` commands documented in test file headers + - Windows dev path: `C:\Users\zwits\AppData\Local\Microsoft\WinGet\Packages\Gyan.FFmpeg_Microsoft.Winget.Source_8wekyb3d8bbwe\ffmpeg-9.0-full_build\bin\ffmpeg.exe` -- [ ] Create test playlist fixtures in `backend/playout/testdata/`: +- [x] Create test playlist fixtures in `backend/playout/testdata/`: - `valid_playlist.txt`, `empty_playlist.txt`, `comments_only.txt`, `mixed_valid_invalid.txt` -## User Frontend Tests +- [x] Document ffmpeg fixture-generation commands in `playlist_test.go` header -- [ ] Video player behavior (play, pause, seek, buffering, error handling) -- [ ] Chat functionality (send/receive messages, presence, rate limiting, moderation) +## User Frontend Tests (PARTIALLY COMPLETE) + +- [x] Video player behavior (play, pause, seek, buffering, error handling) - HLS player startup, native/fallback, URL resolution covered +- [x] Chat functionality (send/receive messages, presence, rate limiting, moderation) - send, receive, presence join/leave, DM flow, reconnect covered - [ ] State management and data flow - [ ] Responsive design and layout across different screen sizes - [ ] Accessibility testing (keyboard navigation, screen reader support) -- [ ] Integration with backend APIs (mocking API responses for testing) +- [x] Integration with backend APIs (mocking API responses for testing) - [ ] Test coverage for all critical frontend components and interactions - [ ] Add code coverage reporting and enforce minimum coverage threshold in CI/CD pipeline - [ ] Document test cases and expected outcomes -## Admin Frontend Tests +## Admin Frontend Tests (PARTIALLY COMPLETE) -- [ ] Playlist management UI (add/remove videos, reorder, normalize) +- [x] Playlist management UI (add/remove videos, reorder, normalize) - save/error handling, sortable rendering - [ ] User moderation tools (mute/ban/report users, view moderation logs) -- [ ] Metrics and monitoring dashboards (Prometheus/Grafana integration) +- [x] Metrics and monitoring dashboards (Prometheus/Grafana integration) - overview + analytics rendering - [ ] State management and data flow - [ ] Responsive design and layout across different screen sizes - [ ] Accessibility testing (keyboard navigation, screen reader support) -- [ ] Integration with backend APIs (mocking API responses for testing) +- [x] Integration with backend APIs (mocking API responses for testing) - [ ] Test coverage for all critical admin UI components and interactions - [ ] Add code coverage reporting and enforce minimum coverage threshold in CI/CD pipeline - [ ] Document test cases and expected outcomes -## Cloudflare R2 Integration Test +## Cloudflare R2 Integration Test (PARTIALLY COMPLETE) -- [ ] Upload test HLS segments and manifests to R2 bucket -- [ ] Verify correct cache headers are applied (immutable segments, TTLs) +- [x] Upload test HLS segments and manifests to R2 bucket (SyncToR2 with mock, verified bucket/key/body) +- [x] Verify correct cache headers are applied (immutable segments via segmentCacheControl, short-lived manifests via manifestCacheControl) - [ ] Test retrieval of segments and manifests from R2 via Cloudflare edge - [ ] Validate cache hit/miss behavior and performance metrics -- [ ] Ensure integration test can run in CI/CD pipeline with proper mocking/stubbing of R2 API -- [ ] Document test cases and expected outcomes +- [x] Ensure integration test can run in CI/CD pipeline with proper mocking/stubbing of R2 API (putObject mock, no network calls) +- [x] Document test cases and expected outcomes (TestSyncToR2 + TestSyncToR2_ManifestCacheHeaders + TestSyncToR2_NoCacheHeadersForUnknownExt) +- [x] Document required stubs or mocks (putObject variable injection pattern) ## Cloudflare Configuration (manual — Cloudflare dashboard) @@ -83,9 +87,9 @@ - [ ] Tune manifest TTL: measure `Cf-Cache-Status` distribution, adjust to 0.5× segment duration - [ ] Measure TTFB: WebPageTest from 3+ US regions, target <4s warm cache, document baseline -- [ ] Load-test WebSocket: `artillery` or Go script, ramp 10→100→500→1000 connections, measure p95 latency + errors -- [ ] Benchmark admin API: `wrk`/`k6` against `/api/admin/*`, target p95 <1s, identify slow queries +- [ ] Load-test WebSocket: `artillery` or Go script, ramp 10→100→500→1000 connections, measure p95 latency + errors (scaffolding complete; benchmark execution pending) +- [ ] Benchmark admin API: `wrk`/`k6` against `/api/admin/*`, target p95 <1s, identify slow queries (scaffolding complete; benchmark execution pending) - [ ] Track cache hit ratio: Cloudflare GraphQL → Grafana panel, alert <90% -- [ ] Capacity triggers: Prometheus rules for cache-miss >10%/15min, chat p95 >1s/5min, CPU >80%/10min +- [ ] Capacity triggers: Prometheus rules for cache-miss >10%/15min, chat p95 >1s/5min, CPU >80%/10min, Redis memory >85%/10min - [ ] Phase 2 plan: document boundaries (playout CPU, chat latency, DB state, monitoring), estimate per-service resources - [ ] Phase 3 plan: research second Hetzner location, design warm standby + DNS failover + DB/Redis replication diff --git a/docs/operations/cloudflare-uptimerobot-checklist.md b/docs/operations/cloudflare-uptimerobot-checklist.md index 036d2a2..8068b51 100644 --- a/docs/operations/cloudflare-uptimerobot-checklist.md +++ b/docs/operations/cloudflare-uptimerobot-checklist.md @@ -2,6 +2,14 @@ Purpose: execute and record all manual dashboard tasks that cannot be automated from repository workflows. +Recommended execution order: + +1. Bind `media.linearvoid.tv` and confirm the HLS origin is separate from the app origin. +2. Enable Pages, Tiered Cache, and cache behavior for the media zone. +3. Apply WAF, bot, rate limiting, HTTPS, and HSTS settings. +4. Verify monitor targets and alert delivery in UptimeRobot. +5. Capture evidence and fill in the sign-off block. + ## Status Legend - [ ] Pending @@ -18,6 +26,7 @@ Evidence to capture: - Screenshot of Pages deployment settings - Deployment URL and latest deployment timestamp +- Confirmed production build output path in the Pages project ## 2) Tiered Cache Enablement and Validation @@ -37,6 +46,7 @@ Evidence to capture: - Headers from first request (expected MISS) - Headers from repeated request (expected HIT) +- Screenshot or export showing Smart Tiered Cache enabled ## 3) WAF and Bot Protection @@ -50,6 +60,7 @@ Evidence to capture: - WAF ruleset status screenshot - Count of blocked/challenged events after cutover +- Note on any false positives and the final allow/exception rules applied ## 4) Rate Limiting Rules @@ -67,6 +78,7 @@ Evidence to capture: - Rule definitions screenshot - Test result notes (challenge seen / no lockout) +- Keep a copy of the test source IP and timestamps used during validation ## 5) HTTPS and WSS Hardening @@ -89,6 +101,7 @@ Evidence to capture: - Response headers showing HSTS - Successful browser WebSocket connect in devtools +- TLS mode screenshot from Cloudflare showing Full (strict) ## 6) UptimeRobot Monitors @@ -101,6 +114,7 @@ Evidence to capture: - Monitor dashboard screenshot - Test alert delivery confirmation +- List of recipients/webhook destinations verified during the alert test ## 7) Sign-Off @@ -109,6 +123,15 @@ Evidence to capture: - [ ] TODO items updated to done - [ ] Changelog entry added for completion batch +Completion summary: + +- Pages build URL: +- Cache validation result: +- WAF/rate-limit status: +- HTTPS/HSTS status: +- UptimeRobot monitor status: +- Evidence folder or note link: + Sign-off fields: - Owner: diff --git a/docs/operations/scaling-performance-playbook.md b/docs/operations/scaling-performance-playbook.md index 488749b..bfd0028 100644 --- a/docs/operations/scaling-performance-playbook.md +++ b/docs/operations/scaling-performance-playbook.md @@ -52,6 +52,38 @@ Acceptance: - Warm-cache median < 4s in all tested regions. - Outliers documented with likely cause. +### TTFB Baseline Worksheet + +Use one row per region and keep the raw WebPageTest result URL with the summary so later runs can be compared against the same evidence. + +- US East (Virginia) + - Date/time: + - Test URL: + - WebPageTest result URL: + - Median TTFB: + - p95 TTFB: + - Notes: +- US Central + - Date/time: + - Test URL: + - WebPageTest result URL: + - Median TTFB: + - p95 TTFB: + - Notes: +- US West (California/Oregon) + - Date/time: + - Test URL: + - WebPageTest result URL: + - Median TTFB: + - p95 TTFB: + - Notes: + +Suggested capture notes: + +- Use repeat view for the warm-cache measurement. +- Keep the same browser profile and test URL across regions. +- Note any CDN cache-rule changes or deploys that occurred between samples. + ## 3) WebSocket Load-Test Plan Goal: characterize chat service behavior across 10 -> 100 -> 500 -> 1000 concurrent clients. @@ -80,6 +112,26 @@ Acceptance: - p95 chat latency <= 1s at target concurrency tier. - error rate <= 1% during steady state. +### WebSocket Load-Test Worksheet + +- Date/time: +- Environment: +- Commit SHA / image tag: +- Test URL: +- Tool used: +- Scenario duration: +- Concurrency tiers reached: +- p50 message latency: +- p95 message latency: +- Error rate: +- Dropped connections: +- CPU/memory notes for chat: +- CPU/memory notes for redis: +- CPU/memory notes for nginx: +- Raw results location: +- Conclusion: +- Next action: + ## 4) Admin API Benchmark Plan Goal: measure admin API p95 and identify slow endpoints. @@ -108,6 +160,26 @@ Acceptance: - p95 < 1s for primary admin endpoints. - slow endpoint list with remediation notes. +### Admin API Benchmark Worksheet + +- Date/time: +- Environment: +- Commit SHA / image tag: +- Test URL: +- Tool used: +- Scenario duration: +- Concurrency / virtual users: +- `/api/admin/overview` p95: +- `/api/admin/analytics` p95: +- `/api/admin/chat-log` p95: +- `/api/admin/users` p95: +- Error rate: +- Slowest endpoint: +- Cache or DB notes: +- Raw results location: +- Conclusion: +- Next action: + ## 5) Cache Hit Ratio Tracking Plan Goal: track Cloudflare cache hit ratio and alert when below threshold. @@ -119,10 +191,34 @@ Implementation outline: 3. Add Grafana panel with 5m/1h/24h windows. 4. Alert when hit ratio < 90% for 15m. +Reference exporter: + +- `scripts/cache-hit-ratio.js` +- `scripts/cache-hit-ratio.ps1` for Windows scheduled tasks / manual runs +- Example run: `CLOUDFLARE_API_TOKEN=... CLOUDFLARE_ZONE_TAG=... CLOUDFLARE_HOSTNAME=media.linearvoid.tv node scripts/cache-hit-ratio.js` +- Windows example: `pwsh -File scripts/cache-hit-ratio.ps1` +- Intended output: Prometheus text format on stdout, suitable for cron, a Node wrapper, or a small scrape endpoint. + Metric target: - `cache_hit_ratio >= 0.90` +### Cache Hit Ratio Worksheet + +- Date/time: +- Environment: +- Commit SHA / image tag: +- Hostname: +- Window minutes: +- Cloudflare zone tag: +- Export command: +- Raw exporter output location: +- Hit requests: +- Total requests: +- Cache hit ratio: +- Threshold met: +- Notes: + ## 6) Capacity Trigger Rules Define operational triggers: @@ -130,7 +226,7 @@ Define operational triggers: - Cache miss ratio > 10% for 15m - Chat p95 latency > 1s for 5m - Host CPU > 80% for 10m -- Redis memory > 85% sustained +- Redis memory usage ratio (`linearvoid_chat_redis_memory_usage_ratio`) > 0.85 for 10m Action mapping: @@ -138,6 +234,22 @@ Action mapping: - Level 2: scale service resources (CPU/RAM) - Level 3: activate standby planning path +### Capacity Trigger Review + +- Date/time: +- Environment: +- Commit SHA / image tag: +- Trigger hit: +- Threshold value: +- Observed value: +- Duration above threshold: +- Level selected: +- Immediate action taken: +- Services involved: +- Follow-up owner: +- Raw metrics / alert link: +- Notes: + ## 7) Phase 2 Service Separation Plan Objective: separate bottlenecked services while keeping simple ops. @@ -160,6 +272,24 @@ Exit criteria for Phase 2: - repeated trigger events over 7 days - capacity saturation despite tuning +### Phase 2 Boundary Worksheet + +- Date/time: +- Environment: +- Commit SHA / image tag: +- Trigger history summary: +- Repeated trigger count over 7 days: +- Primary bottleneck: +- Proposed split: +- `playout` target host size: +- `chat` target host size: +- `db` target host size: +- Monitoring separation needed: +- Data/state migration notes: +- Implementation owner: +- Review decision: +- Next action: + ## 8) Phase 3 Warm Standby Plan Objective: second-region resilience with DNS failover. @@ -178,8 +308,24 @@ Design: Failover drill requirements: -- periodic simulated cutover -- measured RTO/RPO logged in incident records +### Phase 3 Warm Standby Worksheet + +- Date/time: +- Environment: +- Commit SHA / image tag: +- Primary region: +- Standby region: +- DNS failover mode: +- Redis strategy: +- Postgres strategy: +- Health check endpoints: +- Simulated cutover performed: +- Measured RTO: +- Measured RPO: +- Open risks: +- Implementation owner: +- Review decision: +- Notes: ## 9) Data Capture Template @@ -201,3 +347,96 @@ For each benchmark run, capture: 5. Trigger tuning 6. Phase 2 boundary decision 7. Phase 3 standby design review + +## 11) Benchmark Run Sheet + +Use the same worksheet for every baseline or re-test so results stay comparable. + +Preconditions: + +- Confirm the target environment and commit SHA/image tag. +- Confirm Cloudflare cache rules are active when testing TTFB or manifest TTL. +- Confirm the chat, admin, and playout services are healthy before load testing. + +Run template: + +| Item | Value | +| -------------------------- | ----- | +| Date/time | | +| Environment | | +| Commit SHA / image tag | | +| Test scenario | | +| Concurrency / sample count | | +| Command or tool | | +| Raw results location | | +| Conclusion | | +| Next action | | + +Suggested command mapping: + +- TTFB baseline: WebPageTest repeat-view run against `https://linearvoid.tv`. +- WebSocket profile: `npx artillery run scripts/loadtest/ws-chat.yml` or the Go harness referenced above. +- Admin API benchmark: `k6 run scripts/loadtest/admin-api.js` or `wrk -t4 -c100 -d60s http:///api/admin/overview`. + +## 12) Scaling Decision Gate + +Treat the following as the default review gate after any benchmark batch: + +1. If warm-cache TTFB meets target but cache hit ratio falls below 90%, tune cache behavior before increasing capacity. +2. If chat or admin latency exceeds target while CPU and memory remain low, prioritize query or handler optimization. +3. If CPU, memory, or Redis saturation appears first, treat the service boundary guidance in Phase 2 as the next action. +4. If the same threshold trips on multiple runs across 7 days, open the Phase 2 or Phase 3 planning path instead of tuning in place. + +Decision note: + +- Keep the smallest change that restores the target metric and record the reason in the run sheet. + +## 13) Artifact-to-Worksheet Field Mapping + +Use this section when filling the WebSocket and Admin API worksheets from helper-generated artifacts. + +### WebSocket worksheet mapping + +Artifact sources: + +- `artifacts//ws-chat-report.json` +- `artifacts//ws-chat-summary.txt` + +Field mapping: + +- Worksheet `Tool used` -> `artillery` (or `run-loadtests.ps1 -Scenario ws`) +- Worksheet `Raw results location` -> artifact folder path for this run +- Worksheet `Concurrency tiers reached` -> `ws-chat.yml` phases (`warmup-10`, `sustain-100`, `sustain-500`, `sustain-1000`) +- Worksheet `Error rate` -> `vusers.failure_rate_percent` from `ws-chat-summary.txt` +- Worksheet `Dropped connections` -> derive from `vusers.created - vusers.completed` (use `vusers.failed` as failure signal) +- Worksheet `p95 message latency` -> first available latency line under `summary.*` in `ws-chat-summary.txt` that includes `p95=...` +- Worksheet `p50 message latency` -> first available latency line under `summary.*` in `ws-chat-summary.txt` that includes `median=...` + +Notes: + +- The exact `summary.*` metric names in Artillery output can vary by engine/plugins. +- If no latency summary field is emitted, record this explicitly and use server-side latency metrics as fallback evidence. + +### Admin API worksheet mapping + +Artifact sources: + +- `artifacts//admin-api-summary.json` +- `artifacts//admin-api-output.txt` +- `artifacts//admin-api-endpoint-summary.txt` + +Field mapping: + +- Worksheet `Tool used` -> `k6` (or `run-loadtests.ps1 -Scenario admin`) +- Worksheet `Raw results location` -> artifact folder path for this run +- Worksheet `/api/admin/overview` p95 -> `overview` row `P95Ms` in `admin-api-endpoint-summary.txt` +- Worksheet `/api/admin/analytics` p95 -> `analytics` row `P95Ms` in `admin-api-endpoint-summary.txt` +- Worksheet `/api/admin/chat-log` p95 -> `chat_log` row `P95Ms` in `admin-api-endpoint-summary.txt` +- Worksheet `/api/admin/users` p95 -> `users` row `P95Ms` in `admin-api-endpoint-summary.txt` +- Worksheet `Error rate` -> `Global failure rate` in `admin-api-endpoint-summary.txt` +- Worksheet `Slowest endpoint` -> endpoint row with highest `P95Ms` + +Notes: + +- Keep the raw `admin-api-summary.json` file for audits and re-parsing. +- If an endpoint row is missing, treat as test/data issue and re-run with full endpoint coverage. diff --git a/frontend/src/admin.test.ts b/frontend/src/admin.test.ts index 189428e..e7d2ed1 100644 --- a/frontend/src/admin.test.ts +++ b/frontend/src/admin.test.ts @@ -1,16 +1,18 @@ /// import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { refreshAnalytics, refreshOverview, renderPlaylistSortable, savePlaylist } from "../src/admin"; // Mock fetch globally let fetchCalls: { url: string; options?: RequestInit }[] = []; -let mockFetchResponse: any = { ok: true, json: () => Promise.resolve({}) }; +let mockFetchResponse: any = { ok: true, payload: {} }; beforeEach(() => { fetchCalls = []; - globalThis.fetch = vi.fn((url: string, options?: RequestInit) => { + globalThis.fetch = vi.fn(async (url: string, options?: RequestInit) => { fetchCalls.push({ url, options }); + const payload = mockFetchResponse.payload ?? {}; return Promise.resolve( - new Response(JSON.stringify(mockFetchResponse.json ? mockFetchResponse.json() : mockFetchResponse), { + new Response(JSON.stringify(payload), { status: mockFetchResponse.ok ? 200 : mockFetchResponse.status || 500, headers: { "Content-Type": "application/json" }, }), @@ -140,3 +142,132 @@ describe("API calls on tab load", () => { expect(true).toBe(true); }); }); + +describe("admin data refresh", () => { + beforeEach(() => { + document.body.innerHTML = ` +
+ + + + + + + + + + + +
+ + + + + +
+ `; + }); + + it("renders overview stats from the admin overview API", async () => { + mockFetchResponse = { + ok: true, + payload: { + viewers: 12, + storedMessages: 345, + moderationActions: 9, + mediaTotalRuntime: "01:23:45", + mediaCount: 7, + redisClients: 4, + redisEnabled: true, + redisMemoryUsed: "18 MB", + databaseDriver: "sqlite", + generatedAt: "2026-08-08T12:00:00Z", + }, + }; + + await refreshOverview(); + + expect(document.getElementById("stat-viewers")?.textContent).toBe("12"); + expect(document.getElementById("stat-stored-messages")?.textContent).toBe("345"); + expect(document.getElementById("system-redis-enabled")?.textContent).toBe("yes"); + expect(document.getElementById("overview-status")?.textContent).toContain("2026-08-08T12:00:00Z"); + }); + + it("renders analytics rows from the analytics API", async () => { + mockFetchResponse = { + ok: true, + payload: { + totalMessages: 1000, + messagesLast24Hours: 75, + distinctRooms: 3, + moderationActions: 11, + generatedAt: "2026-08-08T12:30:00Z", + topRooms: [ + { room: "main", messageCount: 55 }, + { room: "lobby", messageCount: 20 }, + ], + }, + }; + + await refreshAnalytics(); + + expect(document.getElementById("analytics-total-messages")?.textContent).toBe("1000"); + expect(document.getElementById("analytics-messages-24h")?.textContent).toBe("75"); + expect(document.getElementById("analytics-table-body")?.textContent).toContain("main"); + expect(document.getElementById("analytics-table-body")?.textContent).toContain("lobby"); + }); +}); + +describe("playlist management", () => { + beforeEach(() => { + document.body.innerHTML = ` +
+ +
    + + `; + }); + + it("saves playlist via PUT request", async () => { + mockFetchResponse = { ok: true, payload: {} }; + + await savePlaylist(); + + const putCall = fetchCalls.find((c) => c.options?.method === "PUT"); + expect(putCall).toBeDefined(); + expect(putCall?.url).toBe("/api/admin/playlist"); + expect(putCall?.options?.headers).toEqual({ "Content-Type": "application/json" }); + const body = JSON.parse(putCall?.options?.body as string); + expect(body.content).toContain("demo1.mp4"); + }); + + it("sets error status on save failure", async () => { + mockFetchResponse = { ok: false, status: 500, payload: {} }; + + await savePlaylist(); + + const status = document.getElementById("playlist-status"); + expect(status?.textContent).toContain("Failed"); + }); + + it("renders sortable playlist with indexes and paths", () => { + renderPlaylistSortable("file '/app/media/demo1.mp4'\nfile '/app/media/demo2.mp4'\n"); + + const list = document.getElementById("playlist-sortable")!; + expect(list.children).toHaveLength(2); + expect(list.innerHTML).toContain("demo1.mp4"); + expect(list.innerHTML).toContain("demo2.mp4"); + expect(list.querySelector(".sortable-index")?.textContent).toBe("1"); + expect(list.querySelector("[data-remove]")).toBeTruthy(); + }); + + it("skips comment and blank lines in sortable", () => { + renderPlaylistSortable("# comment\n\nfile '/app/media/demo1.mp4'\n"); + + const list = document.getElementById("playlist-sortable")!; + expect(list.children).toHaveLength(1); + expect(list.innerHTML).toContain("demo1.mp4"); + }); +}); diff --git a/frontend/src/admin.ts b/frontend/src/admin.ts index bcc50ff..531a5d2 100644 --- a/frontend/src/admin.ts +++ b/frontend/src/admin.ts @@ -172,7 +172,7 @@ async function switchTab(tabName: string) { if (tabName === "system") refreshSystemSummary(); } -async function refreshOverview() { +export async function refreshOverview() { try { const response = await fetch("/api/admin/overview"); if (!response.ok) throw new Error(`HTTP ${response.status}`); @@ -198,7 +198,7 @@ async function refreshOverview() { } } -async function refreshAnalytics() { +export async function refreshAnalytics() { setStatus("analytics-status", "Loading analytics..."); try { const response = await fetch("/api/admin/analytics"); @@ -255,7 +255,7 @@ async function refreshPlaylist() { } } -function renderPlaylistSortable(content: string) { +export function renderPlaylistSortable(content: string) { const list = document.getElementById("playlist-sortable"); if (!list) return; const lines = content.split("\n").filter((l) => l.trim() && !l.startsWith("#")); @@ -352,7 +352,7 @@ async function showPlaylistWarnings(content: string) { } } -async function savePlaylist() { +export async function savePlaylist() { if (state.playlistSaving) return; state.playlistSaving = true; setStatus("playlist-status", "Saving playlist..."); diff --git a/frontend/src/main.test.ts b/frontend/src/main.test.ts index 0e42331..0aaa943 100644 --- a/frontend/src/main.test.ts +++ b/frontend/src/main.test.ts @@ -1,18 +1,29 @@ /// import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { resetViewerStateForTests, resolveManifestUrl, resolveWebSocketBaseUrl } from "../src/main"; + +const hlsInstance = { + loadSource: vi.fn(), + attachMedia: vi.fn(), + on: vi.fn(), + destroy: vi.fn(), +}; + +const HlsMock = Object.assign( + vi.fn().mockImplementation(() => hlsInstance), + { + isSupported: vi.fn(() => true), + Events: { MANIFEST_PARSED: "hlsManifestParsed" }, + }, +); // Mock hls.js vi.mock("hls.js", () => ({ - default: vi.fn().mockImplementation(() => ({ - loadSource: vi.fn(), - attachMedia: vi.fn(), - on: vi.fn(), - destroy: vi.fn(), - })), - Events: { MANIFEST_PARSED: "hlsManifestParsed" }, + default: HlsMock, })); // Mock WebSocket +const websocketInstances: MockWebSocket[] = []; class MockWebSocket { url: string; readyState = 0; @@ -23,7 +34,8 @@ class MockWebSocket { close = vi.fn(); constructor(url: string) { - this.url = url; + this.url = String(url); + websocketInstances.push(this); } } @@ -41,6 +53,8 @@ beforeEach(() => { origWebSocket = (globalThis as any).WebSocket; origFetch = globalThis.fetch; (globalThis as any).WebSocket = MockWebSocket; + websocketInstances.length = 0; + resetViewerStateForTests(); // Mock fetch (schedule, etc.) — return empty schedule globalThis.fetch = vi.fn().mockResolvedValue({ @@ -82,16 +96,44 @@ beforeEach(() => { // Stub prompt for DM (globalThis as any).prompt = vi.fn(); + + vi.useRealTimers(); }); afterEach(() => { (globalThis as any).WebSocket = origWebSocket; globalThis.fetch = origFetch; + vi.useRealTimers(); vi.restoreAllMocks(); }); -// Must import after DOM setup -import "../src/main"; +describe("player startup", () => { + it("starts native HLS playback on loadedmetadata", async () => { + const video = document.getElementById("player") as HTMLVideoElement; + vi.spyOn(video, "canPlayType").mockReturnValue("probably"); + const playSpy = vi.spyOn(video, "play").mockResolvedValue(undefined as any); + + await import("../src/main").then((mod) => mod.initPlayerAndChat()); + + expect(video.src).toContain("/hls/playlist.m3u8?stream="); + video.dispatchEvent(new Event("loadedmetadata")); + expect(playSpy).toHaveBeenCalled(); + }); + + it("starts hls.js playback when native HLS is unavailable", async () => { + const video = document.getElementById("player") as HTMLVideoElement; + Object.defineProperty(video, "canPlayType", { + value: vi.fn(() => ""), + configurable: true, + }); + const playSpy = vi.spyOn(video, "play").mockResolvedValue(undefined as any); + + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + expect(HlsMock.isSupported).toHaveBeenCalled(); + expect(HlsMock).toHaveBeenCalled(); + }); +}); describe("chat toggle", () => { it("toggles chat panel visibility", () => { @@ -112,13 +154,7 @@ describe("sendMessage", () => { const input = document.getElementById("message-input") as HTMLInputElement; input.value = "hello!"; - // sendMessage requires ws to be non-null — ws set in initPlayerAndChat - // The DOMContentLoaded triggers initPlayerAndChat which creates ws - // Wait for async init, then set ws manually - // For now just verify input state logic window.sendMessage(); - // ws is null at this point, so message won't send but input won't clear either - // Verify input not cleared (ws check fails before clear) expect(input.value).toBe("hello!"); }); @@ -158,3 +194,145 @@ describe("togglePrivateDM", () => { switchRoomSpy.mockRestore(); }); }); + +describe("WebSocket reconnect", () => { + it("reconnects after onclose", async () => { + vi.useFakeTimers(); + + window.switchRoom("lobby"); + expect(websocketInstances).toHaveLength(1); + + websocketInstances[0].onclose?.(); + await vi.advanceTimersByTimeAsync(3000); + + expect(websocketInstances).toHaveLength(2); + expect(websocketInstances[1].url).toContain("room=lobby"); + }); +}); + +describe("URL resolution", () => { + it("falls back to same-origin manifest and websocket URLs", () => { + vi.stubEnv("VITE_PUBLIC_HLS_URL", ""); + vi.stubEnv("VITE_PUBLIC_WS_URL", ""); + vi.stubEnv("VITE_PUBLIC_HLS_CROSS_ORIGIN", ""); + + Object.defineProperty(window, "location", { + value: new URL("https://linearvoid.tv/viewer"), + writable: true, + }); + + expect(resolveManifestUrl()).toMatch(/^https:\/\/linearvoid\.tv\/hls\/playlist\.m3u8\?stream=/); + expect(resolveWebSocketBaseUrl()).toBe("wss://linearvoid.tv/ws"); + }); + + it("preserves an allowed cross-origin manifest URL", () => { + vi.stubEnv("VITE_PUBLIC_HLS_URL", "https://media.linearvoid.tv/hls/playlist.m3u8"); + vi.stubEnv("VITE_PUBLIC_HLS_CROSS_ORIGIN", "true"); + + expect(resolveManifestUrl()).toMatch(/^https:\/\/media\.linearvoid\.tv\/hls\/playlist\.m3u8\?stream=/); + }); + + it("falls back to same-origin manifest when cross-origin is not allowed", () => { + vi.stubEnv("VITE_PUBLIC_HLS_URL", "https://media.linearvoid.tv/hls/playlist.m3u8"); + vi.stubEnv("VITE_PUBLIC_HLS_CROSS_ORIGIN", "false"); + + expect(resolveManifestUrl()).toMatch(/^https:\/\/linearvoid\.tv\/hls\/playlist\.m3u8\?stream=/); + }); + + it("uses configured websocket URL when present", () => { + vi.stubEnv("VITE_PUBLIC_WS_URL", "wss://chat.linearvoid.tv/ws"); + + expect(resolveWebSocketBaseUrl()).toBe("wss://chat.linearvoid.tv/ws"); + }); +}); + +describe("presence updates", () => { + it("adds user to presence list on join system message", async () => { + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + const ws = websocketInstances[0]; + ws.onmessage?.({ data: JSON.stringify({ type: "system", text: "alice joined the room", username: "system" }) }); + + const presenceList = document.getElementById("presence-list")!; + expect(presenceList.textContent).toContain("alice"); + }); + + it("removes user from presence list on leave system message", async () => { + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + const ws = websocketInstances[0]; + ws.onmessage?.({ data: JSON.stringify({ type: "system", text: "alice joined the room", username: "system" }) }); + ws.onmessage?.({ data: JSON.stringify({ type: "system", text: "alice left the room", username: "system" }) }); + + const presenceList = document.getElementById("presence-list")!; + expect(presenceList.textContent).not.toContain("alice"); + }); + + it("renders multiple users sorted", async () => { + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + const ws = websocketInstances[0]; + ws.onmessage?.({ data: JSON.stringify({ type: "system", text: "zoe joined the room", username: "system" }) }); + ws.onmessage?.({ data: JSON.stringify({ type: "system", text: "alice joined the room", username: "system" }) }); + + const presenceList = document.getElementById("presence-list")!; + const text = presenceList.textContent ?? ""; + const aliceIdx = text.indexOf("alice"); + const zoeIdx = text.indexOf("zoe"); + expect(aliceIdx).toBeLessThan(zoeIdx); + }); +}); + +describe("private DM flow", () => { + it("creates dm: room name from sorted usernames", async () => { + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + vi.spyOn(window, "prompt").mockReturnValue("bob"); + window.togglePrivateDM(); + + const roomName = document.getElementById("room-name")!; + // username is "viewer" (reset by tests), bob is target -> sorted = "bob-viewer" + // switchRoom capitalizes first char: "Dm:bob-viewer Room" + expect(roomName.textContent).toContain("bob-viewer"); + }); +}); + +describe("message rendering", () => { + it("renders system messages", async () => { + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + const ws = websocketInstances[0]; + ws.onmessage?.({ data: JSON.stringify({ type: "system", text: "Server restarting", username: "system" }) }); + + const messages = document.getElementById("messages")!; + expect(messages.innerHTML).toContain("Server restarting"); + expect(messages.querySelector(".message-system")).toBeTruthy(); + }); + + it("renders self messages as message-self", async () => { + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + const ws = websocketInstances[0]; + ws.onmessage?.({ + data: JSON.stringify({ type: "public", text: "hello world", username: "viewer", timestamp: Date.now() }), + }); + + const messages = document.getElementById("messages")!; + expect(messages.innerHTML).toContain("hello world"); + expect(messages.querySelector(".message-self")).toBeTruthy(); + }); + + it("renders other-user messages as message-other", async () => { + await import("../src/main").then((mod) => mod.initPlayerAndChat(async () => ({ default: HlsMock }) as any)); + + const ws = websocketInstances[0]; + ws.onmessage?.({ + data: JSON.stringify({ type: "public", text: "hey", username: "alice", timestamp: Date.now() }), + }); + + const messages = document.getElementById("messages")!; + expect(messages.innerHTML).toContain("alice"); + expect(messages.innerHTML).toContain("hey"); + expect(messages.querySelector(".message-other")).toBeTruthy(); + }); +}); diff --git a/frontend/src/main.ts b/frontend/src/main.ts index 8419bb0..b6308b8 100644 --- a/frontend/src/main.ts +++ b/frontend/src/main.ts @@ -18,7 +18,16 @@ let username = "viewer" + Math.floor(Math.random() * 1000); let chatVisible = true; const onlineUsers = new Set(); -function resolveWebSocketBaseUrl() { +export function resetViewerStateForTests() { + hls = null; + ws = null; + currentRoom = "main"; + username = "viewer"; + chatVisible = true; + onlineUsers.clear(); +} + +export function resolveWebSocketBaseUrl() { const configuredUrl = import.meta.env.VITE_PUBLIC_WS_URL?.trim(); if (configuredUrl) { @@ -29,7 +38,7 @@ function resolveWebSocketBaseUrl() { return `${protocol}//${window.location.host}/ws`; } -function resolveManifestUrl() { +export function resolveManifestUrl() { const configuredUrl = import.meta.env.VITE_PUBLIC_HLS_URL?.trim(); const cacheBust = `stream=${Date.now()}`; const sameOriginManifestUrl = new URL("/hls/playlist.m3u8", window.location.origin); @@ -139,7 +148,7 @@ async function refreshSchedule() { } } -async function initPlayerAndChat() { +export async function initPlayerAndChat(loadHlsModule: () => Promise<{ default: HlsClass }> = () => import("hls.js")) { const video = document.getElementById("player") as HTMLVideoElement; const manifestUrl = resolveManifestUrl(); @@ -148,7 +157,7 @@ async function initPlayerAndChat() { video.addEventListener("loadedmetadata", () => video.play()); } else { try { - const { default: Hls } = await import("hls.js"); + const { default: Hls } = await loadHlsModule(); if (Hls.isSupported()) { hls = new Hls(); hls.loadSource(manifestUrl); diff --git a/monitoring/grafana/dashboard.json b/monitoring/grafana/dashboard.json index bfe3d65..f6e781f 100644 --- a/monitoring/grafana/dashboard.json +++ b/monitoring/grafana/dashboard.json @@ -34,6 +34,33 @@ "title": "Chat Scrape Health", "type": "stat", "targets": [{ "expr": "up{job=\"chat\"}" }] + }, + { + "title": "Cache Hit Ratio", + "type": "timeseries", + "targets": [ + { + "expr": "linearvoid_cache_hit_ratio" + } + ] + }, + { + "title": "Redis Memory Usage", + "type": "timeseries", + "targets": [ + { + "expr": "linearvoid_chat_redis_memory_usage_ratio" + } + ] + }, + { + "title": "Chat Request Latency p95", + "type": "timeseries", + "targets": [ + { + "expr": "histogram_quantile(0.95, sum(rate(linearvoid_chat_request_duration_seconds_bucket[5m])) by (le))" + } + ] } ] } diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml index 500c217..8b6c6f0 100644 --- a/monitoring/prometheus/alerts.yml +++ b/monitoring/prometheus/alerts.yml @@ -38,13 +38,13 @@ groups: description: "Playout has switched to emergency slate. Check playout logs." - alert: HighChatLatency - expr: rate(linearvoid_chat_messages_total[5m]) < 0.01 + expr: histogram_quantile(0.95, sum(rate(linearvoid_chat_request_duration_seconds_bucket[5m])) by (le)) > 1 for: 5m labels: severity: warning annotations: - summary: "Unusually low chat message rate" - description: "Message rate below 0.01/s for 5 minutes. Check chat service health." + summary: "Chat request latency is high" + description: "Chat request p95 has been above 1s for 5 minutes. Check chat handlers and dependencies." - alert: PrometheusWALFailing expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0 @@ -64,6 +64,24 @@ groups: summary: "High CPU usage" description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}." + - alert: LowCacheHitRatio + expr: linearvoid_cache_hit_ratio < 0.9 + for: 15m + labels: + severity: warning + annotations: + summary: "Cloudflare cache hit ratio is below target" + description: "Cache hit ratio has been below 90% for 15 minutes on {{ $labels.hostname }}." + + - alert: HighRedisMemoryUsage + expr: linearvoid_chat_redis_memory_usage_ratio > 0.85 + for: 10m + labels: + severity: warning + annotations: + summary: "Redis memory usage is above target" + description: "Redis memory usage has been above 85% for 10 minutes on {{ $labels.instance }}." + - alert: LowDiskSpace expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10 for: 5m diff --git a/scripts/cache-hit-ratio.js b/scripts/cache-hit-ratio.js new file mode 100644 index 0000000..1092ad9 --- /dev/null +++ b/scripts/cache-hit-ratio.js @@ -0,0 +1,106 @@ +#!/usr/bin/env node + +const apiToken = process.env.CLOUDFLARE_API_TOKEN; +const zoneTag = process.env.CLOUDFLARE_ZONE_TAG; +const hostname = process.env.CLOUDFLARE_HOSTNAME || "linearvoid.tv"; +const windowMinutes = Number.parseInt(process.env.CLOUDFLARE_CACHE_HIT_RATIO_WINDOW_MINUTES || "60", 10); +const endpoint = process.env.CLOUDFLARE_GRAPHQL_ENDPOINT || "https://api.cloudflare.com/client/v4/graphql"; + +if (!apiToken || !zoneTag) { + console.error("Missing required environment variables: CLOUDFLARE_API_TOKEN, CLOUDFLARE_ZONE_TAG"); + process.exit(1); +} + +const end = new Date(); +const start = new Date(end.getTime() - windowMinutes * 60 * 1000); + +const iso = (value) => value.toISOString(); + +const query = ` + query CacheHitRatio($zoneTag: string, $start: Time, $end: Time, $hostname: String) { + viewer { + zones(filter: { zoneTag: $zoneTag }) { + requests: httpRequestsAdaptiveGroups( + limit: 1000 + filter: { + datetime_geq: $start + datetime_lt: $end + clientRequestHTTPHost: $hostname + requestSource: "eyeball" + } + ) { + count + dimensions { + cacheStatus + } + } + } + } + } +`; + +async function main() { + const response = await fetch(endpoint, { + method: "POST", + headers: { + Authorization: `Bearer ${apiToken}`, + "Content-Type": "application/json", + Accept: "application/json", + }, + body: JSON.stringify({ + query, + variables: { + zoneTag, + hostname, + start: iso(start), + end: iso(end), + }, + }), + }); + + const payload = await response.json(); + if (!response.ok || payload.errors) { + const message = payload.errors ? JSON.stringify(payload.errors) : `HTTP ${response.status}`; + throw new Error(message); + } + + const groups = payload?.data?.viewer?.zones?.[0]?.requests || []; + let totalRequests = 0; + let hitRequests = 0; + + for (const group of groups) { + const count = Number(group?.count || 0); + const status = String(group?.dimensions?.cacheStatus || "").toUpperCase(); + totalRequests += count; + if (status === "HIT") { + hitRequests += count; + } + } + + const ratio = totalRequests > 0 ? hitRequests / totalRequests : 0; + + console.log( + `# HELP linearvoid_cache_hit_ratio Cloudflare cache hit ratio for the selected hostname and time window.`, + ); + console.log(`# TYPE linearvoid_cache_hit_ratio gauge`); + console.log( + `linearvoid_cache_hit_ratio{zone="${zoneTag}",hostname="${hostname}",window_minutes="${windowMinutes}"} ${ratio.toFixed(6)}`, + ); + console.log(`# HELP linearvoid_cache_hit_requests_total Cloudflare cache HIT requests counted in the time window.`); + console.log(`# TYPE linearvoid_cache_hit_requests_total counter`); + console.log( + `linearvoid_cache_hit_requests_total{zone="${zoneTag}",hostname="${hostname}",window_minutes="${windowMinutes}"} ${hitRequests}`, + ); + console.log( + `# HELP linearvoid_cache_total_requests_total Cloudflare total eyeball requests counted in the time window.`, + ); + console.log(`# TYPE linearvoid_cache_total_requests_total counter`); + console.log( + `linearvoid_cache_total_requests_total{zone="${zoneTag}",hostname="${hostname}",window_minutes="${windowMinutes}"} ${totalRequests}`, + ); +} + +main().catch((error) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +}); diff --git a/scripts/cache-hit-ratio.ps1 b/scripts/cache-hit-ratio.ps1 new file mode 100644 index 0000000..0e9f7c2 --- /dev/null +++ b/scripts/cache-hit-ratio.ps1 @@ -0,0 +1,8 @@ +#!/usr/bin/env pwsh +$ErrorActionPreference = 'Stop' + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$nodeScript = Join-Path $scriptDir 'cache-hit-ratio.js' + +& node $nodeScript @args +exit $LASTEXITCODE diff --git a/scripts/loadtest/README.md b/scripts/loadtest/README.md new file mode 100644 index 0000000..d38598b --- /dev/null +++ b/scripts/loadtest/README.md @@ -0,0 +1,90 @@ +# Load Test Scaffolding + +These files back the scaling/performance playbook. + +## WebSocket chat + +Run with an explicit WebSocket URL, for example: + +```bash +WS_URL=wss://linearvoid.tv/ws?username=loadtest&room=main npx artillery run scripts/loadtest/ws-chat.yml +``` + +Capture output to an artifact file for the worksheet: + +```bash +WS_URL=wss://linearvoid.tv/ws?username=loadtest&room=main npx artillery run scripts/loadtest/ws-chat.yml --output artifacts/ws-chat-report.json +``` + +PowerShell equivalent: + +```powershell +$env:WS_URL = 'wss://linearvoid.tv/ws?username=loadtest&room=main' +New-Item -ItemType Directory -Force artifacts | Out-Null +npx artillery run scripts/loadtest/ws-chat.yml --output artifacts/ws-chat-report.json +``` + +Summarize key counters, rates, and available percentile stats from a saved Artillery JSON report: + +```powershell +pwsh -File scripts/loadtest/summarize-artillery.ps1 -ReportFile artifacts/ws-chat-report.json +``` + +## Admin API + +Run with a cookie or session header that can access admin endpoints: + +```bash +ADMIN_BASE_URL=https://linearvoid.tv ADMIN_SESSION_COOKIE="session=..." k6 run scripts/loadtest/admin-api.js +``` + +Capture machine-readable metrics and a text summary: + +```bash +ADMIN_BASE_URL=https://linearvoid.tv ADMIN_SESSION_COOKIE="session=..." k6 run --summary-export artifacts/admin-api-summary.json scripts/loadtest/admin-api.js > artifacts/admin-api-output.txt +``` + +PowerShell equivalent: + +```powershell +$env:ADMIN_BASE_URL = 'https://linearvoid.tv' +$env:ADMIN_SESSION_COOKIE = 'session=...' +New-Item -ItemType Directory -Force artifacts | Out-Null +k6 run --summary-export artifacts/admin-api-summary.json scripts/loadtest/admin-api.js | Tee-Object -FilePath artifacts/admin-api-output.txt +``` + +The script emits global and per-endpoint thresholds (overview, analytics, chat-log, users) so p95 and error-rate regressions are visible per route. + +Summarize endpoint p95 and failure rates from a saved k6 JSON summary: + +```powershell +pwsh -File scripts/loadtest/summarize-k6.ps1 -SummaryFile artifacts/admin-api-summary.json +``` + +## PowerShell helper + +Use the helper to run one scenario or both and automatically write outputs into a dated artifact folder: + +- The helper resolves scenario paths from its own script location, so it can be run from any current working directory. +- WebSocket runs also emit `ws-chat-summary.txt` with connection/error counters and available summary percentiles. +- Admin runs also emit `admin-api-endpoint-summary.txt` with per-endpoint p95/error values. + +```powershell +pwsh -File scripts/loadtest/run-loadtests.ps1 -Scenario ws -WsUrl 'wss://linearvoid.tv/ws?username=loadtest&room=main' +``` + +```powershell +pwsh -File scripts/loadtest/run-loadtests.ps1 -Scenario admin -AdminBaseUrl 'https://linearvoid.tv' -AdminSessionCookie 'session=...' +``` + +```powershell +pwsh -File scripts/loadtest/run-loadtests.ps1 -Scenario all -WsUrl 'wss://linearvoid.tv/ws?username=loadtest&room=main' -AdminBaseUrl 'https://linearvoid.tv' -AdminSessionCookie 'session=...' +``` + +## Notes + +- Adjust target URLs for test lab or production. +- Keep the same commit SHA or image tag when comparing runs. +- Record results using the worksheet in `docs/operations/scaling-performance-playbook.md`. +- Store raw outputs under an `artifacts/` folder per run date (for example `artifacts/2026-08-09/`). +- The repository `.gitignore` excludes benchmark artifacts by default. diff --git a/scripts/loadtest/admin-api.js b/scripts/loadtest/admin-api.js new file mode 100644 index 0000000..884fad3 --- /dev/null +++ b/scripts/loadtest/admin-api.js @@ -0,0 +1,58 @@ +import http from "k6/http"; +import { check, sleep } from "k6"; + +const baseUrl = __ENV.ADMIN_BASE_URL || "https://linearvoid.tv"; +const adminCookie = __ENV.ADMIN_SESSION_COOKIE || ""; +const requestParams = adminCookie ? { headers: { Cookie: adminCookie } } : {}; + +const endpoints = [ + { name: "overview", path: "/api/admin/overview" }, + { name: "analytics", path: "/api/admin/analytics" }, + { name: "chat_log", path: "/api/admin/chat-log" }, + { name: "users", path: "/api/admin/users" }, +]; + +export const options = { + stages: [ + { duration: "30s", target: 10 }, + { duration: "60s", target: 50 }, + { duration: "60s", target: 100 }, + { duration: "30s", target: 0 }, + ], + thresholds: { + http_req_duration: ["p(95)<1000"], + http_req_failed: ["rate<0.01"], + "http_req_duration{endpoint:overview}": ["p(95)<1000"], + "http_req_duration{endpoint:analytics}": ["p(95)<1000"], + "http_req_duration{endpoint:chat_log}": ["p(95)<1000"], + "http_req_duration{endpoint:users}": ["p(95)<1000"], + "http_req_failed{endpoint:overview}": ["rate<0.01"], + "http_req_failed{endpoint:analytics}": ["rate<0.01"], + "http_req_failed{endpoint:chat_log}": ["rate<0.01"], + "http_req_failed{endpoint:users}": ["rate<0.01"], + }, +}; + +export default function () { + const requests = endpoints.map((endpoint) => [ + "GET", + `${baseUrl}${endpoint.path}`, + null, + { + ...requestParams, + tags: { + endpoint: endpoint.name, + }, + }, + ]); + const responses = http.batch(requests); + + responses.forEach((response, index) => { + const endpointPath = endpoints[index].path; + check(response, { + [`status is 200 for ${endpointPath}`]: (r) => r.status === 200, + }); + }); + + sleep(1); +} diff --git a/scripts/loadtest/run-loadtests.ps1 b/scripts/loadtest/run-loadtests.ps1 new file mode 100644 index 0000000..b09ec7b --- /dev/null +++ b/scripts/loadtest/run-loadtests.ps1 @@ -0,0 +1,99 @@ +#!/usr/bin/env pwsh +param( + [ValidateSet('ws', 'admin', 'all')] + [string]$Scenario = 'all', + [string]$ArtifactRoot = 'artifacts', + [string]$WsUrl = $env:WS_URL, + [string]$AdminBaseUrl = $env:ADMIN_BASE_URL, + [string]$AdminSessionCookie = $env:ADMIN_SESSION_COOKIE +) + +$ErrorActionPreference = 'Stop' + +$scriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +$repoRoot = Resolve-Path (Join-Path $scriptDir '..\..') +$wsScenarioPath = Join-Path $scriptDir 'ws-chat.yml' +$wsSummaryScriptPath = Join-Path $scriptDir 'summarize-artillery.ps1' +$adminScriptPath = Join-Path $scriptDir 'admin-api.js' +$adminSummaryScriptPath = Join-Path $scriptDir 'summarize-k6.ps1' + +if ([System.IO.Path]::IsPathRooted($ArtifactRoot)) { + $artifactBase = $ArtifactRoot +} else { + $artifactBase = Join-Path $repoRoot $ArtifactRoot +} + +if ($Scenario -in @('ws', 'all')) { + if (-not (Get-Command npx -ErrorAction SilentlyContinue)) { + throw 'npx is required for ws scenario but was not found in PATH.' + } +} + +if ($Scenario -in @('admin', 'all')) { + if (-not (Get-Command k6 -ErrorAction SilentlyContinue)) { + throw 'k6 is required for admin scenario but was not found in PATH.' + } +} + +$timestamp = Get-Date -Format 'yyyy-MM-dd_HHmmss' +$artifactDir = Join-Path $artifactBase $timestamp +New-Item -ItemType Directory -Force -Path $artifactDir | Out-Null + +function Invoke-WebSocketLoadTest { + param( + [string]$TargetWsUrl, + [string]$OutputPath + ) + + if ([string]::IsNullOrWhiteSpace($TargetWsUrl)) { + throw 'WS_URL is required for ws scenario. Pass -WsUrl or set $env:WS_URL.' + } + + $env:WS_URL = $TargetWsUrl + & npx artillery run $wsScenarioPath --output $OutputPath +} + +function Invoke-AdminApiLoadTest { + param( + [string]$BaseUrl, + [string]$SessionCookie, + [string]$SummaryPath, + [string]$OutputPath + ) + + if ([string]::IsNullOrWhiteSpace($BaseUrl)) { + throw 'ADMIN_BASE_URL is required for admin scenario. Pass -AdminBaseUrl or set $env:ADMIN_BASE_URL.' + } + if ([string]::IsNullOrWhiteSpace($SessionCookie)) { + throw 'ADMIN_SESSION_COOKIE is required for admin scenario. Pass -AdminSessionCookie or set $env:ADMIN_SESSION_COOKIE.' + } + + $env:ADMIN_BASE_URL = $BaseUrl + $env:ADMIN_SESSION_COOKIE = $SessionCookie + & k6 run --summary-export $SummaryPath $adminScriptPath | Tee-Object -FilePath $OutputPath +} + +if ($Scenario -in @('ws', 'all')) { + if (-not (Test-Path $wsSummaryScriptPath)) { + throw "Missing summary helper: $wsSummaryScriptPath" + } + + $wsOutput = Join-Path $artifactDir 'ws-chat-report.json' + $wsSummary = Join-Path $artifactDir 'ws-chat-summary.txt' + Invoke-WebSocketLoadTest -TargetWsUrl $WsUrl -OutputPath $wsOutput + & pwsh -File $wsSummaryScriptPath -ReportFile $wsOutput | Tee-Object -FilePath $wsSummary +} + +if ($Scenario -in @('admin', 'all')) { + if (-not (Test-Path $adminSummaryScriptPath)) { + throw "Missing summary helper: $adminSummaryScriptPath" + } + + $adminSummary = Join-Path $artifactDir 'admin-api-summary.json' + $adminOutput = Join-Path $artifactDir 'admin-api-output.txt' + $adminEndpointSummary = Join-Path $artifactDir 'admin-api-endpoint-summary.txt' + Invoke-AdminApiLoadTest -BaseUrl $AdminBaseUrl -SessionCookie $AdminSessionCookie -SummaryPath $adminSummary -OutputPath $adminOutput + & pwsh -File $adminSummaryScriptPath -SummaryFile $adminSummary | Tee-Object -FilePath $adminEndpointSummary +} + +Write-Host "Artifacts written to $artifactDir" \ No newline at end of file diff --git a/scripts/loadtest/summarize-artillery.ps1 b/scripts/loadtest/summarize-artillery.ps1 new file mode 100644 index 0000000..6422a7d --- /dev/null +++ b/scripts/loadtest/summarize-artillery.ps1 @@ -0,0 +1,115 @@ +#!/usr/bin/env pwsh +param( + [Parameter(Mandatory = $true)] + [string]$ReportFile +) + +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path $ReportFile)) { + throw "Report file not found: $ReportFile" +} + +$report = Get-Content -Raw -Path $ReportFile | ConvertFrom-Json -Depth 100 +$aggregate = $report.aggregate + +if (-not $aggregate) { + throw 'Artillery report is missing aggregate data.' +} + +function Get-PropertyValue { + param( + [Parameter(Mandatory = $true)] + [object]$Object, + [Parameter(Mandatory = $true)] + [string]$Name + ) + + if (-not $Object) { + return $null + } + + $property = $Object.PSObject.Properties[$Name] + if ($property) { + return $property.Value + } + + return $null +} + +function Format-StatLine { + param( + [string]$Prefix, + [object]$Stats + ) + + if (-not $Stats) { + return $null + } + + $parts = @() + foreach ($name in @('count', 'min', 'max', 'mean', 'median', 'p95', 'p99')) { + $value = Get-PropertyValue -Object $Stats -Name $name + if ($null -ne $value) { + $parts += "$name=$value" + } + } + + if ($parts.Count -eq 0) { + return $null + } + + return "$Prefix: $($parts -join ', ')" +} + +$counters = Get-PropertyValue -Object $aggregate -Name 'counters' +$rates = Get-PropertyValue -Object $aggregate -Name 'rates' +$summaries = Get-PropertyValue -Object $aggregate -Name 'summaries' + +$created = [double](Get-PropertyValue -Object $counters -Name 'vusers.created' ?? 0) +$failed = [double](Get-PropertyValue -Object $counters -Name 'vusers.failed' ?? 0) +$completed = [double](Get-PropertyValue -Object $counters -Name 'vusers.completed' ?? 0) +$errorRate = if ($created -gt 0) { [math]::Round(($failed / $created) * 100, 2) } else { 0 } + +Write-Output '=== Artillery WebSocket Summary ===' +Write-Output "source: $ReportFile" + +$timestamp = Get-PropertyValue -Object $report -Name 'timestamp' +if ($timestamp) { + Write-Output "timestamp: $timestamp" +} + +if ($null -ne $created) { + Write-Output "vusers.created: $created" +} +if ($null -ne $completed) { + Write-Output "vusers.completed: $completed" +} +if ($null -ne $failed) { + Write-Output "vusers.failed: $failed" +} +Write-Output "vusers.failure_rate_percent: $errorRate" + +if ($counters) { + foreach ($metric in @('ws.messages_sent', 'ws.messages_received', 'ws.ping', 'plugins.metrics-by-endpoint.response_time')) { + $value = Get-PropertyValue -Object $counters -Name $metric + if ($null -ne $value) { + Write-Output "$metric: $value" + } + } +} + +if ($rates) { + foreach ($property in $rates.PSObject.Properties) { + Write-Output "rate.$($property.Name): $($property.Value)" + } +} + +if ($summaries) { + foreach ($property in $summaries.PSObject.Properties) { + $line = Format-StatLine -Prefix "summary.$($property.Name)" -Stats $property.Value + if ($line) { + Write-Output $line + } + } +} diff --git a/scripts/loadtest/summarize-k6.ps1 b/scripts/loadtest/summarize-k6.ps1 new file mode 100644 index 0000000..726233c --- /dev/null +++ b/scripts/loadtest/summarize-k6.ps1 @@ -0,0 +1,68 @@ +#!/usr/bin/env pwsh +param( + [Parameter(Mandatory = $true)] + [string]$SummaryFile +) + +$ErrorActionPreference = 'Stop' + +if (-not (Test-Path $SummaryFile)) { + throw "Summary file not found: $SummaryFile" +} + +$summary = Get-Content -Raw -Path $SummaryFile | ConvertFrom-Json +$metrics = $summary.metrics + +if (-not $metrics) { + throw 'Invalid k6 summary file: missing metrics object.' +} + +$endpoints = @('overview', 'analytics', 'chat_log', 'users') + +function Get-MetricValue { + param( + [object]$Metric, + [string]$Key + ) + + if (-not $Metric) { + return $null + } + if (-not $Metric.values) { + return $null + } + + $value = $Metric.values.$Key + if ($null -ne $value) { + return [double]$value + } + return $null +} + +$globalDuration = Get-MetricValue -Metric $metrics.http_req_duration -Key 'p(95)' +$globalFailRate = Get-MetricValue -Metric $metrics.http_req_failed -Key 'rate' + +Write-Output "k6 Summary: $SummaryFile" +if ($null -ne $globalDuration) { + Write-Output ("Global p95 duration (ms): {0:N2}" -f $globalDuration) +} +if ($null -ne $globalFailRate) { + Write-Output ("Global failure rate: {0:P2}" -f $globalFailRate) +} +Write-Output '' + +$rows = foreach ($endpoint in $endpoints) { + $durationMetricName = "http_req_duration{endpoint:$endpoint}" + $failMetricName = "http_req_failed{endpoint:$endpoint}" + + $durationP95 = Get-MetricValue -Metric $metrics.$durationMetricName -Key 'p(95)' + $failRate = Get-MetricValue -Metric $metrics.$failMetricName -Key 'rate' + + [PSCustomObject]@{ + Endpoint = $endpoint + P95Ms = if ($null -ne $durationP95) { [math]::Round($durationP95, 2) } else { $null } + FailurePct = if ($null -ne $failRate) { [math]::Round($failRate * 100, 4) } else { $null } + } +} + +$rows | Format-Table -AutoSize | Out-String | Write-Output diff --git a/scripts/loadtest/ws-chat.yml b/scripts/loadtest/ws-chat.yml new file mode 100644 index 0000000..8044df8 --- /dev/null +++ b/scripts/loadtest/ws-chat.yml @@ -0,0 +1,28 @@ +config: + target: "{{ $processEnvironment.WS_URL }}" + phases: + - duration: 60 + arrivalRate: 10 + name: warmup-10 + - duration: 120 + arrivalRate: 100 + name: sustain-100 + - duration: 180 + arrivalRate: 500 + name: sustain-500 + - duration: 240 + arrivalRate: 1000 + name: sustain-1000 + engines: + ws: {} +scenarios: + - engine: ws + name: chat-presence-and-send + flow: + - think: 1 + - send: + text: '{"type":"ping","text":"loadtest"}' + - think: 1 + - send: + text: '{"type":"ping","text":"loadtest-2"}' + - think: 1