From 2c06265d11b2d299a43bc66364c78dbe01b98a68 Mon Sep 17 00:00:00 2001 From: zwitschi Date: Tue, 4 Aug 2026 20:21:53 +0200 Subject: [PATCH] feat: implement resumable chunked media upload with progress indicators in admin UI --- CHANGELOG.md | 2 + backend/chat/src/main.go | 535 +++++++++++++++++++++++++++++++++++- docs/architecture/README.md | 195 ++++++++++++- frontend/admin/index.html | 7 +- frontend/src/admin.ts | 185 ++++++++++++- frontend/src/style.css | 16 ++ 6 files changed, 926 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c07cfc0..55e5988 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ - Extended admin schedule and dashboard data endpoints with media duration totals, runtime strings, and media counts. - Added admin UI runtime display for schedule/dashboard plus media library edit/delete actions. - Added a dedicated plain TypeScript admin module for tab navigation, media CRUD actions, schedule/system analytics refresh, and moderation workflows without Alpine dependencies. +- Added resumable chunked admin media upload flow with upload-session endpoints and client progress indicators. ### Changed @@ -27,6 +28,7 @@ - Implemented media upload duration probing with ffprobe and persisted media metadata to the database alongside filesystem storage. - Updated media listing to read from the media table and return duration metadata with HH:MM:SS runtime formatting. - Removed Tailwind and Alpine runtime/build dependencies from the frontend package configuration and eliminated Tailwind directives/warnings during production build. +- Enforced 2GB upload ceiling plus server-side media file type/size validation for admin uploads. ## [1.0.3] - 2026-08-03 diff --git a/backend/chat/src/main.go b/backend/chat/src/main.go index b702e3b..8a46f19 100644 --- a/backend/chat/src/main.go +++ b/backend/chat/src/main.go @@ -195,6 +195,36 @@ type AdminMediaResponse struct { GeneratedAt string `json:"generatedAt"` } +type AdminMediaUploadInitRequest struct { + UploadID string `json:"uploadId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + MimeType string `json:"mimeType"` + ChunkSize int64 `json:"chunkSize"` + TotalChunks int `json:"totalChunks"` +} + +type AdminMediaUploadSession struct { + UploadID string `json:"uploadId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + MimeType string `json:"mimeType"` + ChunkSize int64 `json:"chunkSize"` + TotalChunks int `json:"totalChunks"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` +} + +type AdminMediaUploadStatusResponse struct { + UploadID string `json:"uploadId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + ChunkSize int64 `json:"chunkSize"` + TotalChunks int `json:"totalChunks"` + UploadedChunks int `json:"uploadedChunks"` + Progress float64 `json:"progress"` +} + type AdminLoginPayload struct { Username string `json:"username"` Password string `json:"password"` @@ -209,6 +239,12 @@ var databaseDriverName = "sqlite" const adminSessionCookieName = "linearvoid_admin_session" +const ( + maxAdminUploadBytes int64 = 2 * 1024 * 1024 * 1024 + defaultAdminChunkSize int64 = 8 * 1024 * 1024 + maxAdminChunkSize int64 = 32 * 1024 * 1024 +) + // initRedis initializes Redis for presence, rate limiting, and pub/sub (ADR-007) func initRedis() { redisAddress := os.Getenv("REDIS_URL") @@ -938,6 +974,136 @@ func getMediaIDFromPath(path string) (uint, error) { return uint(id), nil } +func isAllowedMediaExtension(fileName string) bool { + ext := strings.ToLower(filepath.Ext(fileName)) + switch ext { + case ".mp4", ".mov", ".mkv", ".webm", ".m4v", ".ts": + return true + default: + return false + } +} + +func isAllowedMediaMime(mime string) bool { + mime = strings.TrimSpace(strings.ToLower(mime)) + if mime == "" { + return true + } + if strings.HasPrefix(mime, "video/") { + return true + } + return mime == "application/octet-stream" +} + +func getUploadWorkspaceDir() string { + return filepath.Join(getMediaDir(), ".uploads") +} + +func getUploadSessionDir(uploadID string) string { + return filepath.Join(getUploadWorkspaceDir(), uploadID) +} + +func getUploadSessionMetaPath(uploadID string) string { + return filepath.Join(getUploadSessionDir(uploadID), "session.json") +} + +func getUploadChunkPath(uploadID string, chunkIndex int) string { + return filepath.Join(getUploadSessionDir(uploadID), fmt.Sprintf("chunk-%06d.part", chunkIndex)) +} + +func loadUploadSession(uploadID string) (AdminMediaUploadSession, error) { + var session AdminMediaUploadSession + data, err := os.ReadFile(getUploadSessionMetaPath(uploadID)) + if err != nil { + return session, err + } + if err := json.Unmarshal(data, &session); err != nil { + return session, err + } + return session, nil +} + +func saveUploadSession(session AdminMediaUploadSession) error { + if err := os.MkdirAll(getUploadSessionDir(session.UploadID), 0o755); err != nil { + return err + } + data, err := json.MarshalIndent(session, "", " ") + if err != nil { + return err + } + return os.WriteFile(getUploadSessionMetaPath(session.UploadID), data, 0o644) +} + +func countUploadedChunks(uploadID string, totalChunks int) int { + uploaded := 0 + for i := 0; i < totalChunks; i++ { + if _, err := os.Stat(getUploadChunkPath(uploadID, i)); err == nil { + uploaded++ + } + } + return uploaded +} + +func validateUploadID(uploadID string) bool { + if len(uploadID) < 16 || len(uploadID) > 128 { + return false + } + for _, r := range uploadID { + if (r < 'a' || r > 'f') && (r < '0' || r > '9') { + return false + } + } + return true +} + +func buildUploadStatus(session AdminMediaUploadSession) AdminMediaUploadStatusResponse { + uploadedChunks := countUploadedChunks(session.UploadID, session.TotalChunks) + progress := 0.0 + if session.TotalChunks > 0 { + progress = (float64(uploadedChunks) / float64(session.TotalChunks)) * 100 + } + + return AdminMediaUploadStatusResponse{ + UploadID: session.UploadID, + FileName: session.FileName, + FileSize: session.FileSize, + ChunkSize: session.ChunkSize, + TotalChunks: session.TotalChunks, + UploadedChunks: uploadedChunks, + Progress: progress, + } +} + +func uniqueMediaTargetPath(mediaDir, fileName string) string { + baseName := filepath.Base(fileName) + ext := filepath.Ext(baseName) + nameOnly := strings.TrimSuffix(baseName, ext) + candidate := filepath.Join(mediaDir, baseName) + + for i := 1; ; i++ { + if _, err := os.Stat(candidate); os.IsNotExist(err) { + return candidate + } + candidate = filepath.Join(mediaDir, fmt.Sprintf("%s-%d%s", nameOnly, i, ext)) + } +} + +func detectMediaMime(filePath string) (string, error) { + f, err := os.Open(filePath) + if err != nil { + return "", err + } + defer f.Close() + + head := make([]byte, 512) + n, err := f.Read(head) + if err != nil && err != io.EOF { + return "", err + } + + return http.DetectContentType(head[:n]), nil +} + func mediaEntryFromRecord(record Media) MediaFileEntry { return MediaFileEntry{ ID: record.ID, @@ -1020,13 +1186,27 @@ func handleAdminMedia(w http.ResponseWriter, r *http.Request) { } defer file.Close() + fileName := filepath.Base(strings.TrimSpace(header.Filename)) + if fileName == "" { + http.Error(w, "missing file name", http.StatusBadRequest) + return + } + if !isAllowedMediaExtension(fileName) { + http.Error(w, "unsupported file type", http.StatusBadRequest) + return + } + if header.Size > maxAdminUploadBytes { + http.Error(w, "file exceeds 2GB limit", http.StatusRequestEntityTooLarge) + return + } + mediaDir := getMediaDir() if err := os.MkdirAll(mediaDir, 0o755); err != nil { http.Error(w, fmt.Sprintf("failed to ensure media directory: %v", err), http.StatusInternalServerError) return } - targetPath := filepath.Join(mediaDir, filepath.Base(header.Filename)) + targetPath := uniqueMediaTargetPath(mediaDir, fileName) targetFile, err := os.Create(targetPath) if err != nil { http.Error(w, fmt.Sprintf("failed to create media file: %v", err), http.StatusInternalServerError) @@ -1051,6 +1231,23 @@ func handleAdminMedia(w http.ResponseWriter, r *http.Request) { http.Error(w, fmt.Sprintf("failed to read media file info: %v", err), http.StatusInternalServerError) return } + if fileInfo.Size() > maxAdminUploadBytes { + _ = os.Remove(targetPath) + http.Error(w, "file exceeds 2GB limit", http.StatusRequestEntityTooLarge) + return + } + + detectedMime, err := detectMediaMime(targetPath) + if err != nil { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("failed to inspect media file: %v", err), http.StatusBadRequest) + return + } + if !strings.HasPrefix(strings.ToLower(detectedMime), "video/") { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("unsupported media content type: %s", detectedMime), http.StatusBadRequest) + return + } durationSeconds, err := probeMediaDurationSeconds(targetPath) if err != nil { @@ -1059,10 +1256,10 @@ func handleAdminMedia(w http.ResponseWriter, r *http.Request) { return } - filename := filepath.Base(header.Filename) + filename := filepath.Base(targetPath) mediaRecord := Media{ Filename: filename, - OriginalName: header.Filename, + OriginalName: fileName, Path: targetPath, SizeBytes: fileInfo.Size(), DurationSeconds: durationSeconds, @@ -1089,6 +1286,336 @@ func handleAdminMedia(w http.ResponseWriter, r *http.Request) { } } +func handleAdminMediaUploads(w http.ResponseWriter, r *http.Request) { + const basePath = "/api/admin/media/uploads" + + if r.URL.Path == basePath { + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + var payload AdminMediaUploadInitRequest + if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { + http.Error(w, fmt.Sprintf("invalid request body: %v", err), http.StatusBadRequest) + return + } + + fileName := filepath.Base(strings.TrimSpace(payload.FileName)) + if fileName == "" { + http.Error(w, "file name is required", http.StatusBadRequest) + return + } + if !isAllowedMediaExtension(fileName) { + http.Error(w, "unsupported file type", http.StatusBadRequest) + return + } + if payload.FileSize <= 0 { + http.Error(w, "file size must be positive", http.StatusBadRequest) + return + } + if payload.FileSize > maxAdminUploadBytes { + http.Error(w, "file exceeds 2GB limit", http.StatusRequestEntityTooLarge) + return + } + if !isAllowedMediaMime(payload.MimeType) { + http.Error(w, "unsupported mime type", http.StatusBadRequest) + return + } + + chunkSize := payload.ChunkSize + if chunkSize <= 0 { + chunkSize = defaultAdminChunkSize + } + if chunkSize > maxAdminChunkSize { + http.Error(w, "chunk size exceeds server limit", http.StatusBadRequest) + return + } + + expectedChunks := int((payload.FileSize + chunkSize - 1) / chunkSize) + if payload.TotalChunks <= 0 { + payload.TotalChunks = expectedChunks + } + if payload.TotalChunks != expectedChunks { + http.Error(w, "total chunk count mismatch", http.StatusBadRequest) + return + } + + uploadID := strings.TrimSpace(payload.UploadID) + if uploadID != "" && !validateUploadID(uploadID) { + http.Error(w, "invalid upload id", http.StatusBadRequest) + return + } + + now := time.Now().UTC().Format(time.RFC3339) + session := AdminMediaUploadSession{} + + if uploadID != "" { + loaded, err := loadUploadSession(uploadID) + if err == nil { + if loaded.FileName != fileName || loaded.FileSize != payload.FileSize { + http.Error(w, "upload id does not match selected file", http.StatusConflict) + return + } + session = loaded + } else if !os.IsNotExist(err) { + http.Error(w, fmt.Sprintf("failed to resume upload session: %v", err), http.StatusInternalServerError) + return + } + } + + if session.UploadID == "" { + randomID, err := generateSessionToken() + if err != nil { + http.Error(w, fmt.Sprintf("failed to create upload session: %v", err), http.StatusInternalServerError) + return + } + + session = AdminMediaUploadSession{ + UploadID: randomID, + FileName: fileName, + FileSize: payload.FileSize, + MimeType: payload.MimeType, + ChunkSize: chunkSize, + TotalChunks: payload.TotalChunks, + CreatedAt: now, + UpdatedAt: now, + } + } else { + session.UpdatedAt = now + } + + if err := saveUploadSession(session); err != nil { + http.Error(w, fmt.Sprintf("failed to persist upload session: %v", err), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(buildUploadStatus(session)) + return + } + + if !strings.HasPrefix(r.URL.Path, basePath+"/") { + http.NotFound(w, r) + return + } + + remainder := strings.TrimPrefix(r.URL.Path, basePath+"/") + parts := strings.Split(strings.Trim(remainder, "/"), "/") + if len(parts) == 0 || strings.TrimSpace(parts[0]) == "" { + http.Error(w, "missing upload id", http.StatusBadRequest) + return + } + + uploadID := strings.TrimSpace(parts[0]) + if !validateUploadID(uploadID) { + http.Error(w, "invalid upload id", http.StatusBadRequest) + return + } + + session, err := loadUploadSession(uploadID) + if err != nil { + if os.IsNotExist(err) { + http.Error(w, "upload session not found", http.StatusNotFound) + return + } + http.Error(w, fmt.Sprintf("failed to read upload session: %v", err), http.StatusInternalServerError) + return + } + + if len(parts) == 1 { + if r.Method != http.MethodGet { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(buildUploadStatus(session)) + return + } + + if len(parts) != 2 { + http.NotFound(w, r) + return + } + + subresource := parts[1] + switch subresource { + case "chunk": + if r.Method != http.MethodPut { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + chunkIndexRaw := strings.TrimSpace(r.URL.Query().Get("index")) + chunkIndex, err := strconv.Atoi(chunkIndexRaw) + if err != nil || chunkIndex < 0 || chunkIndex >= session.TotalChunks { + http.Error(w, "invalid chunk index", http.StatusBadRequest) + return + } + + chunkPath := getUploadChunkPath(uploadID, chunkIndex) + tmpPath := chunkPath + ".tmp" + + chunkFile, err := os.Create(tmpPath) + if err != nil { + http.Error(w, fmt.Sprintf("failed to open chunk target: %v", err), http.StatusInternalServerError) + return + } + + maxRead := session.ChunkSize + 1 + written, copyErr := io.Copy(chunkFile, io.LimitReader(r.Body, maxRead)) + closeErr := chunkFile.Close() + if copyErr != nil || closeErr != nil { + _ = os.Remove(tmpPath) + http.Error(w, "failed to persist upload chunk", http.StatusInternalServerError) + return + } + + expectedMax := session.ChunkSize + if chunkIndex == session.TotalChunks-1 { + remaining := session.FileSize - (int64(chunkIndex) * session.ChunkSize) + if remaining > 0 { + expectedMax = remaining + } + } + + if written <= 0 || written > expectedMax { + _ = os.Remove(tmpPath) + http.Error(w, "invalid chunk size", http.StatusBadRequest) + return + } + + if err := os.Rename(tmpPath, chunkPath); err != nil { + _ = os.Remove(tmpPath) + http.Error(w, fmt.Sprintf("failed to finalize chunk: %v", err), http.StatusInternalServerError) + return + } + + session.UpdatedAt = time.Now().UTC().Format(time.RFC3339) + if err := saveUploadSession(session); err != nil { + http.Error(w, fmt.Sprintf("failed to update upload session: %v", err), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(buildUploadStatus(session)) + case "complete": + if r.Method != http.MethodPost { + http.Error(w, "method not allowed", http.StatusMethodNotAllowed) + return + } + + if countUploadedChunks(uploadID, session.TotalChunks) != session.TotalChunks { + http.Error(w, "upload is incomplete", http.StatusBadRequest) + return + } + + mediaDir := getMediaDir() + if err := os.MkdirAll(mediaDir, 0o755); err != nil { + http.Error(w, fmt.Sprintf("failed to ensure media directory: %v", err), http.StatusInternalServerError) + return + } + + targetPath := uniqueMediaTargetPath(mediaDir, session.FileName) + targetFile, err := os.Create(targetPath) + if err != nil { + http.Error(w, fmt.Sprintf("failed to create media file: %v", err), http.StatusInternalServerError) + return + } + + var totalWritten int64 + for i := 0; i < session.TotalChunks; i++ { + chunkPath := getUploadChunkPath(uploadID, i) + chunkReader, err := os.Open(chunkPath) + if err != nil { + targetFile.Close() + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("missing chunk %d", i), http.StatusBadRequest) + return + } + + written, err := io.Copy(targetFile, chunkReader) + _ = chunkReader.Close() + if err != nil { + targetFile.Close() + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("failed to assemble media file: %v", err), http.StatusInternalServerError) + return + } + totalWritten += written + } + + if err := targetFile.Close(); err != nil { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("failed to finalize media file: %v", err), http.StatusInternalServerError) + return + } + + if totalWritten != session.FileSize { + _ = os.Remove(targetPath) + http.Error(w, "assembled file size mismatch", http.StatusBadRequest) + return + } + + detectedMime, err := detectMediaMime(targetPath) + if err != nil { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("failed to inspect media file: %v", err), http.StatusBadRequest) + return + } + if !strings.HasPrefix(strings.ToLower(detectedMime), "video/") { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("unsupported media content type: %s", detectedMime), http.StatusBadRequest) + return + } + + durationSeconds, err := probeMediaDurationSeconds(targetPath) + if err != nil { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("failed to probe media duration: %v", err), http.StatusBadRequest) + return + } + + fileInfo, err := os.Stat(targetPath) + if err != nil { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("failed to read media file info: %v", err), http.StatusInternalServerError) + return + } + + filename := filepath.Base(targetPath) + mediaRecord := Media{ + Filename: filename, + OriginalName: session.FileName, + Path: targetPath, + SizeBytes: fileInfo.Size(), + DurationSeconds: durationSeconds, + Status: "ready", + Title: strings.TrimSuffix(filename, filepath.Ext(filename)), + Tags: "", + } + if err := db.Create(&mediaRecord).Error; err != nil { + _ = os.Remove(targetPath) + http.Error(w, fmt.Sprintf("failed to persist media metadata: %v", err), http.StatusInternalServerError) + return + } + + _ = os.RemoveAll(getUploadSessionDir(uploadID)) + + files, err := listMediaFiles() + if err != nil { + http.Error(w, fmt.Sprintf("failed to list media files: %v", err), http.StatusInternalServerError) + return + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(AdminMediaResponse{Files: files, GeneratedAt: time.Now().UTC().Format(time.RFC3339)}) + default: + http.NotFound(w, r) + } +} + func handleAdminMediaByID(w http.ResponseWriter, r *http.Request) { mediaID, err := getMediaIDFromPath(r.URL.Path) if err != nil { @@ -1424,6 +1951,8 @@ func main() { http.HandleFunc("/api/admin/users", requireAdminAuth(handleAdminUsers)) http.HandleFunc("/api/admin/schedule", requireAdminAuth(handleAdminSchedule)) http.HandleFunc("/api/admin/media", requireAdminAuth(handleAdminMedia)) + http.HandleFunc("/api/admin/media/uploads", requireAdminAuth(handleAdminMediaUploads)) + http.HandleFunc("/api/admin/media/uploads/", requireAdminAuth(handleAdminMediaUploads)) http.HandleFunc("/api/admin/media/", requireAdminAuth(handleAdminMediaByID)) http.HandleFunc("/metrics", handleMetrics) http.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 056af3b..3fe2589 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -315,11 +315,96 @@ All source assets must be pre-normalized (H.264/AVC, AAC-LC, constant 30 fps, 2s ### Monitoring & Resilience -Alert on no new segment (>30–60s), low cache hits, high R2 ops, WebSocket errors. Maintain emergency slate, auto-restart, weekly backups, incident runbook. +Linear Void uses a two-layer monitoring model: + +1. **Application layer (chat + platform health)** via Prometheus metrics and a lightweight Grafana dashboard. +2. **Infrastructure/runtime layer (Prometheus self-observability, TSDB, query engine, Go runtime)** via a detailed Grafana dashboard. + +#### Prometheus Baseline + +- Config file: `monitoring/prometheus/prometheus.yml` +- Default scrape interval: `15s` +- Current scrape jobs: `chat` -> `chat:3000`, `prometheus` -> `prometheus:9090`. + +Recommended next-step scrape targets (if/when exporters are introduced): + +- `nginx` (via nginx-exporter) +- `redis` (via redis-exporter) +- `postgres` (via postgres-exporter) +- `node` (via node-exporter) + +#### Grafana Dashboards + +- `monitoring/grafana/dashboard.json` (Linear Void app metrics: connected clients, messages, moderation actions, Redis reachability, admin sessions, scrape health). +- `monitoring/grafana/dashboards/dashboard.json` (Prometheus operational visibility: status, uptime, CPU/memory, TSDB ingestion/churn/errors, HTTP latencies, query load, GC/scheduler). + +Use the app dashboard for service-level behavior and the Prometheus overview dashboard for platform diagnostics and capacity planning. + +#### Alerting Rules (Prometheus Rule Groups) + +Define alert rules as grouped policies (for example in `monitoring/prometheus/alerts.yml`) and include them from `prometheus.yml`. + +Minimum recommended alert set: + +- **Availability** + `up{job="chat"} == 0` for `2m`; `up{job="prometheus"} == 0` for `2m`. +- **Chat service health** + `linearvoid_chat_redis_up == 0` for `2m`; sudden disconnect spike / connected clients collapse. +- **Playout continuity** + No HLS segment progression for `30-60s` (from playout-specific metric when available). +- **Prometheus capacity and ingestion health** + Sustained growth in `prometheus_tsdb_wal_writes_failed_total` or `prometheus_tsdb_compactions_failed_total`, abnormal jump in `prometheus_tsdb_out_of_order_samples_total`, and high `prometheus_engine_queries` near concurrent limit. +- **Latency and user-impact** + Elevated p99 handler latency from `prometheus_http_request_duration_seconds_bucket` and elevated p99 query execution latency from `prometheus_engine_query_duration_histogram_seconds_bucket`. + +Route warning/critical severities to the team channel and keep runbook links attached to each alert definition. + +Operational resilience baseline: + +- Emergency slate fallback for playout failure. +- Auto-restart policies for playout/chat/nginx containers. +- Weekly backup and restore drills for stateful data (`data/chat`, playlist metadata, Postgres). +- Incident runbook with triage order: edge reachability -> origin health -> playout progression -> chat connectivity -> data-plane dependencies. ### Security & Moderation -HTTPS/WSS only, strict CORS, Cloudflare WAF, authenticated admin paths, rate limiting, immutable moderation log. Private DMs not E2EE by default. +Security is split between edge controls (Cloudflare), reverse proxy controls (nginx), and application controls (chat/admin backend). + +#### Transport and Edge Protection + +- Public traffic uses HTTPS/WSS at the edge. +- Cloudflare WAF and rate limiting are part of the baseline (see `infrastructure/cloudflare/cache-rules.md`). +- Cloudflare Worker can enforce immutable cache headers for segment paths (`infrastructure/cloudflare/worker.js`). + +#### Reverse Proxy Enforcement (nginx) + +- Admin UI and admin API paths are protected by `auth_request` before reaching backend handlers (`/admin/`, `/api/admin/`). +- Unauthorized admin UI requests are redirected to `/admin/login/`. +- Unauthorized admin API requests return JSON `401` from a dedicated location handler. +- WebSocket upgrade headers are explicitly forwarded for `/ws`. + +#### Admin Authentication and Session Security + +- Admin authentication is cookie-session based (`linearvoid_admin_session`). +- Session cookie controls in backend include `HttpOnly`, `SameSite=Strict`, and `Secure` when request is HTTPS (derived from TLS or `X-Forwarded-Proto`). +- Session expiration is enforced server-side. + +#### Rate Limiting and Abuse Controls + +- Chat message rate limiting uses Redis `INCR + EXPIRE` windowing. +- Current baseline in chat handler: `>10` messages within `10s` triggers a user-facing throttle response. +- If Redis is unavailable, presence/rate-limit features degrade gracefully (warning logged; core chat remains available). + +#### CORS and Cross-Origin Behavior + +- HLS endpoints exposed by nginx include explicit `Access-Control-Allow-Origin: *` for player compatibility. +- Admin/authenticated routes are expected to run same-origin behind nginx auth gates. + +#### Moderation and Privacy + +- Moderation commands (`/mod`, `/mute`, `/ban`) are handled in backend chat flow and persisted through chat storage paths. +- Private DMs are application-level private channels, not end-to-end encrypted in current architecture. +- Moderation and audit retention policies should be formalized in runbook/ops docs as part of production hardening. ### Chat @@ -354,6 +439,112 @@ These ADRs together form the canonical decision log. See **Quality Goals** table in the Introduction section. Primary scenarios focus on cost (near-zero egress), availability (auto-restart + fallback), scalability (edge caching), and usability (synchronized player + responsive chat). +### Performance Benchmarks and Capacity Planning + +#### Baseline SLO Targets (v1) + +- Playback startup: target < 4s after first manifest request on warm cache. +- Steady-state playback: no segment starvation during normal operation. +- End-to-end HLS latency: expected 15-35s (standard HLS trade-off). +- Chat delivery latency: p95 < 500ms inside same region under normal load. +- Admin page responsiveness: p95 API response < 1s for common read operations. + +#### Workload Assumptions + +- Primary audience in US; origin in US-East. +- Edge cache serves most segment traffic; origin handles manifest churn and cache misses. +- Chat traffic scales with concurrent viewers but remains low-bandwidth compared to video. + +#### Initial Capacity Model + +- Video bitrate envelopes (from ADR-001): + - 720p30: ~2.6-4.2 Mbps total with audio. + - 1080p30: ~5.1-8.2 Mbps total with audio. +- Storage planning for 48h rolling library: + - 720p30: ~57-90 GB raw (allocate ~80-110 GB with headroom). + - 1080p30: ~110-175 GB raw (allocate ~130-180 GB with headroom). +- Cost behavior: + - Viewer scale is mostly constrained by cache efficiency and request volume, not by direct origin egress. + +#### Benchmark Plan + +- Playback path: + - Measure time-to-first-frame and rebuffer events from multiple US regions. + - Track manifest freshness and segment availability at edge. +- Chat path: + - Load-test concurrent websocket connections and message fan-out. + - Measure p50/p95 send-to-receive latency and Redis fallback behavior. +- Admin/API path: + - Benchmark playlist/system endpoints for p95 latency and error rate. +- Platform path: + - Observe TSDB ingestion, query p99, and GC pause behavior in Grafana. + +#### Capacity Triggers and Actions + +- Trigger: sustained viewer growth with rising cache-miss ratio. + - Action: tighten cache policy, enable/verify Tiered Cache, review manifest TTL. +- Trigger: chat p95 latency drift or websocket error growth. + - Action: scale chat service, tune Redis, separate noisy moderation workloads. +- Trigger: Prometheus WAL/compaction failures or query saturation. + - Action: tune retention/cardinality, increase resources, split dashboards/queries. +- Trigger: origin CPU/memory pressure on playout host. + - Action: optimize FFmpeg profile, isolate services, add secondary origin. + +### Disaster Recovery and Backup Procedures + +#### Recovery Targets + +- RPO target: <= 24h for chat/moderation/admin metadata. +- RTO target: <= 60 minutes for restoring baseline streaming + chat operations. + +#### Backup Scope + +- Persistent service volumes: + - Postgres data (`postgres-data` volume). + - Redis append-only persistence (`redis-data` volume) where enabled. + - Prometheus long-term TSDB (`prometheus-data` volume) if historical metrics retention is required. + - Grafana state (`grafana-data` volume) if dashboards/users are managed in-instance. +- Host-mounted state directories: + - `data/chat` + - `data/playlist` + - `data/media` (authoritative source may be R2/object storage) + - `data/hls` (optional; can be regenerated by playout) +- Configuration and infra source of truth: + - Compose files, nginx config, monitoring configs, and environment template. + +#### Backup Cadence + +- Daily logical Postgres backups (plus weekly full snapshot). +- Daily sync of chat/playlist state directories. +- Weekly integrity check by restoring backups into an isolated environment. +- Retention policy baseline: + - Daily backups: 14 days + - Weekly backups: 8 weeks + - Monthly backups: 6 months + +#### Recovery Runbook (Minimum) + +1. Restore infrastructure configuration (compose, nginx, env, monitoring config). +2. Restore Postgres and Redis persisted state. +3. Restore `data/playlist` and chat persistence directories. +4. Rehydrate media from authoritative object storage if local media is missing. +5. Start stack and validate in order: + - `playout` health endpoint + - `chat` health endpoint + - nginx `/health` + - HLS playlist availability + - websocket chat connectivity +6. Verify admin authentication and moderation audit visibility. +7. Announce recovery and continue elevated monitoring for at least one playlist cycle. + +#### Failure Modes and Strategy + +- Single-host loss: redeploy stack on standby host, restore data backups, repoint DNS/edge origin. +- Storage corruption: restore last known-good snapshot and replay latest logical backups. +- Partial service failure: replace only impacted service and restore scoped data (for example Postgres only). + +Document and rehearse this runbook quarterly; recovery that is not exercised is not reliable. + ## Risks and Technical Debts - Cloudflare ToS risk for heavy video on free tier (use R2 path; fallback to BunnyCDN if challenged). diff --git a/frontend/admin/index.html b/frontend/admin/index.html index 359b0ce..b73f506 100644 --- a/frontend/admin/index.html +++ b/frontend/admin/index.html @@ -64,9 +64,14 @@ -
+
+

Resumable chunk upload enabled • up to 2GB per file.

+

Loading media library...

diff --git a/frontend/src/admin.ts b/frontend/src/admin.ts index ccead29..ca33e20 100644 --- a/frontend/src/admin.ts +++ b/frontend/src/admin.ts @@ -33,12 +33,84 @@ const adminConfig = { : 50, }; +const MAX_UPLOAD_BYTES = 2 * 1024 * 1024 * 1024; +const UPLOAD_CHUNK_BYTES = 8 * 1024 * 1024; +const UPLOAD_RESUME_KEY_PREFIX = "linearvoid-upload-resume:"; + const state = { mediaUploading: false, moderationSubmitting: false, playlistSaving: false, }; +const uploadProgressWrap = () => + document.getElementById("media-upload-progress-wrap") as HTMLElement | null; + +const uploadProgressBar = () => + document.getElementById("media-upload-progress") as HTMLProgressElement | null; + +const uploadProgressText = () => + document.getElementById("media-upload-progress-text") as HTMLElement | null; + +const setUploadProgress = (value: number, total: number, label: string) => { + const wrap = uploadProgressWrap(); + const progress = uploadProgressBar(); + const text = uploadProgressText(); + if (!wrap || !progress || !text) return; + + wrap.hidden = false; + progress.max = total; + progress.value = value; + + const pct = total > 0 ? Math.min(100, Math.round((value / total) * 100)) : 0; + text.textContent = `${label} (${pct}%)`; +}; + +const hideUploadProgress = () => { + const wrap = uploadProgressWrap(); + if (wrap) wrap.hidden = true; +}; + +const getFileFingerprint = (file: File) => + `${file.name}:${file.size}:${file.lastModified}`; + +const getResumeStorageKey = (fingerprint: string) => + `${UPLOAD_RESUME_KEY_PREFIX}${fingerprint}`; + +const getStoredUploadId = (fingerprint: string) => { + try { + return window.localStorage.getItem(getResumeStorageKey(fingerprint)) || ""; + } catch { + return ""; + } +}; + +const setStoredUploadId = (fingerprint: string, uploadId: string) => { + try { + window.localStorage.setItem(getResumeStorageKey(fingerprint), uploadId); + } catch { + // ignore storage failures + } +}; + +const clearStoredUploadId = (fingerprint: string) => { + try { + window.localStorage.removeItem(getResumeStorageKey(fingerprint)); + } catch { + // ignore storage failures + } +}; + +const readErrorMessage = async (response: Response) => { + const fallback = `HTTP ${response.status}`; + try { + const text = (await response.text()).trim(); + return text || fallback; + } catch { + return fallback; + } +}; + const setText = (id: string, text: string) => { const el = document.getElementById(id); if (el) el.textContent = text; @@ -330,30 +402,110 @@ async function refreshMediaLibrary() { async function uploadVideo(file: File) { if (state.mediaUploading) return; + + if (file.size <= 0) { + setStatus("media-status", "Cannot upload an empty file.", true); + return; + } + if (file.size > MAX_UPLOAD_BYTES) { + setStatus("media-status", "File exceeds 2GB upload limit.", true); + return; + } + state.mediaUploading = true; - setStatus("media-status", `Uploading ${file.name}...`); + setStatus("media-status", `Preparing upload for ${file.name}...`); try { - const formData = new FormData(); - formData.append("file", file); + const totalChunks = Math.ceil(file.size / UPLOAD_CHUNK_BYTES); + const fingerprint = getFileFingerprint(file); - const response = await fetch("/api/admin/media", { + const initResponse = await fetch("/api/admin/media/uploads", { method: "POST", - body: formData, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + uploadId: getStoredUploadId(fingerprint), + fileName: file.name, + fileSize: file.size, + mimeType: file.type, + chunkSize: UPLOAD_CHUNK_BYTES, + totalChunks, + }), }); - if (!response.ok) throw new Error(`HTTP ${response.status}`); - const payload = await response.json(); + if (!initResponse.ok) { + throw new Error(await readErrorMessage(initResponse)); + } + + const initPayload = await initResponse.json(); + const uploadId = String(initPayload.uploadId || ""); + if (!uploadId) { + throw new Error("Upload session could not be initialized."); + } + + setStoredUploadId(fingerprint, uploadId); + + const alreadyUploaded = Number(initPayload.uploadedChunks || 0); + const effectiveTotalChunks = Number(initPayload.totalChunks || totalChunks); + const resumeFrom = Math.min(Math.max(alreadyUploaded, 0), effectiveTotalChunks); + + setUploadProgress(resumeFrom, effectiveTotalChunks, "Uploading chunks"); + + for (let chunkIndex = resumeFrom; chunkIndex < effectiveTotalChunks; chunkIndex += 1) { + const start = chunkIndex * UPLOAD_CHUNK_BYTES; + const end = Math.min(file.size, start + UPLOAD_CHUNK_BYTES); + const chunk = file.slice(start, end); + + const chunkResponse = await fetch( + `/api/admin/media/uploads/${encodeURIComponent(uploadId)}/chunk?index=${chunkIndex}`, + { + method: "PUT", + headers: { "Content-Type": "application/octet-stream" }, + body: chunk, + }, + ); + + if (!chunkResponse.ok) { + throw new Error(await readErrorMessage(chunkResponse)); + } + + setUploadProgress(chunkIndex + 1, effectiveTotalChunks, "Uploading chunks"); + setStatus( + "media-status", + `Uploading ${file.name} (${chunkIndex + 1}/${effectiveTotalChunks} chunks)...`, + ); + } + + const completeResponse = await fetch( + `/api/admin/media/uploads/${encodeURIComponent(uploadId)}/complete`, + { + method: "POST", + }, + ); + + if (!completeResponse.ok) { + throw new Error(await readErrorMessage(completeResponse)); + } + + setUploadProgress(effectiveTotalChunks, effectiveTotalChunks, "Finalizing"); + + const payload = await completeResponse.json(); renderMediaRows(payload.files || []); setText("media-generated-at", payload.generatedAt || "unavailable"); setStatus("media-status", `${file.name} uploaded.`); + clearStoredUploadId(fingerprint); await refreshOverview(); await refreshSchedule(); } catch (error) { console.error("Failed to upload media", error); - setStatus("media-status", "Failed to upload media file.", true); + const details = error instanceof Error ? error.message : "Unknown upload error."; + setStatus( + "media-status", + `Upload interrupted. Re-select the same file to resume. (${details})`, + true, + ); } finally { state.mediaUploading = false; + setTimeout(() => hideUploadProgress(), 1200); } } @@ -565,9 +717,26 @@ function bindEvents() { const uploadInput = document.getElementById( "video-upload", ) as HTMLInputElement | null; + const uploadDropzone = document.getElementById("upload-dropzone"); document .getElementById("upload-trigger-btn") ?.addEventListener("click", () => uploadInput?.click()); + + uploadDropzone?.addEventListener("dragover", (event) => { + event.preventDefault(); + uploadDropzone.classList.add("is-dragover"); + }); + uploadDropzone?.addEventListener("dragleave", () => { + uploadDropzone.classList.remove("is-dragover"); + }); + uploadDropzone?.addEventListener("drop", async (event) => { + event.preventDefault(); + uploadDropzone.classList.remove("is-dragover"); + const droppedFiles = event.dataTransfer?.files; + if (!droppedFiles || droppedFiles.length === 0) return; + await uploadVideo(droppedFiles[0]); + }); + uploadInput?.addEventListener("change", async (event) => { const target = event.target as HTMLInputElement; if (!target.files || target.files.length === 0) return; diff --git a/frontend/src/style.css b/frontend/src/style.css index cfea6e6..de1a2cb 100644 --- a/frontend/src/style.css +++ b/frontend/src/style.css @@ -645,6 +645,22 @@ select { gap: var(--space-2); } +.upload-dropzone.is-dragover { + border-color: var(--brand); + background: #e6f6ef; +} + +.upload-progress { + display: grid; + gap: var(--space-1); + width: min(540px, 100%); +} + +.upload-progress progress { + width: 100%; + height: 0.75rem; +} + .stat-grid { display: grid; grid-template-columns: repeat(4, minmax(0, 1fr));