diff --git a/.github/instructions/BACKLOG.md b/.github/instructions/BACKLOG.md index 591ab89..4f73d35 100644 --- a/.github/instructions/BACKLOG.md +++ b/.github/instructions/BACKLOG.md @@ -1,5 +1,7 @@ # BACKLOG +Status reflects 2026-08-06 code scan + implementation batches. + ## Playout & Delivery - [x] Implement FFmpeg + MediaMTX/SRS continuous HLS playout from playlist @@ -55,15 +57,15 @@ ## Infrastructure — Docker - [x] Create Dockerfiles for playout, chat, nginx services -- [x] Create docker-compose.yml for all services +- [x] Create docker-compose.yml for all services (11 containers) - [x] Create docker-compose.prod.yml override for production ## Infrastructure — CI/CD (Gitea Actions) -- [x] Implement CI workflow (lint, build, test, push images) +- [x] Implement CI workflow (lint, build, vet, push images) - [x] Implement CD workflow (pull, deploy, health checks — IP-agnostic) - [x] Generate .env from Gitea secrets in CD workflow -- [x] Seed Prometheus config into named volume during deploy +- [x] Seed Prometheus config + alerts into named volume during deploy - [x] Implement health check validation with retry in CD pipeline ## Cloudflare Configuration @@ -79,10 +81,10 @@ ## Monitoring & Observability -- [x] Set up Prometheus with chat scrape target -- [ ] Add nginx/redis/postgres/node exporters +- [x] Set up Prometheus with chat/playout/nginx/redis/postgres/node scrape targets +- [x] Add nginx-exporter, redis-exporter, postgres-exporter, node-exporter - [x] Import Grafana app + operational dashboards -- [ ] Define Prometheus alerting rules +- [x] Define Prometheus alerting rules (8 rules: ChatDown, PlayoutDown, RedisDown, etc.) - [ ] Set up UptimeRobot external monitoring ## Security @@ -90,15 +92,18 @@ - [x] Implement admin cookie-session auth (HttpOnly, SameSite=Strict, Secure when TLS) - [x] Implement server-side session expiration - [ ] Enforce HTTPS/WSS at edge and origin -- [x] Implement rate limiting for chat messages with Redis windowing +- [x] Implement rate limiting for chat messages (Redis windowing) +- [x] Implement WebSocket connection rate limiting (5/IP/60s) +- [x] Implement admin login rate limiting (5/IP/5min) ## Data & Storage - [x] Set up PostgreSQL/SQLite for chat history and moderation data - [x] Set up Redis with append-only persistence + maxmemory policy -- [ ] Implement daily/weekly/monthly Postgres backups -- [ ] Implement backup integrity verification -- [ ] Document and rehearse recovery runbook +- [x] Implement daily Postgres backups to R2 (14-day retention) +- [x] Implement weekly full backup snapshots (8-week retention) +- [x] Implement backup integrity verification (restore test) +- [ ] Document and rehearse recovery runbook (quarterly drill) ## Scaling & Performance @@ -117,5 +122,5 @@ - [ ] Document deployment runbook in DEPLOYMENT.md - [ ] Document monitoring setup in MONITORING.md - [ ] Document security model in SECURITY.md -- [ ] Create incident runbook +- [x] Create incident runbook (docs/incident-runbook.md) - [ ] Write Level 2/3 building block views diff --git a/MONITORING.md b/MONITORING.md index 8c333ac..3af392b 100644 --- a/MONITORING.md +++ b/MONITORING.md @@ -4,12 +4,12 @@ Prometheus + Grafana monitoring for Linear Void. ## Architecture -``` -┌─────────────┐ ┌──────────────┐ ┌───────────┐ -│ Exporters │───▶│ Prometheus │───▶│ Grafana │ -│ (7 targets) │ │ :9090 │ │ :3001 │ -└─────────────┘ └──────┬───────┘ └───────────┘ - │ +```text +┌──────────────┐ ┌──────────────┐ ┌───────────┐ +│ Exporters │───> │ Prometheus │───>│ Grafana │ +│ (7 targets) │ │ :9090 │ │ :3001 │ +└──────────────┘ └───────┬──────┘ └───────────┘ + │ alerts.yml (8 rules) ``` diff --git a/README.md b/README.md index 8ead4a3..867e635 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The architecture (detailed in [`docs/architecture/README.md`](docs/architecture/README.md)) uses: -- Hetzner Ashburn VPS for playout (FFmpeg + MediaMTX/SRS) and chat *(planned production; currently test lab at 192.168.88.40)*. +- Hetzner Ashburn VPS for playout (FFmpeg + MediaMTX/SRS) and chat _(planned production; currently test lab at 192.168.88.40)_. - Cloudflare R2 + CDN with aggressive immutable caching of HLS `.ts` segments ($0 egress). - Lightweight static frontend and WebSocket chat backend. - Gitea self-hosted CI/CD (Actions) and Container Registry. diff --git a/backend/chat/src/main.go b/backend/chat/src/main.go index ab7791a..47dbe4c 100644 --- a/backend/chat/src/main.go +++ b/backend/chat/src/main.go @@ -41,8 +41,17 @@ var ( // Rate limit config (env-overridable) rateLimitMax = 10 rateLimitWindow = 10 * time.Second + + // Instance ID for multi-instance pub/sub deduplication + instanceID = fmt.Sprintf("chat-%s", generateShortID()) ) +func generateShortID() string { + b := make([]byte, 6) + rand.Read(b) + return hex.EncodeToString(b) +} + type clientInfo struct { Username string Room string @@ -2161,7 +2170,7 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) { fmt.Fprintln(w, "# HELP linearvoid_chat_build_info Static build info for chat service.") fmt.Fprintln(w, "# TYPE linearvoid_chat_build_info gauge") - fmt.Fprintln(w, "linearvoid_chat_build_info{service=\"chat\",database_driver=\""+databaseDriverName+"\"} 1") + fmt.Fprintln(w, "linearvoid_chat_build_info{service=\"chat\",database_driver=\""+databaseDriverName+"\",instance_id=\""+instanceID+"\"} 1") } // broadcast sends a message to all connected clients in a room and saves to DB @@ -2172,12 +2181,23 @@ func broadcast(msg Message) { // Persist to configured SQL backend. db.Create(&msg) + // Local fan-out for client := range clients { if err := client.WriteJSON(msg); err != nil { client.Close() delete(clients, client) + delete(clientMeta, client) } } + + // Redis pub/sub for multi-instance fan-out + if redisClient != nil && msg.Room != "" { + payload, _ := json.Marshal(map[string]any{ + "instance": instanceID, + "msg": msg, + }) + redisClient.Publish(context.Background(), fmt.Sprintf("room:%s:messages", msg.Room), payload) + } } // handleConnection handles WebSocket connections for chat (public rooms, private DMs, moderation) @@ -2333,6 +2353,47 @@ func pushChatHistory(conn *websocket.Conn, room string, limit int) { // dmsSent tracks total private messages for Prometheus metrics (set from handleMetrics). var dmsSent int64 +// subscribeRoomMessages listens for Redis pub/sub messages and fans out locally. +func subscribeRoomMessages() { + if redisClient == nil { + return + } + pubsub := redisClient.PSubscribe(context.Background(), "room:*:messages") + defer pubsub.Close() + + log.Printf("Redis pub/sub subscriber started (instance %s)", instanceID) + + for { + msg, err := pubsub.ReceiveMessage(context.Background()) + if err != nil { + log.Printf("Redis pub/sub error: %v — retrying in 5s", err) + time.Sleep(5 * time.Second) + pubsub = redisClient.PSubscribe(context.Background(), "room:*:messages") + continue + } + + var envelope struct { + Instance string `json:"instance"` + Msg Message `json:"msg"` + } + if err := json.Unmarshal([]byte(msg.Payload), &envelope); err != nil { + continue + } + + // Skip messages originating from this instance + if envelope.Instance == instanceID { + continue + } + + // Fan out to local clients + mutex.Lock() + for client := range clients { + client.WriteJSON(envelope.Msg) + } + mutex.Unlock() + } +} + // handlePrivateDM routes private messages to the specific user pair func handlePrivateDM(msg Message) { msg.Type = "private" @@ -2640,6 +2701,11 @@ func main() { initRedis() initDB() + // Subscribe to Redis pub/sub for multi-instance message fan-out + go subscribeRoomMessages() + + log.Printf("Instance ID: %s", instanceID) + http.HandleFunc("/ws", handleConnection) http.HandleFunc("/api/admin/login", handleAdminLogin) http.HandleFunc("/api/admin/logout", handleAdminLogout)