feat: monitoring exporters, alerting, rate limiting, backups, incident runbook

- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100) - Create 8 alerting rules in monitoring/prometheus/alerts.yml - Update prometheus.yml with all scrape targets + rule_files - WS connection rate limiting: max 5/IP/60s (Redis INCR) - Admin login rate limiting: max 5/IP/5min (Redis INCR) - 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test - docs/incident-runbook.md with 5-tier triage + escalation - Update CD workflow: full prometheus config + alerts seeding
2026-08-06 19:31:40 +02:00
1 parent b5710548b1
commit 0f81a81029
@@ -57,14 +57,78 @@ jobs:
            scrape_interval: 15s
            evaluation_interval: 15s

+          rule_files:
+            - "/etc/prometheus/alerts.yml"
+
          scrape_configs:
+            - job_name: 'chat'
+              static_configs:
+                - targets: ['chat:3000']
+            - job_name: 'playout'
+              static_configs:
+                - targets: ['playout:8081']
+            - job_name: 'nginx'
+              static_configs:
+                - targets: ['nginx-exporter:9113']
+            - job_name: 'redis'
+              static_configs:
+                - targets: ['redis-exporter:9121']
+            - job_name: 'postgres'
+              static_configs:
+                - targets: ['postgres-exporter:9187']
+            - job_name: 'node'
+              static_configs:
+                - targets: ['node-exporter:9100']
            - job_name: 'prometheus'
              static_configs:
                - targets: ['localhost:9090']
-
-            - job_name: 'linearvoid'
-              static_configs:
-                - targets: ['playout:8081', 'chat:3000', 'nginx:80']
+          EOF
+          cat > monitoring/prometheus/alerts.yml << 'EOF'
+          groups:
+            - name: linearvoid
+              rules:
+                - alert: ChatDown
+                  expr: up{job="chat"} == 0
+                  for: 2m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: "Chat service is down"
+                - alert: PlayoutDown
+                  expr: up{job="playout"} == 0
+                  for: 2m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: "Playout service is down"
+                - alert: RedisDown
+                  expr: linearvoid_chat_redis_up == 0
+                  for: 2m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Redis is unreachable"
+                - alert: PlayoutSlateActive
+                  expr: linearvoid_playout_mode == 1
+                  for: 1m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Playout in slate fallback"
+                - alert: HighCPU
+                  expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "High CPU usage"
+                - alert: LowDiskSpace
+                  expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
+                  for: 5m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Low disk space"
          EOF

      - name: Deploy with docker-compose
@@ -1,9 +1 @@
 # MEMORY.md
-
-## 2026-08-05: Chat Message Admin + CI/CD + Release 1.0.5
-
-**Completed**: Chat Message Admin, CI/CD, Release 1.0.5 committed + tagged + pushed.
-
-**Bug — nested admin page**: Fixed. Root cause: page HTML files were in `frontend/admin/` (Vite source root). Vite multi-page mode injected scripts/styles into every `.html` entry. `fetch("/admin/dashboard.html")` returned a full self-contained page, injected into `#tab-content-container`, nesting duplicate admin shell. Fix: moved all 9 page files to `frontend/public/admin/` — Vite serves `public/` contents verbatim with no processing. Fetch now returns raw HTML fragment.
-
-**Key lesson**: Page fragments loaded via `fetch()` must live under `public/` in Vite projects. Any `.html` in the project source root gets processed as a multi-page entry.
@@ -32,9 +32,9 @@ var (
 	upgrader    = websocket.Upgrader{
 		CheckOrigin: func(r *http.Request) bool { return true },
 	}
-	clients      = make(map[*websocket.Conn]bool)
-	clientMeta   = make(map[*websocket.Conn]clientInfo)
-	mutex        = sync.Mutex{}
+	clients        = make(map[*websocket.Conn]bool)
+	clientMeta     = make(map[*websocket.Conn]clientInfo)
+	mutex          = sync.Mutex{}
 	authSessions   = map[string]time.Time{}
 	authSessionsMu = sync.Mutex{}

@@ -101,17 +101,17 @@ type Presence struct {
 }

 type AdminOverview struct {
-	Viewers           int    `json:"viewers"`
-	StoredMessages    int64  `json:"storedMessages"`
-	ModerationActions int64  `json:"moderationActions"`
-	MediaCount        int64  `json:"mediaCount"`
+	Viewers                   int    `json:"viewers"`
+	StoredMessages            int64  `json:"storedMessages"`
+	ModerationActions         int64  `json:"moderationActions"`
+	MediaCount                int64  `json:"mediaCount"`
 	MediaTotalDurationSeconds int64  `json:"mediaTotalDurationSeconds"`
-	MediaTotalRuntime string `json:"mediaTotalRuntime"`
-	RedisEnabled      bool   `json:"redisEnabled"`
-	RedisClients      int    `json:"redisClients"`
-	RedisMemoryUsed   string `json:"redisMemoryUsed"`
-	DatabaseDriver    string `json:"databaseDriver"`
-	GeneratedAt       string `json:"generatedAt"`
+	MediaTotalRuntime         string `json:"mediaTotalRuntime"`
+	RedisEnabled              bool   `json:"redisEnabled"`
+	RedisClients              int    `json:"redisClients"`
+	RedisMemoryUsed           string `json:"redisMemoryUsed"`
+	DatabaseDriver            string `json:"databaseDriver"`
+	GeneratedAt               string `json:"generatedAt"`
 }

 type PlaylistPayload struct {
@@ -136,15 +136,15 @@ type RoomAnalytics struct {
 }

 type AdminAnalytics struct {
-	TotalMessages       int64           `json:"totalMessages"`
-	MessagesLast24Hours int64           `json:"messagesLast24Hours"`
-	DistinctRooms       int64           `json:"distinctRooms"`
-	ModerationActions   int64           `json:"moderationActions"`
-	MediaCount          int64           `json:"mediaCount"`
-	MediaTotalDurationSeconds int64     `json:"mediaTotalDurationSeconds"`
-	MediaTotalRuntime   string          `json:"mediaTotalRuntime"`
-	TopRooms            []RoomAnalytics `json:"topRooms"`
-	GeneratedAt         string          `json:"generatedAt"`
+	TotalMessages             int64           `json:"totalMessages"`
+	MessagesLast24Hours       int64           `json:"messagesLast24Hours"`
+	DistinctRooms             int64           `json:"distinctRooms"`
+	ModerationActions         int64           `json:"moderationActions"`
+	MediaCount                int64           `json:"mediaCount"`
+	MediaTotalDurationSeconds int64           `json:"mediaTotalDurationSeconds"`
+	MediaTotalRuntime         string          `json:"mediaTotalRuntime"`
+	TopRooms                  []RoomAnalytics `json:"topRooms"`
+	GeneratedAt               string          `json:"generatedAt"`
 }

 type ServiceHealth struct {
@@ -171,9 +171,9 @@ type ActiveUser struct {
 }

 type AdminUsersResponse struct {
-	Users       []ActiveUser `json:"users"`
-	RedisEnabled bool       `json:"redisEnabled"`
-	GeneratedAt string      `json:"generatedAt"`
+	Users        []ActiveUser `json:"users"`
+	RedisEnabled bool         `json:"redisEnabled"`
+	GeneratedAt  string       `json:"generatedAt"`
 }

 type ChatLogEntry struct {
@@ -205,12 +205,12 @@ type ScheduleEntry struct {
 }

 type AdminScheduleResponse struct {
-	Entries               []ScheduleEntry `json:"entries"`
-	TotalEntries          int             `json:"totalEntries"`
-	MatchedEntries        int             `json:"matchedEntries"`
-	TotalDurationSeconds  int             `json:"totalDurationSeconds"`
-	TotalRuntime          string          `json:"totalRuntime"`
-	GeneratedAt           string          `json:"generatedAt"`
+	Entries              []ScheduleEntry `json:"entries"`
+	TotalEntries         int             `json:"totalEntries"`
+	MatchedEntries       int             `json:"matchedEntries"`
+	TotalDurationSeconds int             `json:"totalDurationSeconds"`
+	TotalRuntime         string          `json:"totalRuntime"`
+	GeneratedAt          string          `json:"generatedAt"`
 }

 type MediaFileEntry struct {
@@ -234,12 +234,12 @@ type AdminMediaResponse struct {
 }

 type AdminMediaUploadInitRequest struct {
-	UploadID   string `json:"uploadId"`
-	FileName   string `json:"fileName"`
-	FileSize   int64  `json:"fileSize"`
-	MimeType   string `json:"mimeType"`
-	ChunkSize  int64  `json:"chunkSize"`
-	TotalChunks int   `json:"totalChunks"`
+	UploadID    string `json:"uploadId"`
+	FileName    string `json:"fileName"`
+	FileSize    int64  `json:"fileSize"`
+	MimeType    string `json:"mimeType"`
+	ChunkSize   int64  `json:"chunkSize"`
+	TotalChunks int    `json:"totalChunks"`
 }

 type AdminMediaUploadSession struct {
@@ -254,13 +254,13 @@ type AdminMediaUploadSession struct {
 }

 type AdminMediaUploadStatusResponse struct {
-	UploadID      string  `json:"uploadId"`
-	FileName      string  `json:"fileName"`
-	FileSize      int64   `json:"fileSize"`
-	ChunkSize     int64   `json:"chunkSize"`
-	TotalChunks   int     `json:"totalChunks"`
-	UploadedChunks int    `json:"uploadedChunks"`
-	Progress      float64 `json:"progress"`
+	UploadID       string  `json:"uploadId"`
+	FileName       string  `json:"fileName"`
+	FileSize       int64   `json:"fileSize"`
+	ChunkSize      int64   `json:"chunkSize"`
+	TotalChunks    int     `json:"totalChunks"`
+	UploadedChunks int     `json:"uploadedChunks"`
+	Progress       float64 `json:"progress"`
 }

 type AdminLoginPayload struct {
@@ -278,9 +278,9 @@ var databaseDriverName = "sqlite"
 const adminSessionCookieName = "linearvoid_admin_session"

 const (
-	maxAdminUploadBytes      int64 = 2 * 1024 * 1024 * 1024
-	defaultAdminChunkSize    int64 = 8 * 1024 * 1024
-	maxAdminChunkSize        int64 = 32 * 1024 * 1024
+	maxAdminUploadBytes   int64 = 2 * 1024 * 1024 * 1024
+	defaultAdminChunkSize int64 = 8 * 1024 * 1024
+	maxAdminChunkSize     int64 = 32 * 1024 * 1024
 )

 // initRedis initializes Redis for presence, rate limiting, and pub/sub (ADR-007)
@@ -380,11 +380,11 @@ func parseRedisInfoValue(info, key string) string {

 func getAdminOverview() AdminOverview {
 	overview := AdminOverview{
-		RedisEnabled:    redisClient != nil,
-		RedisMemoryUsed: "unavailable",
-		DatabaseDriver:  databaseDriverName,
+		RedisEnabled:      redisClient != nil,
+		RedisMemoryUsed:   "unavailable",
+		DatabaseDriver:    databaseDriverName,
 		MediaTotalRuntime: "00:00:00",
-		GeneratedAt:     time.Now().UTC().Format(time.RFC3339),
+		GeneratedAt:       time.Now().UTC().Format(time.RFC3339),
 	}

 	mutex.Lock()
@@ -422,6 +422,23 @@ func handleAdminLogin(w http.ResponseWriter, r *http.Request) {
 		return
 	}

+	// Rate limit: max 5 attempts per IP per 5 minutes
+	clientIP := r.Header.Get("X-Forwarded-For")
+	if clientIP == "" {
+		clientIP = r.RemoteAddr
+	}
+	if redisClient != nil {
+		loginKey := fmt.Sprintf("login_attempts:%s", clientIP)
+		count, _ := redisClient.Incr(context.Background(), loginKey).Result()
+		if count == 1 {
+			redisClient.Expire(context.Background(), loginKey, 5*time.Minute)
+		}
+		if count > 5 {
+			http.Error(w, "too many login attempts, try again later", http.StatusTooManyRequests)
+			return
+		}
+	}
+
 	var payload AdminLoginPayload
 	if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
 		http.Error(w, fmt.Sprintf("invalid request body: %v", err), http.StatusBadRequest)
@@ -907,9 +924,9 @@ func handleAdminModerationAction(w http.ResponseWriter, r *http.Request) {

 func getAdminAnalytics() AdminAnalytics {
 	analytics := AdminAnalytics{
-		TopRooms:    []RoomAnalytics{},
+		TopRooms:          []RoomAnalytics{},
 		MediaTotalRuntime: "00:00:00",
-		GeneratedAt: time.Now().UTC().Format(time.RFC3339),
+		GeneratedAt:       time.Now().UTC().Format(time.RFC3339),
 	}

 	if db == nil {
@@ -1114,7 +1131,7 @@ func handleAdminUsers(w http.ResponseWriter, r *http.Request) {

 	case http.MethodPost:
 		var payload struct {
-			Action   string `json:"action"`   // mute, unmute, ban, unban
+			Action   string `json:"action"` // mute, unmute, ban, unban
 			Username string `json:"username"`
 			Duration int    `json:"duration"` // minutes, for mute
 		}
@@ -1309,13 +1326,13 @@ func buildUploadStatus(session AdminMediaUploadSession) AdminMediaUploadStatusRe
 	}

 	return AdminMediaUploadStatusResponse{
-		UploadID:      session.UploadID,
-		FileName:      session.FileName,
-		FileSize:      session.FileSize,
-		ChunkSize:     session.ChunkSize,
-		TotalChunks:   session.TotalChunks,
+		UploadID:       session.UploadID,
+		FileName:       session.FileName,
+		FileSize:       session.FileSize,
+		ChunkSize:      session.ChunkSize,
+		TotalChunks:    session.TotalChunks,
 		UploadedChunks: uploadedChunks,
-		Progress:      progress,
+		Progress:       progress,
 	}
 }

@@ -2165,6 +2182,25 @@ func broadcast(msg Message) {

 // handleConnection handles WebSocket connections for chat (public rooms, private DMs, moderation)
 func handleConnection(w http.ResponseWriter, r *http.Request) {
+	// IP-based connection rate limiting: max 5 concurrent per IP
+	clientIP := r.Header.Get("X-Forwarded-For")
+	if clientIP == "" {
+		clientIP = r.RemoteAddr
+	}
+	if redisClient != nil {
+		ipKey := fmt.Sprintf("ws_conn:%s", clientIP)
+		count, _ := redisClient.Incr(context.Background(), ipKey).Result()
+		if count == 1 {
+			redisClient.Expire(context.Background(), ipKey, 60*time.Second)
+		}
+		if count > 5 {
+			http.Error(w, "too many connections from this IP", http.StatusTooManyRequests)
+			return
+		}
+		// Decrement on disconnect
+		defer redisClient.Decr(context.Background(), ipKey)
+	}
+
 	conn, err := upgrader.Upgrade(w, r, nil)
 	if err != nil {
 		log.Println("Upgrade error:", err)
@@ -2649,4 +2685,4 @@ func main() {

 	log.Printf("Chat Backend starting on :%s (WebSocket at /ws, health at /health, database persistence enabled)", port)
 	log.Fatal(http.ListenAndServe(":"+port, nil))
-}
+}
@@ -104,6 +104,56 @@ services:
      timeout: 5s
      retries: 5

+  nginx-exporter:
+    image: nginx/nginx-prometheus-exporter:latest
+    container_name: linearvoid-nginx-exporter
+    restart: unless-stopped
+    command:
+      - "-nginx.scrape-uri=http://nginx/nginx-status"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - nginx
+
+  redis-exporter:
+    image: oliver006/redis_exporter:latest
+    container_name: linearvoid-redis-exporter
+    restart: unless-stopped
+    command:
+      - "-redis.addr=redis://redis:6379"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - redis
+
+  postgres-exporter:
+    image: prometheuscommunity/postgres-exporter:latest
+    container_name: linearvoid-postgres-exporter
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: "postgresql://linearvoid:${POSTGRES_PASSWORD:-linearvoid}@db:5432/linearvoid?sslmode=disable"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - db
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: linearvoid-node-exporter
+    restart: unless-stopped
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    networks:
+      - linearvoid-net
+
  prometheus:
    image: prom/prometheus:latest
    container_name: linearvoid-prometheus
@@ -97,6 +97,56 @@ services:
    networks:
      - linearvoid-net

+  nginx-exporter:
+    image: nginx/nginx-prometheus-exporter:latest
+    container_name: linearvoid-nginx-exporter
+    restart: unless-stopped
+    command:
+      - "-nginx.scrape-uri=http://nginx/nginx-status"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - nginx
+
+  redis-exporter:
+    image: oliver006/redis_exporter:latest
+    container_name: linearvoid-redis-exporter
+    restart: unless-stopped
+    command:
+      - "-redis.addr=redis://redis:6379"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - redis
+
+  postgres-exporter:
+    image: prometheuscommunity/postgres-exporter:latest
+    container_name: linearvoid-postgres-exporter
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: "postgresql://linearvoid:${POSTGRES_PASSWORD:-linearvoid}@db:5432/linearvoid?sslmode=disable"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - db
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: linearvoid-node-exporter
+    restart: unless-stopped
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    networks:
+      - linearvoid-net
+
  prometheus:
    image: prom/prometheus:latest
    container_name: linearvoid-prometheus
@@ -0,0 +1,170 @@
+# Incident Runbook
+
+Triage order for Linear Void incidents. Work top-to-bottom — each tier depends on the one above.
+
+## Triage Order
+
+```
+1. Edge Reachability (Cloudflare)
+2. Origin Health (VPS/Docker)
+3. Playout Progression (HLS)
+4. Chat Connectivity (WebSocket)
+5. Data-Plane (Redis/DB)
+```
+
+---
+
+## 1. Edge Reachability (Cloudflare)
+
+**Symptoms:** `linearvoid.tv` not resolving, SSL errors, 5xx from Cloudflare.
+
+```bash
+# Check DNS
+dig linearvoid.tv
+dig www.linearvoid.tv
+
+# Check Cloudflare status
+curl -I https://linearvoid.tv/health
+```
+
+**Common causes:**
+- DNS misconfiguration (A record pointing to wrong IP)
+- Cloudflare proxy disabled (grey cloud → origin exposed directly)
+- SSL certificate expired (check Cloudflare dashboard → SSL/TLS)
+- Cloudflare outage → https://www.cloudflarestatus.com
+
+**Mitigation:**
+- Verify A record: `linearvoid.tv` → origin IP (proxied, orange cloud)
+- Bypass Cloudflare temporarily: `curl -H "Host: linearvoid.tv" http://<origin-ip>/health`
+
+---
+
+## 2. Origin Health (VPS/Docker)
+
+**Symptoms:** Cloudflare returns 502/504, containers down.
+
+```bash
+# Remote Docker (test lab)
+docker -H tcp://192.168.88.40:2375 ps
+docker -H tcp://192.168.88.40:2375 logs linearvoid-nginx --tail 50
+
+# Direct health check
+curl http://192.168.88.40/health
+```
+
+**Common causes:**
+- Host out of disk space: `df -h`
+- Docker daemon stopped: `systemctl status docker`
+- Container restart loop: `docker ps -a --filter "status=exited"`
+- Resource exhaustion (OOM): `dmesg | grep -i oom`
+
+**Mitigation:**
+- Restart stack: `docker compose -f docker-compose.prod.yml up -d`
+- Free disk: `docker system prune -f`
+- Check logs per service: `docker logs linearvoid-<service> --tail 100`
+
+---
+
+## 3. Playout Progression (HLS)
+
+**Symptoms:** Viewer sees slate or buffering, playlist not advancing.
+
+```bash
+# Check playout health
+curl http://192.168.88.40:8081/health
+# Response includes "mode": "live" or "slate"
+
+# Check HLS output
+curl -I http://192.168.88.40/hls/playlist.m3u8
+ls -la data/hls/
+
+# Check playout logs
+docker logs linearvoid-playout --tail 100
+```
+
+**Common causes:**
+- FFmpeg crash → slate fallback active (health shows `degraded`)
+- Playlist file empty/missing: `cat data/playlist/playlist.txt`
+- Media file missing from playlist: `ls data/media/`
+- Disk full → HLS segments can't write
+
+**Mitigation:**
+- Verify playlist has valid entries
+- Check media files exist at paths in playlist
+- Restart playout: `docker compose restart playout`
+- Generate demo playlist if empty: see `backend/playout/start-playout.sh`
+
+---
+
+## 4. Chat Connectivity (WebSocket)
+
+**Symptoms:** Chat not loading, messages not sending, presence broken.
+
+```bash
+# Check chat health
+curl http://192.168.88.40:3000/health
+
+# Test WebSocket
+wscat -c ws://192.168.88.40/ws?username=test&room=main
+
+# Check chat logs
+docker logs linearvoid-chat --tail 100
+```
+
+**Common causes:**
+- Redis down → presence/rate-limiting degraded (chat still works)
+- Database down → messages not persisting
+- Port not exposed: `docker port linearvoid-chat`
+
+**Mitigation:**
+- Restart chat: `docker compose restart chat`
+- Check Redis: `docker exec linearvoid-redis redis-cli ping`
+- Check DB: `docker exec linearvoid-db pg_isready -U linearvoid`
+- Chat works without Redis (graceful degradation) — check logs for "Redis connection warning"
+
+---
+
+## 5. Data-Plane (Redis/DB)
+
+**Symptoms:** Chat works but no persistence/presence, admin stats empty.
+
+```bash
+# Redis
+docker exec linearvoid-redis redis-cli ping
+docker exec linearvoid-redis redis-cli info memory
+
+# PostgreSQL
+docker exec linearvoid-db pg_isready -U linearvoid
+docker exec linearvoid-db psql -U linearvoid -c "SELECT count(*) FROM messages;"
+```
+
+**Common causes:**
+- Redis OOM (check `maxmemory` setting)
+- PostgreSQL connection limit reached
+- Disk full on volume mount
+
+**Mitigation:**
+- Redis: `docker compose restart redis` (AOF persist enabled)
+- PostgreSQL: `docker compose restart db`
+- Restore from backup: see `scripts/verify-backups.sh` for procedure
+
+---
+
+## Escalation
+
+| Severity | When | Action |
+|---|---|---|
+| Critical | Slate active >30min, complete outage | Trigger DR plan, provision fallback host |
+| Warning | Redis down, high CPU >30min | Investigate during business hours, restart service |
+| Info | Single service blip | Monitor, no action unless recurring |
+
+## Post-Incident
+
+1. Document timeline in incident channel
+2. Identify root cause
+3. Add prevention to TODO.md if actionable
+4. Update this runbook if new failure mode discovered
+
+---
+
+**Recovery targets:** RPO ≤24h, RTO ≤60min. Quarterly DR drill: simulate host loss, restore from backup.
@@ -0,0 +1,74 @@
+groups:
+  - name: linearvoid
+    rules:
+      - alert: ChatDown
+        expr: up{job="chat"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Chat service is down"
+          description: "Chat service has been unreachable for 2 minutes."
+
+      - alert: PlayoutDown
+        expr: up{job="playout"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Playout service is down"
+          description: "Playout service has been unreachable for 2 minutes."
+
+      - alert: RedisDown
+        expr: linearvoid_chat_redis_up == 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Redis is unreachable"
+          description: "Chat service cannot reach Redis. Rate limiting and presence tracking degraded."
+
+      - alert: PlayoutSlateActive
+        expr: linearvoid_playout_mode == 1
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Playout in slate fallback mode"
+          description: "Playout has switched to emergency slate. Check playout logs."
+
+      - alert: HighChatLatency
+        expr: rate(linearvoid_chat_messages_total[5m]) < 0.01
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Unusually low chat message rate"
+          description: "Message rate below 0.01/s for 5 minutes. Check chat service health."
+
+      - alert: PrometheusWALFailing
+        expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Prometheus WAL writes failing"
+          description: "Prometheus TSDB WAL write failures detected."
+
+      - alert: HighCPU
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High CPU usage"
+          description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}."
+
+      - alert: LowDiskSpace
+        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Low disk space"
+          description: "Less than 10% disk space available on {{ $labels.instance }}."
@@ -1,10 +1,28 @@
 global:
  scrape_interval: 15s

+rule_files:
+  - "/etc/prometheus/alerts.yml"
+
 scrape_configs:
  - job_name: "chat"
    static_configs:
      - targets: ["chat:3000"]
+  - job_name: "playout"
+    static_configs:
+      - targets: ["playout:8081"]
+  - job_name: "nginx"
+    static_configs:
+      - targets: ["nginx-exporter:9113"]
+  - job_name: "redis"
+    static_configs:
+      - targets: ["redis-exporter:9121"]
+  - job_name: "postgres"
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+  - job_name: "node"
+    static_configs:
+      - targets: ["node-exporter:9100"]
  - job_name: "prometheus"
    static_configs:
      - targets: ["prometheus:9090"]
@@ -0,0 +1,45 @@
+#!/bin/bash
+# Daily PostgreSQL backup to R2 (14-day retention)
+# Schedule: cron daily. Requires: pg_dump, aws CLI or rclone for R2.
+set -e
+
+BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}"
+RETENTION_DAYS="${RETENTION_DAYS:-14}"
+R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
+DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}"
+
+TIMESTAMP=$(date +%Y%m%d)
+BACKUP_FILE="$BACKUP_DIR/daily/db_${TIMESTAMP}.dump"
+
+mkdir -p "$(dirname "$BACKUP_FILE")"
+
+echo "[$(date -Iseconds)] Starting daily backup..."
+
+# Extract connection details from DATABASE_URL
+DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
+DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
+DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p')
+DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p')
+DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p')
+
+export PGPASSWORD="${DB_PASS}"
+
+# Dump
+pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \
+        -Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}"
+
+echo "  Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
+
+# Upload to R2
+if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then
+    aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/daily/db_${TIMESTAMP}.dump" \
+        --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \
+        echo "  Uploaded to R2: daily/db_${TIMESTAMP}.dump" || \
+        echo "  WARNING: R2 upload failed"
+fi
+
+# Cleanup old backups
+find "$BACKUP_DIR/daily" -name "db_*.dump" -mtime "+${RETENTION_DAYS}" -delete 2>/dev/null || true
+echo "  Cleaned up backups older than ${RETENTION_DAYS} days"
+
+echo "[$(date -Iseconds)] Daily backup complete."
@@ -0,0 +1,48 @@
+#!/bin/bash
+# Weekly full PostgreSQL backup to R2 (8-week retention)
+# Schedule: cron weekly (runs backup-db.sh with weekly prefix)
+set -e
+
+BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}"
+RETENTION_WEEKS="${RETENTION_WEEKS:-8}"
+R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
+DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}"
+
+TIMESTAMP=$(date +%Y%m%d)
+BACKUP_FILE="$BACKUP_DIR/weekly/db_${TIMESTAMP}.dump"
+
+mkdir -p "$(dirname "$BACKUP_FILE")"
+
+echo "[$(date -Iseconds)] Starting weekly full backup..."
+
+DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
+DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
+DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p')
+DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p')
+DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p')
+
+export PGPASSWORD="${DB_PASS}"
+
+pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \
+        -Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}"
+
+echo "  Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
+
+# Verify integrity
+pg_restore --list "$BACKUP_FILE" > /dev/null 2>&1 && \
+    echo "  Integrity check: PASSED" || \
+    echo "  WARNING: Integrity check FAILED"
+
+# Upload
+if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then
+    aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/weekly/db_${TIMESTAMP}.dump" \
+        --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \
+        echo "  Uploaded to R2: weekly/db_${TIMESTAMP}.dump" || \
+        echo "  WARNING: R2 upload failed"
+fi
+
+# Cleanup
+find "$BACKUP_DIR/weekly" -name "db_*.dump" -mtime "+$((RETENTION_WEEKS * 7))" -delete 2>/dev/null || true
+echo "  Cleaned up backups older than ${RETENTION_WEEKS} weeks"
+
+echo "[$(date -Iseconds)] Weekly backup complete."
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Weekly backup integrity verification.
+# Downloads latest backup from R2, restores into temp container, runs sanity checks.
+# Schedule: cron weekly (after weekly backup).
+set -e
+
+R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
+TEMP_DIR=$(mktemp -d)
+trap "rm -rf $TEMP_DIR" EXIT
+
+echo "[$(date -Iseconds)] Starting backup integrity verification..."
+
+# Download latest weekly backup from R2
+if [ -z "$R2_ENDPOINT" ] || [ -z "$R2_ACCESS_KEY_ID" ]; then
+    echo "  SKIP: R2 credentials not configured (R2_ENDPOINT, R2_ACCESS_KEY_ID)"
+    exit 0
+fi
+
+LATEST=$(aws s3 ls "s3://${R2_BUCKET}/weekly/" --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null | sort | tail -1 | awk '{print $4}')
+if [ -z "$LATEST" ]; then
+    echo "  SKIP: No weekly backups found in R2"
+    exit 0
+fi
+
+echo "  Downloading: $LATEST"
+aws s3 cp "s3://${R2_BUCKET}/weekly/${LATEST}" "$TEMP_DIR/backup.dump" \
+    --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null
+
+echo "  File size: $(du -h "$TEMP_DIR/backup.dump" | cut -f1)"
+
+# Restore into temp PostgreSQL container
+echo "  Starting temp PostgreSQL..."
+docker run -d --name lv-backup-test-$$ \
+    -e POSTGRES_PASSWORD=test \
+    -e POSTGRES_DB=linearvoid \
+    -p 15432:5432 \
+    postgres:15-alpine 2>/dev/null
+
+sleep 5
+
+# Wait for PG to be ready
+for i in $(seq 1 10); do
+    if docker exec lv-backup-test-$$ pg_isready -U postgres 2>/dev/null; then
+        break
+    fi
+    sleep 2
+done
+
+# Restore
+echo "  Restoring backup..."
+docker cp "$TEMP_DIR/backup.dump" "lv-backup-test-$$:/tmp/backup.dump"
+docker exec lv-backup-test-$$ pg_restore -U postgres -d linearvoid /tmp/backup.dump 2>/dev/null && \
+    echo "  Restore: OK" || { echo "  Restore: FAILED"; docker rm -f lv-backup-test-$$ 2>/dev/null; exit 1; }
+
+# Sanity checks
+MSG_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM messages;" 2>/dev/null || echo "0")
+MOD_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM moderation_logs;" 2>/dev/null || echo "0")
+echo "  Messages: $MSG_COUNT, Moderation: $MOD_COUNT"
+
+# Cleanup
+docker rm -f lv-backup-test-$$ 2>/dev/null
+
+echo "[$(date -Iseconds)] Backup verification complete. OK."