feat: monitoring exporters, alerting, rate limiting, backups, incident runbook

- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100)
- Create 8 alerting rules in monitoring/prometheus/alerts.yml
- Update prometheus.yml with all scrape targets + rule_files
- WS connection rate limiting: max 5/IP/60s (Redis INCR)
- Admin login rate limiting: max 5/IP/5min (Redis INCR)
- 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test
- docs/incident-runbook.md with 5-tier triage + escalation
- Update CD workflow: full prometheus config + alerts seeding

.gitea/workflows/cd.yml

@@ -57,14 +57,78 @@ jobs:
             scrape_interval: 15s
             evaluation_interval: 15s
 
+          rule_files:
+            - "/etc/prometheus/alerts.yml"
+
           scrape_configs:
+            - job_name: 'chat'
+              static_configs:
+                - targets: ['chat:3000']
+            - job_name: 'playout'
+              static_configs:
+                - targets: ['playout:8081']
+            - job_name: 'nginx'
+              static_configs:
+                - targets: ['nginx-exporter:9113']
+            - job_name: 'redis'
+              static_configs:
+                - targets: ['redis-exporter:9121']
+            - job_name: 'postgres'
+              static_configs:
+                - targets: ['postgres-exporter:9187']
+            - job_name: 'node'
+              static_configs:
+                - targets: ['node-exporter:9100']
             - job_name: 'prometheus'
               static_configs:
                 - targets: ['localhost:9090']
-
-            - job_name: 'linearvoid'
-              static_configs:
-                - targets: ['playout:8081', 'chat:3000', 'nginx:80']
+          EOF
+          cat > monitoring/prometheus/alerts.yml << 'EOF'
+          groups:
+            - name: linearvoid
+              rules:
+                - alert: ChatDown
+                  expr: up{job="chat"} == 0
+                  for: 2m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: "Chat service is down"
+                - alert: PlayoutDown
+                  expr: up{job="playout"} == 0
+                  for: 2m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: "Playout service is down"
+                - alert: RedisDown
+                  expr: linearvoid_chat_redis_up == 0
+                  for: 2m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Redis is unreachable"
+                - alert: PlayoutSlateActive
+                  expr: linearvoid_playout_mode == 1
+                  for: 1m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Playout in slate fallback"
+                - alert: HighCPU
+                  expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "High CPU usage"
+                - alert: LowDiskSpace
+                  expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
+                  for: 5m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Low disk space"
           EOF
 
       - name: Deploy with docker-compose