feat: monitoring exporters, alerting, rate limiting, backups, incident runbook

- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100)
- Create 8 alerting rules in monitoring/prometheus/alerts.yml
- Update prometheus.yml with all scrape targets + rule_files
- WS connection rate limiting: max 5/IP/60s (Redis INCR)
- Admin login rate limiting: max 5/IP/5min (Redis INCR)
- 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test
- docs/incident-runbook.md with 5-tier triage + escalation
- Update CD workflow: full prometheus config + alerts seeding

monitoring/prometheus/alerts.yml

@@ -0,0 +1,74 @@
+groups:
+  - name: linearvoid
+    rules:
+      - alert: ChatDown
+        expr: up{job="chat"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Chat service is down"
+          description: "Chat service has been unreachable for 2 minutes."
+
+      - alert: PlayoutDown
+        expr: up{job="playout"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Playout service is down"
+          description: "Playout service has been unreachable for 2 minutes."
+
+      - alert: RedisDown
+        expr: linearvoid_chat_redis_up == 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Redis is unreachable"
+          description: "Chat service cannot reach Redis. Rate limiting and presence tracking degraded."
+
+      - alert: PlayoutSlateActive
+        expr: linearvoid_playout_mode == 1
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Playout in slate fallback mode"
+          description: "Playout has switched to emergency slate. Check playout logs."
+
+      - alert: HighChatLatency
+        expr: rate(linearvoid_chat_messages_total[5m]) < 0.01
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Unusually low chat message rate"
+          description: "Message rate below 0.01/s for 5 minutes. Check chat service health."
+
+      - alert: PrometheusWALFailing
+        expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Prometheus WAL writes failing"
+          description: "Prometheus TSDB WAL write failures detected."
+
+      - alert: HighCPU
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High CPU usage"
+          description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}."
+
+      - alert: LowDiskSpace
+        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Low disk space"
+          description: "Less than 10% disk space available on {{ $labels.instance }}."

monitoring/prometheus/prometheus.yml

@@ -1,10 +1,28 @@
 global:
   scrape_interval: 15s
 
+rule_files:
+  - "/etc/prometheus/alerts.yml"
+
 scrape_configs:
   - job_name: "chat"
     static_configs:
       - targets: ["chat:3000"]
+  - job_name: "playout"
+    static_configs:
+      - targets: ["playout:8081"]
+  - job_name: "nginx"
+    static_configs:
+      - targets: ["nginx-exporter:9113"]
+  - job_name: "redis"
+    static_configs:
+      - targets: ["redis-exporter:9121"]
+  - job_name: "postgres"
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+  - job_name: "node"
+    static_configs:
+      - targets: ["node-exporter:9100"]
   - job_name: "prometheus"
     static_configs:
       - targets: ["prometheus:9090"]