feat: add metrics, observability, test fixtures, and scaling playbook artifacts

monitoring/grafana/dashboard.json

@@ -34,6 +34,33 @@
       "title": "Chat Scrape Health",
       "type": "stat",
       "targets": [{ "expr": "up{job=\"chat\"}" }]
+    },
+    {
+      "title": "Cache Hit Ratio",
+      "type": "timeseries",
+      "targets": [
+        {
+          "expr": "linearvoid_cache_hit_ratio"
+        }
+      ]
+    },
+    {
+      "title": "Redis Memory Usage",
+      "type": "timeseries",
+      "targets": [
+        {
+          "expr": "linearvoid_chat_redis_memory_usage_ratio"
+        }
+      ]
+    },
+    {
+      "title": "Chat Request Latency p95",
+      "type": "timeseries",
+      "targets": [
+        {
+          "expr": "histogram_quantile(0.95, sum(rate(linearvoid_chat_request_duration_seconds_bucket[5m])) by (le))"
+        }
+      ]
     }
   ]
 }

monitoring/prometheus/alerts.yml

@@ -38,13 +38,13 @@ groups:
           description: "Playout has switched to emergency slate. Check playout logs."
 
       - alert: HighChatLatency
-        expr: rate(linearvoid_chat_messages_total[5m]) < 0.01
+        expr: histogram_quantile(0.95, sum(rate(linearvoid_chat_request_duration_seconds_bucket[5m])) by (le)) > 1
         for: 5m
         labels:
           severity: warning
         annotations:
-          summary: "Unusually low chat message rate"
-          description: "Message rate below 0.01/s for 5 minutes. Check chat service health."
+          summary: "Chat request latency is high"
+          description: "Chat request p95 has been above 1s for 5 minutes. Check chat handlers and dependencies."
 
       - alert: PrometheusWALFailing
         expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
@@ -64,6 +64,24 @@ groups:
           summary: "High CPU usage"
           description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}."
 
+      - alert: LowCacheHitRatio
+        expr: linearvoid_cache_hit_ratio < 0.9
+        for: 15m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Cloudflare cache hit ratio is below target"
+          description: "Cache hit ratio has been below 90% for 15 minutes on {{ $labels.hostname }}."
+
+      - alert: HighRedisMemoryUsage
+        expr: linearvoid_chat_redis_memory_usage_ratio > 0.85
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Redis memory usage is above target"
+          description: "Redis memory usage has been above 85% for 10 minutes on {{ $labels.instance }}."
+
       - alert: LowDiskSpace
         expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
         for: 5m