fix: seed correct monitoring volumes and verify monitoring deployment

.gitea/workflows/cd.yml

@@ -48,7 +48,7 @@ jobs:
           DOCKER_HOST: tcp://192.168.88.40:2375
         run: |
           docker run --rm \
-            -v prometheus-config:/etc/prometheus \
+            -v linearvoid_prometheus-config:/etc/prometheus \
             alpine:latest sh -c '
               cat > /etc/prometheus/prometheus.yml << "PROMEOF"
           global:
@@ -113,6 +113,20 @@ jobs:
                     severity: warning
                   annotations:
                     summary: "Playout in slate fallback"
+                - alert: HighChatLatency
+                  expr: rate(linearvoid_chat_messages_total[5m]) < 0.01
+                  for: 5m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Unusually low chat message rate"
+                - alert: PrometheusWALFailing
+                  expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
+                  for: 5m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Prometheus WAL writes failing"
                 - alert: HighCPU
                   expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
                   for: 10m
@@ -130,6 +144,66 @@ jobs:
           ALERTSEOF
             '
 
+      - name: Seed Grafana provisioning on remote host
+        env:
+          DOCKER_HOST: tcp://192.168.88.40:2375
+        run: |
+          docker run --rm \
+            -v linearvoid_grafana-provisioning:/etc/grafana/provisioning \
+            alpine:latest sh -c '
+              mkdir -p /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards
+              cat > /etc/grafana/provisioning/datasources/datasource.yml << "DSEOF"
+          apiVersion: 1
+          datasources:
+            - name: Prometheus
+              type: prometheus
+              access: proxy
+              url: http://prometheus:9090
+              isDefault: true
+              editable: true
+          DSEOF
+              cat > /etc/grafana/provisioning/dashboards/dashboard.yml << "PROVEOF"
+          apiVersion: 1
+          providers:
+            - name: "default"
+              orgId: 1
+              folder: ""
+              type: file
+              disableDeletion: false
+              updateIntervalSeconds: 10
+              allowUiUpdates: true
+              options:
+                path: /etc/grafana/provisioning/dashboards
+          PROVEOF
+              cat > /etc/grafana/provisioning/dashboards/linearvoid-app.json << "DBEOF"
+          {
+            "title": "Linear Void Monitoring",
+            "panels": [
+              {
+                "title": "Concurrent Viewers",
+                "type": "timeseries",
+                "targets": [{ "expr": "linearvoid_chat_connected_clients" }]
+              },
+              {
+                "title": "Stored Messages",
+                "type": "stat",
+                "targets": [{ "expr": "linearvoid_chat_messages_total" }]
+              },
+              {
+                "title": "Redis Reachable",
+                "type": "stat",
+                "targets": [{ "expr": "linearvoid_chat_redis_up" }]
+              },
+              {
+                "title": "Playout Mode",
+                "type": "stat",
+                "targets": [{ "expr": "linearvoid_playout_mode" }]
+              }
+            ]
+          }
+          DBEOF
+            '
+
       - name: Deploy with docker-compose
         env:
           DOCKER_HOST: tcp://192.168.88.40:2375