zwitschi
0f81a81029
- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100) - Create 8 alerting rules in monitoring/prometheus/alerts.yml - Update prometheus.yml with all scrape targets + rule_files - WS connection rate limiting: max 5/IP/60s (Redis INCR) - Admin login rate limiting: max 5/IP/5min (Redis INCR) - 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test - docs/incident-runbook.md with 5-tier triage + escalation - Update CD workflow: full prometheus config + alerts seeding
75 lines
2.4 KiB
YAML
75 lines
2.4 KiB
YAML
groups:
|
|
- name: linearvoid
|
|
rules:
|
|
- alert: ChatDown
|
|
expr: up{job="chat"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Chat service is down"
|
|
description: "Chat service has been unreachable for 2 minutes."
|
|
|
|
- alert: PlayoutDown
|
|
expr: up{job="playout"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Playout service is down"
|
|
description: "Playout service has been unreachable for 2 minutes."
|
|
|
|
- alert: RedisDown
|
|
expr: linearvoid_chat_redis_up == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Redis is unreachable"
|
|
description: "Chat service cannot reach Redis. Rate limiting and presence tracking degraded."
|
|
|
|
- alert: PlayoutSlateActive
|
|
expr: linearvoid_playout_mode == 1
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Playout in slate fallback mode"
|
|
description: "Playout has switched to emergency slate. Check playout logs."
|
|
|
|
- alert: HighChatLatency
|
|
expr: rate(linearvoid_chat_messages_total[5m]) < 0.01
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Unusually low chat message rate"
|
|
description: "Message rate below 0.01/s for 5 minutes. Check chat service health."
|
|
|
|
- alert: PrometheusWALFailing
|
|
expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Prometheus WAL writes failing"
|
|
description: "Prometheus TSDB WAL write failures detected."
|
|
|
|
- alert: HighCPU
|
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High CPU usage"
|
|
description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}."
|
|
|
|
- alert: LowDiskSpace
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Low disk space"
|
|
description: "Less than 10% disk space available on {{ $labels.instance }}."
|