93 lines
3.1 KiB
YAML
93 lines
3.1 KiB
YAML
groups:
|
|
- name: linearvoid
|
|
rules:
|
|
- alert: ChatDown
|
|
expr: up{job="chat"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Chat service is down"
|
|
description: "Chat service has been unreachable for 2 minutes."
|
|
|
|
- alert: PlayoutDown
|
|
expr: up{job="playout"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Playout service is down"
|
|
description: "Playout service has been unreachable for 2 minutes."
|
|
|
|
- alert: RedisDown
|
|
expr: linearvoid_chat_redis_up == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Redis is unreachable"
|
|
description: "Chat service cannot reach Redis. Rate limiting and presence tracking degraded."
|
|
|
|
- alert: PlayoutSlateActive
|
|
expr: linearvoid_playout_mode == 1
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Playout in slate fallback mode"
|
|
description: "Playout has switched to emergency slate. Check playout logs."
|
|
|
|
- alert: HighChatLatency
|
|
expr: histogram_quantile(0.95, sum(rate(linearvoid_chat_request_duration_seconds_bucket[5m])) by (le)) > 1
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Chat request latency is high"
|
|
description: "Chat request p95 has been above 1s for 5 minutes. Check chat handlers and dependencies."
|
|
|
|
- alert: PrometheusWALFailing
|
|
expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Prometheus WAL writes failing"
|
|
description: "Prometheus TSDB WAL write failures detected."
|
|
|
|
- alert: HighCPU
|
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High CPU usage"
|
|
description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}."
|
|
|
|
- alert: LowCacheHitRatio
|
|
expr: linearvoid_cache_hit_ratio < 0.9
|
|
for: 15m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Cloudflare cache hit ratio is below target"
|
|
description: "Cache hit ratio has been below 90% for 15 minutes on {{ $labels.hostname }}."
|
|
|
|
- alert: HighRedisMemoryUsage
|
|
expr: linearvoid_chat_redis_memory_usage_ratio > 0.85
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Redis memory usage is above target"
|
|
description: "Redis memory usage has been above 85% for 10 minutes on {{ $labels.instance }}."
|
|
|
|
- alert: LowDiskSpace
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Low disk space"
|
|
description: "Less than 10% disk space available on {{ $labels.instance }}."
|