diff --git a/.gitea/workflows/cd.yml b/.gitea/workflows/cd.yml index c5f7d78..194ab9b 100644 --- a/.gitea/workflows/cd.yml +++ b/.gitea/workflows/cd.yml @@ -57,14 +57,78 @@ jobs: scrape_interval: 15s evaluation_interval: 15s + rule_files: + - "/etc/prometheus/alerts.yml" + scrape_configs: + - job_name: 'chat' + static_configs: + - targets: ['chat:3000'] + - job_name: 'playout' + static_configs: + - targets: ['playout:8081'] + - job_name: 'nginx' + static_configs: + - targets: ['nginx-exporter:9113'] + - job_name: 'redis' + static_configs: + - targets: ['redis-exporter:9121'] + - job_name: 'postgres' + static_configs: + - targets: ['postgres-exporter:9187'] + - job_name: 'node' + static_configs: + - targets: ['node-exporter:9100'] - job_name: 'prometheus' static_configs: - targets: ['localhost:9090'] - - - job_name: 'linearvoid' - static_configs: - - targets: ['playout:8081', 'chat:3000', 'nginx:80'] + EOF + cat > monitoring/prometheus/alerts.yml << 'EOF' + groups: + - name: linearvoid + rules: + - alert: ChatDown + expr: up{job="chat"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Chat service is down" + - alert: PlayoutDown + expr: up{job="playout"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Playout service is down" + - alert: RedisDown + expr: linearvoid_chat_redis_up == 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Redis is unreachable" + - alert: PlayoutSlateActive + expr: linearvoid_playout_mode == 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Playout in slate fallback" + - alert: HighCPU + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU usage" + - alert: LowDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Low disk space" EOF - name: Deploy with docker-compose diff --git a/.github/MEMORY.md b/.github/MEMORY.md index 655565b..6145eb9 100644 --- a/.github/MEMORY.md +++ b/.github/MEMORY.md @@ -1,9 +1 @@ # MEMORY.md - -## 2026-08-05: Chat Message Admin + CI/CD + Release 1.0.5 - -**Completed**: Chat Message Admin, CI/CD, Release 1.0.5 committed + tagged + pushed. - -**Bug — nested admin page**: Fixed. Root cause: page HTML files were in `frontend/admin/` (Vite source root). Vite multi-page mode injected scripts/styles into every `.html` entry. `fetch("/admin/dashboard.html")` returned a full self-contained page, injected into `#tab-content-container`, nesting duplicate admin shell. Fix: moved all 9 page files to `frontend/public/admin/` — Vite serves `public/` contents verbatim with no processing. Fetch now returns raw HTML fragment. - -**Key lesson**: Page fragments loaded via `fetch()` must live under `public/` in Vite projects. Any `.html` in the project source root gets processed as a multi-page entry. diff --git a/backend/chat/src/main.go b/backend/chat/src/main.go index f088eb3..ab7791a 100644 --- a/backend/chat/src/main.go +++ b/backend/chat/src/main.go @@ -32,9 +32,9 @@ var ( upgrader = websocket.Upgrader{ CheckOrigin: func(r *http.Request) bool { return true }, } - clients = make(map[*websocket.Conn]bool) - clientMeta = make(map[*websocket.Conn]clientInfo) - mutex = sync.Mutex{} + clients = make(map[*websocket.Conn]bool) + clientMeta = make(map[*websocket.Conn]clientInfo) + mutex = sync.Mutex{} authSessions = map[string]time.Time{} authSessionsMu = sync.Mutex{} @@ -101,17 +101,17 @@ type Presence struct { } type AdminOverview struct { - Viewers int `json:"viewers"` - StoredMessages int64 `json:"storedMessages"` - ModerationActions int64 `json:"moderationActions"` - MediaCount int64 `json:"mediaCount"` + Viewers int `json:"viewers"` + StoredMessages int64 `json:"storedMessages"` + ModerationActions int64 `json:"moderationActions"` + MediaCount int64 `json:"mediaCount"` MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"` - MediaTotalRuntime string `json:"mediaTotalRuntime"` - RedisEnabled bool `json:"redisEnabled"` - RedisClients int `json:"redisClients"` - RedisMemoryUsed string `json:"redisMemoryUsed"` - DatabaseDriver string `json:"databaseDriver"` - GeneratedAt string `json:"generatedAt"` + MediaTotalRuntime string `json:"mediaTotalRuntime"` + RedisEnabled bool `json:"redisEnabled"` + RedisClients int `json:"redisClients"` + RedisMemoryUsed string `json:"redisMemoryUsed"` + DatabaseDriver string `json:"databaseDriver"` + GeneratedAt string `json:"generatedAt"` } type PlaylistPayload struct { @@ -136,15 +136,15 @@ type RoomAnalytics struct { } type AdminAnalytics struct { - TotalMessages int64 `json:"totalMessages"` - MessagesLast24Hours int64 `json:"messagesLast24Hours"` - DistinctRooms int64 `json:"distinctRooms"` - ModerationActions int64 `json:"moderationActions"` - MediaCount int64 `json:"mediaCount"` - MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"` - MediaTotalRuntime string `json:"mediaTotalRuntime"` - TopRooms []RoomAnalytics `json:"topRooms"` - GeneratedAt string `json:"generatedAt"` + TotalMessages int64 `json:"totalMessages"` + MessagesLast24Hours int64 `json:"messagesLast24Hours"` + DistinctRooms int64 `json:"distinctRooms"` + ModerationActions int64 `json:"moderationActions"` + MediaCount int64 `json:"mediaCount"` + MediaTotalDurationSeconds int64 `json:"mediaTotalDurationSeconds"` + MediaTotalRuntime string `json:"mediaTotalRuntime"` + TopRooms []RoomAnalytics `json:"topRooms"` + GeneratedAt string `json:"generatedAt"` } type ServiceHealth struct { @@ -171,9 +171,9 @@ type ActiveUser struct { } type AdminUsersResponse struct { - Users []ActiveUser `json:"users"` - RedisEnabled bool `json:"redisEnabled"` - GeneratedAt string `json:"generatedAt"` + Users []ActiveUser `json:"users"` + RedisEnabled bool `json:"redisEnabled"` + GeneratedAt string `json:"generatedAt"` } type ChatLogEntry struct { @@ -205,12 +205,12 @@ type ScheduleEntry struct { } type AdminScheduleResponse struct { - Entries []ScheduleEntry `json:"entries"` - TotalEntries int `json:"totalEntries"` - MatchedEntries int `json:"matchedEntries"` - TotalDurationSeconds int `json:"totalDurationSeconds"` - TotalRuntime string `json:"totalRuntime"` - GeneratedAt string `json:"generatedAt"` + Entries []ScheduleEntry `json:"entries"` + TotalEntries int `json:"totalEntries"` + MatchedEntries int `json:"matchedEntries"` + TotalDurationSeconds int `json:"totalDurationSeconds"` + TotalRuntime string `json:"totalRuntime"` + GeneratedAt string `json:"generatedAt"` } type MediaFileEntry struct { @@ -234,12 +234,12 @@ type AdminMediaResponse struct { } type AdminMediaUploadInitRequest struct { - UploadID string `json:"uploadId"` - FileName string `json:"fileName"` - FileSize int64 `json:"fileSize"` - MimeType string `json:"mimeType"` - ChunkSize int64 `json:"chunkSize"` - TotalChunks int `json:"totalChunks"` + UploadID string `json:"uploadId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + MimeType string `json:"mimeType"` + ChunkSize int64 `json:"chunkSize"` + TotalChunks int `json:"totalChunks"` } type AdminMediaUploadSession struct { @@ -254,13 +254,13 @@ type AdminMediaUploadSession struct { } type AdminMediaUploadStatusResponse struct { - UploadID string `json:"uploadId"` - FileName string `json:"fileName"` - FileSize int64 `json:"fileSize"` - ChunkSize int64 `json:"chunkSize"` - TotalChunks int `json:"totalChunks"` - UploadedChunks int `json:"uploadedChunks"` - Progress float64 `json:"progress"` + UploadID string `json:"uploadId"` + FileName string `json:"fileName"` + FileSize int64 `json:"fileSize"` + ChunkSize int64 `json:"chunkSize"` + TotalChunks int `json:"totalChunks"` + UploadedChunks int `json:"uploadedChunks"` + Progress float64 `json:"progress"` } type AdminLoginPayload struct { @@ -278,9 +278,9 @@ var databaseDriverName = "sqlite" const adminSessionCookieName = "linearvoid_admin_session" const ( - maxAdminUploadBytes int64 = 2 * 1024 * 1024 * 1024 - defaultAdminChunkSize int64 = 8 * 1024 * 1024 - maxAdminChunkSize int64 = 32 * 1024 * 1024 + maxAdminUploadBytes int64 = 2 * 1024 * 1024 * 1024 + defaultAdminChunkSize int64 = 8 * 1024 * 1024 + maxAdminChunkSize int64 = 32 * 1024 * 1024 ) // initRedis initializes Redis for presence, rate limiting, and pub/sub (ADR-007) @@ -380,11 +380,11 @@ func parseRedisInfoValue(info, key string) string { func getAdminOverview() AdminOverview { overview := AdminOverview{ - RedisEnabled: redisClient != nil, - RedisMemoryUsed: "unavailable", - DatabaseDriver: databaseDriverName, + RedisEnabled: redisClient != nil, + RedisMemoryUsed: "unavailable", + DatabaseDriver: databaseDriverName, MediaTotalRuntime: "00:00:00", - GeneratedAt: time.Now().UTC().Format(time.RFC3339), + GeneratedAt: time.Now().UTC().Format(time.RFC3339), } mutex.Lock() @@ -422,6 +422,23 @@ func handleAdminLogin(w http.ResponseWriter, r *http.Request) { return } + // Rate limit: max 5 attempts per IP per 5 minutes + clientIP := r.Header.Get("X-Forwarded-For") + if clientIP == "" { + clientIP = r.RemoteAddr + } + if redisClient != nil { + loginKey := fmt.Sprintf("login_attempts:%s", clientIP) + count, _ := redisClient.Incr(context.Background(), loginKey).Result() + if count == 1 { + redisClient.Expire(context.Background(), loginKey, 5*time.Minute) + } + if count > 5 { + http.Error(w, "too many login attempts, try again later", http.StatusTooManyRequests) + return + } + } + var payload AdminLoginPayload if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { http.Error(w, fmt.Sprintf("invalid request body: %v", err), http.StatusBadRequest) @@ -907,9 +924,9 @@ func handleAdminModerationAction(w http.ResponseWriter, r *http.Request) { func getAdminAnalytics() AdminAnalytics { analytics := AdminAnalytics{ - TopRooms: []RoomAnalytics{}, + TopRooms: []RoomAnalytics{}, MediaTotalRuntime: "00:00:00", - GeneratedAt: time.Now().UTC().Format(time.RFC3339), + GeneratedAt: time.Now().UTC().Format(time.RFC3339), } if db == nil { @@ -1114,7 +1131,7 @@ func handleAdminUsers(w http.ResponseWriter, r *http.Request) { case http.MethodPost: var payload struct { - Action string `json:"action"` // mute, unmute, ban, unban + Action string `json:"action"` // mute, unmute, ban, unban Username string `json:"username"` Duration int `json:"duration"` // minutes, for mute } @@ -1309,13 +1326,13 @@ func buildUploadStatus(session AdminMediaUploadSession) AdminMediaUploadStatusRe } return AdminMediaUploadStatusResponse{ - UploadID: session.UploadID, - FileName: session.FileName, - FileSize: session.FileSize, - ChunkSize: session.ChunkSize, - TotalChunks: session.TotalChunks, + UploadID: session.UploadID, + FileName: session.FileName, + FileSize: session.FileSize, + ChunkSize: session.ChunkSize, + TotalChunks: session.TotalChunks, UploadedChunks: uploadedChunks, - Progress: progress, + Progress: progress, } } @@ -2165,6 +2182,25 @@ func broadcast(msg Message) { // handleConnection handles WebSocket connections for chat (public rooms, private DMs, moderation) func handleConnection(w http.ResponseWriter, r *http.Request) { + // IP-based connection rate limiting: max 5 concurrent per IP + clientIP := r.Header.Get("X-Forwarded-For") + if clientIP == "" { + clientIP = r.RemoteAddr + } + if redisClient != nil { + ipKey := fmt.Sprintf("ws_conn:%s", clientIP) + count, _ := redisClient.Incr(context.Background(), ipKey).Result() + if count == 1 { + redisClient.Expire(context.Background(), ipKey, 60*time.Second) + } + if count > 5 { + http.Error(w, "too many connections from this IP", http.StatusTooManyRequests) + return + } + // Decrement on disconnect + defer redisClient.Decr(context.Background(), ipKey) + } + conn, err := upgrader.Upgrade(w, r, nil) if err != nil { log.Println("Upgrade error:", err) @@ -2649,4 +2685,4 @@ func main() { log.Printf("Chat Backend starting on :%s (WebSocket at /ws, health at /health, database persistence enabled)", port) log.Fatal(http.ListenAndServe(":"+port, nil)) -} \ No newline at end of file +} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index e9c0c06..fa97aa4 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -104,6 +104,56 @@ services: timeout: 5s retries: 5 + nginx-exporter: + image: nginx/nginx-prometheus-exporter:latest + container_name: linearvoid-nginx-exporter + restart: unless-stopped + command: + - "-nginx.scrape-uri=http://nginx/nginx-status" + networks: + - linearvoid-net + depends_on: + - nginx + + redis-exporter: + image: oliver006/redis_exporter:latest + container_name: linearvoid-redis-exporter + restart: unless-stopped + command: + - "-redis.addr=redis://redis:6379" + networks: + - linearvoid-net + depends_on: + - redis + + postgres-exporter: + image: prometheuscommunity/postgres-exporter:latest + container_name: linearvoid-postgres-exporter + restart: unless-stopped + environment: + DATA_SOURCE_NAME: "postgresql://linearvoid:${POSTGRES_PASSWORD:-linearvoid}@db:5432/linearvoid?sslmode=disable" + networks: + - linearvoid-net + depends_on: + - db + + node-exporter: + image: prom/node-exporter:latest + container_name: linearvoid-node-exporter + restart: unless-stopped + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + networks: + - linearvoid-net + prometheus: image: prom/prometheus:latest container_name: linearvoid-prometheus diff --git a/docker-compose.yml b/docker-compose.yml index bf4fea4..a407d3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,6 +97,56 @@ services: networks: - linearvoid-net + nginx-exporter: + image: nginx/nginx-prometheus-exporter:latest + container_name: linearvoid-nginx-exporter + restart: unless-stopped + command: + - "-nginx.scrape-uri=http://nginx/nginx-status" + networks: + - linearvoid-net + depends_on: + - nginx + + redis-exporter: + image: oliver006/redis_exporter:latest + container_name: linearvoid-redis-exporter + restart: unless-stopped + command: + - "-redis.addr=redis://redis:6379" + networks: + - linearvoid-net + depends_on: + - redis + + postgres-exporter: + image: prometheuscommunity/postgres-exporter:latest + container_name: linearvoid-postgres-exporter + restart: unless-stopped + environment: + DATA_SOURCE_NAME: "postgresql://linearvoid:${POSTGRES_PASSWORD:-linearvoid}@db:5432/linearvoid?sslmode=disable" + networks: + - linearvoid-net + depends_on: + - db + + node-exporter: + image: prom/node-exporter:latest + container_name: linearvoid-node-exporter + restart: unless-stopped + pid: host + volumes: + - /proc:/host/proc:ro + - /sys:/host/sys:ro + - /:/rootfs:ro + command: + - "--path.procfs=/host/proc" + - "--path.sysfs=/host/sys" + - "--path.rootfs=/rootfs" + - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)" + networks: + - linearvoid-net + prometheus: image: prom/prometheus:latest container_name: linearvoid-prometheus diff --git a/docs/incident-runbook.md b/docs/incident-runbook.md new file mode 100644 index 0000000..5cf7d81 --- /dev/null +++ b/docs/incident-runbook.md @@ -0,0 +1,170 @@ +# Incident Runbook + +Triage order for Linear Void incidents. Work top-to-bottom — each tier depends on the one above. + +## Triage Order + +``` +1. Edge Reachability (Cloudflare) +2. Origin Health (VPS/Docker) +3. Playout Progression (HLS) +4. Chat Connectivity (WebSocket) +5. Data-Plane (Redis/DB) +``` + +--- + +## 1. Edge Reachability (Cloudflare) + +**Symptoms:** `linearvoid.tv` not resolving, SSL errors, 5xx from Cloudflare. + +```bash +# Check DNS +dig linearvoid.tv +dig www.linearvoid.tv + +# Check Cloudflare status +curl -I https://linearvoid.tv/health +``` + +**Common causes:** +- DNS misconfiguration (A record pointing to wrong IP) +- Cloudflare proxy disabled (grey cloud → origin exposed directly) +- SSL certificate expired (check Cloudflare dashboard → SSL/TLS) +- Cloudflare outage → https://www.cloudflarestatus.com + +**Mitigation:** +- Verify A record: `linearvoid.tv` → origin IP (proxied, orange cloud) +- Bypass Cloudflare temporarily: `curl -H "Host: linearvoid.tv" http:///health` + +--- + +## 2. Origin Health (VPS/Docker) + +**Symptoms:** Cloudflare returns 502/504, containers down. + +```bash +# Remote Docker (test lab) +docker -H tcp://192.168.88.40:2375 ps +docker -H tcp://192.168.88.40:2375 logs linearvoid-nginx --tail 50 + +# Direct health check +curl http://192.168.88.40/health +``` + +**Common causes:** +- Host out of disk space: `df -h` +- Docker daemon stopped: `systemctl status docker` +- Container restart loop: `docker ps -a --filter "status=exited"` +- Resource exhaustion (OOM): `dmesg | grep -i oom` + +**Mitigation:** +- Restart stack: `docker compose -f docker-compose.prod.yml up -d` +- Free disk: `docker system prune -f` +- Check logs per service: `docker logs linearvoid- --tail 100` + +--- + +## 3. Playout Progression (HLS) + +**Symptoms:** Viewer sees slate or buffering, playlist not advancing. + +```bash +# Check playout health +curl http://192.168.88.40:8081/health +# Response includes "mode": "live" or "slate" + +# Check HLS output +curl -I http://192.168.88.40/hls/playlist.m3u8 +ls -la data/hls/ + +# Check playout logs +docker logs linearvoid-playout --tail 100 +``` + +**Common causes:** +- FFmpeg crash → slate fallback active (health shows `degraded`) +- Playlist file empty/missing: `cat data/playlist/playlist.txt` +- Media file missing from playlist: `ls data/media/` +- Disk full → HLS segments can't write + +**Mitigation:** +- Verify playlist has valid entries +- Check media files exist at paths in playlist +- Restart playout: `docker compose restart playout` +- Generate demo playlist if empty: see `backend/playout/start-playout.sh` + +--- + +## 4. Chat Connectivity (WebSocket) + +**Symptoms:** Chat not loading, messages not sending, presence broken. + +```bash +# Check chat health +curl http://192.168.88.40:3000/health + +# Test WebSocket +wscat -c ws://192.168.88.40/ws?username=test&room=main + +# Check chat logs +docker logs linearvoid-chat --tail 100 +``` + +**Common causes:** +- Redis down → presence/rate-limiting degraded (chat still works) +- Database down → messages not persisting +- Port not exposed: `docker port linearvoid-chat` + +**Mitigation:** +- Restart chat: `docker compose restart chat` +- Check Redis: `docker exec linearvoid-redis redis-cli ping` +- Check DB: `docker exec linearvoid-db pg_isready -U linearvoid` +- Chat works without Redis (graceful degradation) — check logs for "Redis connection warning" + +--- + +## 5. Data-Plane (Redis/DB) + +**Symptoms:** Chat works but no persistence/presence, admin stats empty. + +```bash +# Redis +docker exec linearvoid-redis redis-cli ping +docker exec linearvoid-redis redis-cli info memory + +# PostgreSQL +docker exec linearvoid-db pg_isready -U linearvoid +docker exec linearvoid-db psql -U linearvoid -c "SELECT count(*) FROM messages;" +``` + +**Common causes:** +- Redis OOM (check `maxmemory` setting) +- PostgreSQL connection limit reached +- Disk full on volume mount + +**Mitigation:** +- Redis: `docker compose restart redis` (AOF persist enabled) +- PostgreSQL: `docker compose restart db` +- Restore from backup: see `scripts/verify-backups.sh` for procedure + +--- + +## Escalation + +| Severity | When | Action | +|---|---|---| +| Critical | Slate active >30min, complete outage | Trigger DR plan, provision fallback host | +| Warning | Redis down, high CPU >30min | Investigate during business hours, restart service | +| Info | Single service blip | Monitor, no action unless recurring | + +## Post-Incident + +1. Document timeline in incident channel +2. Identify root cause +3. Add prevention to TODO.md if actionable +4. Update this runbook if new failure mode discovered + +--- + +**Recovery targets:** RPO ≤24h, RTO ≤60min. Quarterly DR drill: simulate host loss, restore from backup. diff --git a/monitoring/prometheus/alerts.yml b/monitoring/prometheus/alerts.yml new file mode 100644 index 0000000..500c217 --- /dev/null +++ b/monitoring/prometheus/alerts.yml @@ -0,0 +1,74 @@ +groups: + - name: linearvoid + rules: + - alert: ChatDown + expr: up{job="chat"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Chat service is down" + description: "Chat service has been unreachable for 2 minutes." + + - alert: PlayoutDown + expr: up{job="playout"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Playout service is down" + description: "Playout service has been unreachable for 2 minutes." + + - alert: RedisDown + expr: linearvoid_chat_redis_up == 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Redis is unreachable" + description: "Chat service cannot reach Redis. Rate limiting and presence tracking degraded." + + - alert: PlayoutSlateActive + expr: linearvoid_playout_mode == 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Playout in slate fallback mode" + description: "Playout has switched to emergency slate. Check playout logs." + + - alert: HighChatLatency + expr: rate(linearvoid_chat_messages_total[5m]) < 0.01 + for: 5m + labels: + severity: warning + annotations: + summary: "Unusually low chat message rate" + description: "Message rate below 0.01/s for 5 minutes. Check chat service health." + + - alert: PrometheusWALFailing + expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Prometheus WAL writes failing" + description: "Prometheus TSDB WAL write failures detected." + + - alert: HighCPU + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU usage" + description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}." + + - alert: LowDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Low disk space" + description: "Less than 10% disk space available on {{ $labels.instance }}." diff --git a/monitoring/prometheus/prometheus.yml b/monitoring/prometheus/prometheus.yml index 5da67f2..a5dd161 100644 --- a/monitoring/prometheus/prometheus.yml +++ b/monitoring/prometheus/prometheus.yml @@ -1,10 +1,28 @@ global: scrape_interval: 15s +rule_files: + - "/etc/prometheus/alerts.yml" + scrape_configs: - job_name: "chat" static_configs: - targets: ["chat:3000"] + - job_name: "playout" + static_configs: + - targets: ["playout:8081"] + - job_name: "nginx" + static_configs: + - targets: ["nginx-exporter:9113"] + - job_name: "redis" + static_configs: + - targets: ["redis-exporter:9121"] + - job_name: "postgres" + static_configs: + - targets: ["postgres-exporter:9187"] + - job_name: "node" + static_configs: + - targets: ["node-exporter:9100"] - job_name: "prometheus" static_configs: - targets: ["prometheus:9090"] diff --git a/scripts/backup-db.sh b/scripts/backup-db.sh new file mode 100644 index 0000000..acfe768 --- /dev/null +++ b/scripts/backup-db.sh @@ -0,0 +1,45 @@ +#!/bin/bash +# Daily PostgreSQL backup to R2 (14-day retention) +# Schedule: cron daily. Requires: pg_dump, aws CLI or rclone for R2. +set -e + +BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}" +RETENTION_DAYS="${RETENTION_DAYS:-14}" +R2_BUCKET="${R2_BUCKET:-linearvoid-backups}" +DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}" + +TIMESTAMP=$(date +%Y%m%d) +BACKUP_FILE="$BACKUP_DIR/daily/db_${TIMESTAMP}.dump" + +mkdir -p "$(dirname "$BACKUP_FILE")" + +echo "[$(date -Iseconds)] Starting daily backup..." + +# Extract connection details from DATABASE_URL +DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p') +DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p') +DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p') +DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p') +DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p') + +export PGPASSWORD="${DB_PASS}" + +# Dump +pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \ + -Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}" + +echo " Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))" + +# Upload to R2 +if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then + aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/daily/db_${TIMESTAMP}.dump" \ + --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \ + echo " Uploaded to R2: daily/db_${TIMESTAMP}.dump" || \ + echo " WARNING: R2 upload failed" +fi + +# Cleanup old backups +find "$BACKUP_DIR/daily" -name "db_*.dump" -mtime "+${RETENTION_DAYS}" -delete 2>/dev/null || true +echo " Cleaned up backups older than ${RETENTION_DAYS} days" + +echo "[$(date -Iseconds)] Daily backup complete." diff --git a/scripts/backup-weekly.sh b/scripts/backup-weekly.sh new file mode 100644 index 0000000..6adcafd --- /dev/null +++ b/scripts/backup-weekly.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# Weekly full PostgreSQL backup to R2 (8-week retention) +# Schedule: cron weekly (runs backup-db.sh with weekly prefix) +set -e + +BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}" +RETENTION_WEEKS="${RETENTION_WEEKS:-8}" +R2_BUCKET="${R2_BUCKET:-linearvoid-backups}" +DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}" + +TIMESTAMP=$(date +%Y%m%d) +BACKUP_FILE="$BACKUP_DIR/weekly/db_${TIMESTAMP}.dump" + +mkdir -p "$(dirname "$BACKUP_FILE")" + +echo "[$(date -Iseconds)] Starting weekly full backup..." + +DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p') +DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p') +DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p') +DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p') +DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p') + +export PGPASSWORD="${DB_PASS}" + +pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \ + -Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}" + +echo " Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))" + +# Verify integrity +pg_restore --list "$BACKUP_FILE" > /dev/null 2>&1 && \ + echo " Integrity check: PASSED" || \ + echo " WARNING: Integrity check FAILED" + +# Upload +if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then + aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/weekly/db_${TIMESTAMP}.dump" \ + --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \ + echo " Uploaded to R2: weekly/db_${TIMESTAMP}.dump" || \ + echo " WARNING: R2 upload failed" +fi + +# Cleanup +find "$BACKUP_DIR/weekly" -name "db_*.dump" -mtime "+$((RETENTION_WEEKS * 7))" -delete 2>/dev/null || true +echo " Cleaned up backups older than ${RETENTION_WEEKS} weeks" + +echo "[$(date -Iseconds)] Weekly backup complete." diff --git a/scripts/verify-backups.sh b/scripts/verify-backups.sh new file mode 100644 index 0000000..aee94ca --- /dev/null +++ b/scripts/verify-backups.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Weekly backup integrity verification. +# Downloads latest backup from R2, restores into temp container, runs sanity checks. +# Schedule: cron weekly (after weekly backup). +set -e + +R2_BUCKET="${R2_BUCKET:-linearvoid-backups}" +TEMP_DIR=$(mktemp -d) +trap "rm -rf $TEMP_DIR" EXIT + +echo "[$(date -Iseconds)] Starting backup integrity verification..." + +# Download latest weekly backup from R2 +if [ -z "$R2_ENDPOINT" ] || [ -z "$R2_ACCESS_KEY_ID" ]; then + echo " SKIP: R2 credentials not configured (R2_ENDPOINT, R2_ACCESS_KEY_ID)" + exit 0 +fi + +LATEST=$(aws s3 ls "s3://${R2_BUCKET}/weekly/" --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null | sort | tail -1 | awk '{print $4}') +if [ -z "$LATEST" ]; then + echo " SKIP: No weekly backups found in R2" + exit 0 +fi + +echo " Downloading: $LATEST" +aws s3 cp "s3://${R2_BUCKET}/weekly/${LATEST}" "$TEMP_DIR/backup.dump" \ + --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null + +echo " File size: $(du -h "$TEMP_DIR/backup.dump" | cut -f1)" + +# Restore into temp PostgreSQL container +echo " Starting temp PostgreSQL..." +docker run -d --name lv-backup-test-$$ \ + -e POSTGRES_PASSWORD=test \ + -e POSTGRES_DB=linearvoid \ + -p 15432:5432 \ + postgres:15-alpine 2>/dev/null + +sleep 5 + +# Wait for PG to be ready +for i in $(seq 1 10); do + if docker exec lv-backup-test-$$ pg_isready -U postgres 2>/dev/null; then + break + fi + sleep 2 +done + +# Restore +echo " Restoring backup..." +docker cp "$TEMP_DIR/backup.dump" "lv-backup-test-$$:/tmp/backup.dump" +docker exec lv-backup-test-$$ pg_restore -U postgres -d linearvoid /tmp/backup.dump 2>/dev/null && \ + echo " Restore: OK" || { echo " Restore: FAILED"; docker rm -f lv-backup-test-$$ 2>/dev/null; exit 1; } + +# Sanity checks +MSG_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM messages;" 2>/dev/null || echo "0") +MOD_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM moderation_logs;" 2>/dev/null || echo "0") +echo " Messages: $MSG_COUNT, Moderation: $MOD_COUNT" + +# Cleanup +docker rm -f lv-backup-test-$$ 2>/dev/null + +echo "[$(date -Iseconds)] Backup verification complete. OK."