From a8df164cc22f616a01b7703f9ae83ddd1a90f234 Mon Sep 17 00:00:00 2001 From: zwitschi Date: Fri, 7 Aug 2026 09:01:12 +0200 Subject: [PATCH] fix: seed correct monitoring volumes and verify monitoring deployment --- .gitea/workflows/cd.yml | 76 ++++++++++++++++- SETUP.md | 180 ++++++++++++++++++++++++++++++++-------- docker-compose.prod.yml | 4 +- 3 files changed, 224 insertions(+), 36 deletions(-) diff --git a/.gitea/workflows/cd.yml b/.gitea/workflows/cd.yml index 65aba62..6629f46 100644 --- a/.gitea/workflows/cd.yml +++ b/.gitea/workflows/cd.yml @@ -48,7 +48,7 @@ jobs: DOCKER_HOST: tcp://192.168.88.40:2375 run: | docker run --rm \ - -v prometheus-config:/etc/prometheus \ + -v linearvoid_prometheus-config:/etc/prometheus \ alpine:latest sh -c ' cat > /etc/prometheus/prometheus.yml << "PROMEOF" global: @@ -113,6 +113,20 @@ jobs: severity: warning annotations: summary: "Playout in slate fallback" + - alert: HighChatLatency + expr: rate(linearvoid_chat_messages_total[5m]) < 0.01 + for: 5m + labels: + severity: warning + annotations: + summary: "Unusually low chat message rate" + - alert: PrometheusWALFailing + expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Prometheus WAL writes failing" - alert: HighCPU expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 for: 10m @@ -130,6 +144,66 @@ jobs: ALERTSEOF ' + - name: Seed Grafana provisioning on remote host + env: + DOCKER_HOST: tcp://192.168.88.40:2375 + run: | + docker run --rm \ + -v linearvoid_grafana-provisioning:/etc/grafana/provisioning \ + alpine:latest sh -c ' + mkdir -p /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards + cat > /etc/grafana/provisioning/datasources/datasource.yml << "DSEOF" + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + DSEOF + cat > /etc/grafana/provisioning/dashboards/dashboard.yml << "PROVEOF" + apiVersion: 1 + providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards + PROVEOF + cat > /etc/grafana/provisioning/dashboards/linearvoid-app.json << "DBEOF" + { + "title": "Linear Void Monitoring", + "panels": [ + { + "title": "Concurrent Viewers", + "type": "timeseries", + "targets": [{ "expr": "linearvoid_chat_connected_clients" }] + }, + { + "title": "Stored Messages", + "type": "stat", + "targets": [{ "expr": "linearvoid_chat_messages_total" }] + }, + { + "title": "Redis Reachable", + "type": "stat", + "targets": [{ "expr": "linearvoid_chat_redis_up" }] + }, + { + "title": "Playout Mode", + "type": "stat", + "targets": [{ "expr": "linearvoid_playout_mode" }] + } + ] + } + DBEOF + ' + - name: Deploy with docker-compose env: DOCKER_HOST: tcp://192.168.88.40:2375 diff --git a/SETUP.md b/SETUP.md index f0c3c80..b4c31da 100644 --- a/SETUP.md +++ b/SETUP.md @@ -162,39 +162,6 @@ flowchart LR - Place normalized MP4 files in `data/media/`. - Create a `data/playlist/playlist.txt` with paths (see ADR-001 for normalization requirements). -## Cloudflare R2 Setup - -1. **Create an R2 bucket**: - - Create a bucket such as `linearvoid-media` in Cloudflare R2. - - Keep the bucket name aligned with `R2_BUCKET_NAME` in `.env`. - -2. **Generate R2 API credentials**: - - Create an access key pair with read/write permissions for the bucket. - - Set `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, and `R2_REGION=auto` in `.env`. - -3. **Find the S3 endpoint**: - - Use the account-scoped endpoint format `https://.r2.cloudflarestorage.com`. - - Store the account ID in both `R2_ENDPOINT` and `CLOUDFLARE_ACCOUNT_ID` where applicable. - -4. **Bind a custom media domain**: - - Attach a domain such as `media.linearvoid.tv` to the bucket. - - Set `MEDIA_DOMAIN` to that hostname. - -5. **Configure app-domain DNS and TLS**: - - Point `linearvoid.tv` to the public entrypoint serving nginx. - - Point `www.linearvoid.tv` to the same target and redirect it to `https://linearvoid.tv`. - - Keep `media.linearvoid.tv` on the R2 custom domain path so HLS delivery stays split from the app origin. - - Use Cloudflare proxied DNS with Full (strict) TLS so browser traffic uses `https://linearvoid.tv`, `wss://linearvoid.tv/ws`, and `https://media.linearvoid.tv`. - -6. **Apply cache behavior**: - - Configure the `.ts` and `.m3u8` cache rules from `infrastructure/cloudflare/cache-rules.md`. - - Enable Smart Tiered Cache for the R2-backed hostname. - - If you need header normalization for immutable segment caching, deploy `infrastructure/cloudflare/worker.js` to the same zone. - -7. **Verify access**: - - Confirm uploads with an S3-compatible client such as `aws s3 ls --endpoint-url `. - - After deployment, check Cloudflare analytics for cache hit ratio and origin request volume. - ## Gitea Setup 1. **Create or verify a Gitea user**: @@ -231,6 +198,153 @@ flowchart LR 7. **Validate image availability**: - Before a production rollout, verify that the expected `:latest` tags can be pulled from the registry on the target server. +## Cloudflare R2 Setup + +1. **Create an R2 bucket**: + - Create a bucket such as `linearvoid-media` in Cloudflare R2. + - Keep the bucket name aligned with `R2_BUCKET_NAME` in `.env`. + +2. **Generate R2 API credentials**: + - Create an access key pair with read/write permissions for the bucket. + - Set `R2_ACCESS_KEY_ID`, `R2_SECRET_ACCESS_KEY`, and `R2_REGION=auto` in `.env`. + +3. **Find the S3 endpoint**: + - Use the account-scoped endpoint format `https://.r2.cloudflarestorage.com`. + - Store the account ID in both `R2_ENDPOINT` and `CLOUDFLARE_ACCOUNT_ID` where applicable. + +4. **Bind a custom media domain**: + - Attach a domain such as `media.linearvoid.tv` to the bucket. + - Set `MEDIA_DOMAIN` to that hostname. + +5. **Configure app-domain DNS and TLS**: + - Point `linearvoid.tv` to the public entrypoint serving nginx. + - Point `www.linearvoid.tv` to the same target and redirect it to `https://linearvoid.tv`. + - Keep `media.linearvoid.tv` on the R2 custom domain path so HLS delivery stays split from the app origin. + - Use Cloudflare proxied DNS with Full (strict) TLS so browser traffic uses `https://linearvoid.tv`, `wss://linearvoid.tv/ws`, and `https://media.linearvoid.tv`. + +6. **Apply cache behavior**: + - Configure the `.ts` and `.m3u8` cache rules from `infrastructure/cloudflare/cache-rules.md`. + - Enable Smart Tiered Cache for the R2-backed hostname. + - If you need header normalization for immutable segment caching, deploy `infrastructure/cloudflare/worker.js` to the same zone. + +7. **Verify access**: + - Confirm uploads with an S3-compatible client such as `aws s3 ls --endpoint-url `. + - After deployment, check Cloudflare analytics for cache hit ratio and origin request volume. + +## Cloudflare + +Use this section for full edge setup (DNS, Pages, cache, security, and monitoring). + +### 1. DNS Setup + +1. Add apex record for `linearvoid.tv` pointing to Cloudflare Pages target. +2. Add `www` CNAME to Cloudflare Pages and configure redirect to `https://linearvoid.tv`. +3. Add `media.linearvoid.tv` for split-domain HLS delivery. +4. Keep all records proxied (orange cloud) unless troubleshooting requires DNS-only mode. + +### 2. R2 Bucket + +1. Create bucket `linearvoid-media`. +2. Set location hint to `enam` (US-East). +3. Attach custom domain `media.linearvoid.tv` to the bucket. +4. Verify object read works from the custom domain. + +### 3. Cache Rules + +Configure Cloudflare Cache Rules: + +1. HLS segment files (`*.ts`, `*.m4s`): + - Cache: eligible + - Edge TTL: `31536000` + - Browser TTL: long-lived + - Cache key: include full path +2. HLS manifests (`*.m3u8`): + - Cache: eligible + - Edge TTL: `2s` + - Revalidation: enabled +3. Validate behavior using response headers: + - `Cf-Cache-Status` should move from `MISS` to `HIT` for hot objects + +Reference: `infrastructure/cloudflare/cache-rules.md`. + +### 4. Cloudflare Pages + +1. Connect Gitea repository to Cloudflare Pages. +2. Build command: + + ```bash + cd frontend && npm run build + ``` + +3. Output directory: + + ```text + frontend/dist + ``` + +4. Configure production domain to `linearvoid.tv`. + +### 5. Tiered Cache + +1. Enable Smart Tiered Cache in Cloudflare dashboard. +2. Warm a segment and manifest path. +3. Confirm tiered behavior through repeated requests and `Cf-Cache-Status: HIT`. + +### 6. WAF and Bot Protection + +1. Enable OWASP Managed Ruleset. +2. Start in `Simulate` mode. +3. Review events for 48 hours. +4. Switch to `Block` once false positives are addressed. +5. Enable Bot Fight Mode. + +### 7. Rate Limiting + +Create rules: + +1. Path: `/api/admin/login` +2. Path: `/api/admin/*` +3. Threshold: `10 requests / 10 seconds / IP` +4. Action: `Managed Challenge` + +### 8. HTTPS and WSS + +1. Enable `Always Use HTTPS`. +2. Enable HSTS with: + - `max-age=31536000` + - `includeSubDomains` + - `preload` +3. Minimum TLS version: `1.2`. +4. Keep SSL mode at `Full (strict)` when origin certs are valid. +5. Enforce HTTPS redirect at both layers: + - Cloudflare edge + - nginx origin +6. Validate websocket endpoint uses `wss://linearvoid.tv/ws`. + +### 9. UptimeRobot Monitors + +Create external monitors: + +1. HTTP monitor: + - URL: `https://linearvoid.tv/health` + - Interval: 5 minutes +2. Keyword monitor: + - URL: `https://linearvoid.tv/hls/playlist.m3u8` + - Keyword: `#EXTM3U` + - Interval: 5 minutes +3. Alert policy: + - Trigger on 2 consecutive failures + - Notify via email and Discord/Slack webhook + +### 10. Verification Checklist + +1. DNS resolves for `linearvoid.tv`, `www.linearvoid.tv`, and `media.linearvoid.tv`. +2. Pages serves frontend for apex and `www` redirects correctly. +3. HLS segments are cacheable long-term, manifests refresh every ~2s. +4. Admin paths are rate-limited and WAF events are visible. +5. HTTPS and WSS both succeed with valid certificates. +6. UptimeRobot alerts fire as expected when endpoints are intentionally made unavailable. + ## Running the Application ### Local Development diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index f6d15f8..8cdb40a 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -180,8 +180,7 @@ services: - "3001:3000" volumes: - grafana-data:/var/lib/grafana - - ./monitoring/grafana/dashboards:/etc/grafana/provisioning/dashboards - - ./monitoring/grafana/datasources:/etc/grafana/provisioning/datasources + - grafana-provisioning:/etc/grafana/provisioning environment: - GF_SECURITY_ADMIN_USER=admin - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin} @@ -201,3 +200,4 @@ volumes: prometheus-config: prometheus-data: grafana-data: + grafana-provisioning: