From 5f1f796a2072f483b2c490d08e447620c8e97f75 Mon Sep 17 00:00:00 2001 From: Zwitschi Date: Wed, 2 Sep 2026 21:07:06 +0200 Subject: [PATCH] chore(cd): publish nginx on port 8082 to avoid Coolify port conflicts --- .gitea/workflows/cd.yml | 577 ++++++++++++++++++++-------------------- 1 file changed, 289 insertions(+), 288 deletions(-) diff --git a/.gitea/workflows/cd.yml b/.gitea/workflows/cd.yml index 6bedb6e..eba9b41 100644 --- a/.gitea/workflows/cd.yml +++ b/.gitea/workflows/cd.yml @@ -1,288 +1,289 @@ -# CD workflow — TEST LAB DEPLOYMENT -# Deploys to test lab host 192.168.88.40:2375. -# Production Hetzner target TBD — switch DOCKER_HOST when provisioned. -# See DEPLOYMENT.md Production Deployment section for cutover plan. - -name: CD - -on: - workflow_run: - workflows: [CI] - types: [completed] - workflow_dispatch: - -jobs: - deploy: - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} - runs-on: [ubuntu-latest] - steps: - - name: Checkout code - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to registry on remote host - env: - DOCKER_HOST: tcp://192.168.88.40:2375 - run: | - echo "${{ secrets.CI_REGISTRY_TOKEN }}" | docker login git.allucanget.biz -u "${{ secrets.CI_REGISTRY_USER }}" --password-stdin - - - name: Prepare environment - run: | - cat > .env << 'EOF' - # Database & Redis - DATABASE_URL="${{ secrets.DATABASE_URL }}" - REDIS_URL=redis://redis:6379/0 - - # Playout - PLAYLIST_PATH=/app/playlist/playlist.txt - HLS_SEGMENT_DURATION=2 - HLS_WINDOW_SIZE=12 - LOG_LEVEL=info - - # Cloudflare R2 (required for HLS segment sync) - R2_ACCESS_KEY_ID="${{ secrets.R2_ACCESS_KEY_ID }}" - R2_SECRET_ACCESS_KEY="${{ secrets.R2_SECRET_ACCESS_KEY }}" - R2_BUCKET_NAME="${{ secrets.R2_BUCKET_NAME }}" - R2_ENDPOINT="${{ secrets.R2_ENDPOINT }}" - R2_REGION=auto - - # Cloudflare - CLOUDFLARE_API_TOKEN="${{ secrets.CLOUDFLARE_API_TOKEN }}" - CLOUDFLARE_ACCOUNT_ID="${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" - MEDIA_DOMAIN=${{ secrets.MEDIA_DOMAIN }} - - # Frontend public URLs - VITE_PUBLIC_APP_URL=${{ secrets.VITE_PUBLIC_APP_URL }} - VITE_PUBLIC_WS_URL=${{ secrets.VITE_PUBLIC_WS_URL }} - VITE_PUBLIC_HLS_URL=${{ secrets.VITE_PUBLIC_HLS_URL }} - VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR=${{ secrets.VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR }} - VITE_PUBLIC_ADMIN_DEFAULT_ROOM=${{ secrets.VITE_PUBLIC_ADMIN_DEFAULT_ROOM }} - VITE_PUBLIC_ADMIN_MODERATION_LOG_LIMIT=50 - - # Admin auth - ADMIN_USERNAME="${{ secrets.ADMIN_USERNAME }}" - ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" - ADMIN_SESSION_TTL_HOURS=12 - - # Nginx - NGINX_PORT=80 - EOF - - - name: Seed Prometheus config on remote host - env: - DOCKER_HOST: tcp://192.168.88.40:2375 - run: | - docker run --rm \ - -v linearvoid_prometheus-config:/etc/prometheus \ - alpine:latest sh -c ' - cat > /etc/prometheus/prometheus.yml << "PROMEOF" - global: - scrape_interval: 15s - evaluation_interval: 15s - - rule_files: - - "/etc/prometheus/alerts.yml" - - scrape_configs: - - job_name: "chat" - static_configs: - - targets: ["chat:3000"] - - job_name: "playout" - static_configs: - - targets: ["playout:8081"] - - job_name: "nginx" - static_configs: - - targets: ["nginx-exporter:9113"] - - job_name: "redis" - static_configs: - - targets: ["redis-exporter:9121"] - - job_name: "postgres" - static_configs: - - targets: ["postgres-exporter:9187"] - - job_name: "node" - static_configs: - - targets: ["node-exporter:9100"] - - job_name: "prometheus" - static_configs: - - targets: ["localhost:9090"] - PROMEOF - cat > /etc/prometheus/alerts.yml << "ALERTSEOF" - groups: - - name: linearvoid - rules: - - alert: ChatDown - expr: up{job="chat"} == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "Chat service is down" - - alert: PlayoutDown - expr: up{job="playout"} == 0 - for: 2m - labels: - severity: critical - annotations: - summary: "Playout service is down" - - alert: RedisDown - expr: linearvoid_chat_redis_up == 0 - for: 2m - labels: - severity: warning - annotations: - summary: "Redis is unreachable" - - alert: PlayoutSlateActive - expr: linearvoid_playout_mode == 1 - for: 1m - labels: - severity: warning - annotations: - summary: "Playout in slate fallback" - - alert: HighChatLatency - expr: rate(linearvoid_chat_messages_total[5m]) < 0.01 - for: 5m - labels: - severity: warning - annotations: - summary: "Unusually low chat message rate" - - alert: PrometheusWALFailing - expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0 - for: 5m - labels: - severity: warning - annotations: - summary: "Prometheus WAL writes failing" - - alert: HighCPU - expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 - for: 10m - labels: - severity: warning - annotations: - summary: "High CPU usage" - - alert: LowDiskSpace - expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10 - for: 5m - labels: - severity: warning - annotations: - summary: "Low disk space" - ALERTSEOF - ' - - - name: Seed Grafana provisioning on remote host - env: - DOCKER_HOST: tcp://192.168.88.40:2375 - run: | - docker run --rm \ - -v linearvoid_grafana-provisioning:/etc/grafana/provisioning \ - alpine:latest sh -c ' - mkdir -p /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards - cat > /etc/grafana/provisioning/datasources/datasource.yml << "DSEOF" - apiVersion: 1 - datasources: - - name: Prometheus - type: prometheus - access: proxy - url: http://prometheus:9090 - isDefault: true - editable: true - DSEOF - cat > /etc/grafana/provisioning/dashboards/dashboard.yml << "PROVEOF" - apiVersion: 1 - providers: - - name: "default" - orgId: 1 - folder: "" - type: file - disableDeletion: false - updateIntervalSeconds: 10 - allowUiUpdates: true - options: - path: /etc/grafana/provisioning/dashboards - PROVEOF - cat > /etc/grafana/provisioning/dashboards/linearvoid-app.json << "DBEOF" - { - "title": "Linear Void Monitoring", - "panels": [ - { - "title": "Concurrent Viewers", - "type": "timeseries", - "targets": [{ "expr": "linearvoid_chat_connected_clients" }] - }, - { - "title": "Stored Messages", - "type": "stat", - "targets": [{ "expr": "linearvoid_chat_messages_total" }] - }, - { - "title": "Redis Reachable", - "type": "stat", - "targets": [{ "expr": "linearvoid_chat_redis_up" }] - }, - { - "title": "Playout Mode", - "type": "stat", - "targets": [{ "expr": "linearvoid_playout_mode" }] - } - ] - } - DBEOF - ' - - - name: Deploy with docker-compose - env: - DOCKER_HOST: tcp://192.168.88.40:2375 - run: | - docker compose -f docker-compose.prod.yml pull - docker compose -f docker-compose.prod.yml up -d --remove-orphans --force-recreate - - - name: Health check - env: - DOCKER_HOST: tcp://192.168.88.40:2375 - run: | - # Wait for services to stabilize - sleep 30 - - # Check chat health (via docker exec — IP-agnostic, works on any DOCKER_HOST) - echo "Checking chat health..." - if ! docker exec linearvoid-chat curl -sf http://localhost:3000/health; then - echo "Chat health check failed" - docker logs linearvoid-chat --tail 50 - exit 1 - fi - - # Check nginx health - echo "Checking nginx health..." - if ! docker exec linearvoid-nginx curl -sf http://localhost/health; then - echo "Nginx health check failed" - docker logs linearvoid-nginx --tail 50 - exit 1 - fi - - # Retry playout health check since it may still be initializing - echo "Checking playout health..." - for i in $(seq 1 6); do - if docker exec linearvoid-playout curl -sf http://localhost:8081/health; then - echo "Playout is healthy" - break - fi - echo "Playout not ready yet (attempt $i/6), waiting 20s..." - sleep 20 - done - if ! docker exec linearvoid-playout curl -sf http://localhost:8081/health; then - echo "Playout failed to become healthy after 6 attempts" - docker logs linearvoid-playout --tail 50 - exit 1 - fi - - echo "All services healthy" - - - name: Cleanup old images - env: - DOCKER_HOST: tcp://192.168.88.40:2375 - run: | - docker image prune -f --filter "until=24h" +# CD workflow — TEST LAB DEPLOYMENT +# Deploys to test lab host 192.168.88.40:2375. +# Production Hetzner target TBD — switch DOCKER_HOST when provisioned. +# See DEPLOYMENT.md Production Deployment section for cutover plan. + +name: CD + +on: + workflow_run: + workflows: [CI] + types: [completed] + workflow_dispatch: + +jobs: + deploy: + if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + runs-on: [ubuntu-latest] + steps: + - name: Checkout code + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to registry on remote host + env: + DOCKER_HOST: tcp://192.168.88.40:2375 + run: | + echo "${{ secrets.CI_REGISTRY_TOKEN }}" | docker login git.allucanget.biz -u "${{ secrets.CI_REGISTRY_USER }}" --password-stdin + + - name: Prepare environment + run: | + cat > .env << 'EOF' + # Database & Redis + DATABASE_URL="${{ secrets.DATABASE_URL }}" + REDIS_URL=redis://redis:6379/0 + + # Playout + PLAYLIST_PATH=/app/playlist/playlist.txt + HLS_SEGMENT_DURATION=2 + HLS_WINDOW_SIZE=12 + LOG_LEVEL=info + + # Cloudflare R2 (required for HLS segment sync) + R2_ACCESS_KEY_ID="${{ secrets.R2_ACCESS_KEY_ID }}" + R2_SECRET_ACCESS_KEY="${{ secrets.R2_SECRET_ACCESS_KEY }}" + R2_BUCKET_NAME="${{ secrets.R2_BUCKET_NAME }}" + R2_ENDPOINT="${{ secrets.R2_ENDPOINT }}" + R2_REGION=auto + + # Cloudflare + CLOUDFLARE_API_TOKEN="${{ secrets.CLOUDFLARE_API_TOKEN }}" + CLOUDFLARE_ACCOUNT_ID="${{ secrets.CLOUDFLARE_ACCOUNT_ID }}" + MEDIA_DOMAIN=${{ secrets.MEDIA_DOMAIN }} + + # Frontend public URLs + VITE_PUBLIC_APP_URL=${{ secrets.VITE_PUBLIC_APP_URL }} + VITE_PUBLIC_WS_URL=${{ secrets.VITE_PUBLIC_WS_URL }} + VITE_PUBLIC_HLS_URL=${{ secrets.VITE_PUBLIC_HLS_URL }} + VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR=${{ secrets.VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR }} + VITE_PUBLIC_ADMIN_DEFAULT_ROOM=${{ secrets.VITE_PUBLIC_ADMIN_DEFAULT_ROOM }} + VITE_PUBLIC_ADMIN_MODERATION_LOG_LIMIT=50 + + # Admin auth + ADMIN_USERNAME="${{ secrets.ADMIN_USERNAME }}" + ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}" + ADMIN_SESSION_TTL_HOURS=12 + + # Nginx + # Host port 80/443 (Coolify Traefik) and 8080 are taken; nginx publishes on 8082. + NGINX_PORT=8082 + EOF + + - name: Seed Prometheus config on remote host + env: + DOCKER_HOST: tcp://192.168.88.40:2375 + run: | + docker run --rm \ + -v linearvoid_prometheus-config:/etc/prometheus \ + alpine:latest sh -c ' + cat > /etc/prometheus/prometheus.yml << "PROMEOF" + global: + scrape_interval: 15s + evaluation_interval: 15s + + rule_files: + - "/etc/prometheus/alerts.yml" + + scrape_configs: + - job_name: "chat" + static_configs: + - targets: ["chat:3000"] + - job_name: "playout" + static_configs: + - targets: ["playout:8081"] + - job_name: "nginx" + static_configs: + - targets: ["nginx-exporter:9113"] + - job_name: "redis" + static_configs: + - targets: ["redis-exporter:9121"] + - job_name: "postgres" + static_configs: + - targets: ["postgres-exporter:9187"] + - job_name: "node" + static_configs: + - targets: ["node-exporter:9100"] + - job_name: "prometheus" + static_configs: + - targets: ["localhost:9090"] + PROMEOF + cat > /etc/prometheus/alerts.yml << "ALERTSEOF" + groups: + - name: linearvoid + rules: + - alert: ChatDown + expr: up{job="chat"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Chat service is down" + - alert: PlayoutDown + expr: up{job="playout"} == 0 + for: 2m + labels: + severity: critical + annotations: + summary: "Playout service is down" + - alert: RedisDown + expr: linearvoid_chat_redis_up == 0 + for: 2m + labels: + severity: warning + annotations: + summary: "Redis is unreachable" + - alert: PlayoutSlateActive + expr: linearvoid_playout_mode == 1 + for: 1m + labels: + severity: warning + annotations: + summary: "Playout in slate fallback" + - alert: HighChatLatency + expr: rate(linearvoid_chat_messages_total[5m]) < 0.01 + for: 5m + labels: + severity: warning + annotations: + summary: "Unusually low chat message rate" + - alert: PrometheusWALFailing + expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0 + for: 5m + labels: + severity: warning + annotations: + summary: "Prometheus WAL writes failing" + - alert: HighCPU + expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 10m + labels: + severity: warning + annotations: + summary: "High CPU usage" + - alert: LowDiskSpace + expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10 + for: 5m + labels: + severity: warning + annotations: + summary: "Low disk space" + ALERTSEOF + ' + + - name: Seed Grafana provisioning on remote host + env: + DOCKER_HOST: tcp://192.168.88.40:2375 + run: | + docker run --rm \ + -v linearvoid_grafana-provisioning:/etc/grafana/provisioning \ + alpine:latest sh -c ' + mkdir -p /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards + cat > /etc/grafana/provisioning/datasources/datasource.yml << "DSEOF" + apiVersion: 1 + datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + DSEOF + cat > /etc/grafana/provisioning/dashboards/dashboard.yml << "PROVEOF" + apiVersion: 1 + providers: + - name: "default" + orgId: 1 + folder: "" + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /etc/grafana/provisioning/dashboards + PROVEOF + cat > /etc/grafana/provisioning/dashboards/linearvoid-app.json << "DBEOF" + { + "title": "Linear Void Monitoring", + "panels": [ + { + "title": "Concurrent Viewers", + "type": "timeseries", + "targets": [{ "expr": "linearvoid_chat_connected_clients" }] + }, + { + "title": "Stored Messages", + "type": "stat", + "targets": [{ "expr": "linearvoid_chat_messages_total" }] + }, + { + "title": "Redis Reachable", + "type": "stat", + "targets": [{ "expr": "linearvoid_chat_redis_up" }] + }, + { + "title": "Playout Mode", + "type": "stat", + "targets": [{ "expr": "linearvoid_playout_mode" }] + } + ] + } + DBEOF + ' + + - name: Deploy with docker-compose + env: + DOCKER_HOST: tcp://192.168.88.40:2375 + run: | + docker compose -f docker-compose.prod.yml pull + docker compose -f docker-compose.prod.yml up -d --remove-orphans --force-recreate + + - name: Health check + env: + DOCKER_HOST: tcp://192.168.88.40:2375 + run: | + # Wait for services to stabilize + sleep 30 + + # Check chat health (via docker exec — IP-agnostic, works on any DOCKER_HOST) + echo "Checking chat health..." + if ! docker exec linearvoid-chat curl -sf http://localhost:3000/health; then + echo "Chat health check failed" + docker logs linearvoid-chat --tail 50 + exit 1 + fi + + # Check nginx health + echo "Checking nginx health..." + if ! docker exec linearvoid-nginx curl -sf http://localhost/health; then + echo "Nginx health check failed" + docker logs linearvoid-nginx --tail 50 + exit 1 + fi + + # Retry playout health check since it may still be initializing + echo "Checking playout health..." + for i in $(seq 1 6); do + if docker exec linearvoid-playout curl -sf http://localhost:8081/health; then + echo "Playout is healthy" + break + fi + echo "Playout not ready yet (attempt $i/6), waiting 20s..." + sleep 20 + done + if ! docker exec linearvoid-playout curl -sf http://localhost:8081/health; then + echo "Playout failed to become healthy after 6 attempts" + docker logs linearvoid-playout --tail 50 + exit 1 + fi + + echo "All services healthy" + + - name: Cleanup old images + env: + DOCKER_HOST: tcp://192.168.88.40:2375 + run: | + docker image prune -f --filter "until=24h"