289 lines
10 KiB
YAML
289 lines
10 KiB
YAML
# CD workflow — TEST LAB DEPLOYMENT
|
|
# Deploys to test lab host 192.168.88.40:2375.
|
|
# Production Hetzner target TBD — switch DOCKER_HOST when provisioned.
|
|
# See DEPLOYMENT.md Production Deployment section for cutover plan.
|
|
|
|
name: CD
|
|
|
|
on:
|
|
workflow_run:
|
|
workflows: [CI]
|
|
types: [completed]
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
deploy:
|
|
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
|
runs-on: [ubuntu-latest]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Login to registry on remote host
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
echo "${{ secrets.CI_REGISTRY_TOKEN }}" | docker login git.allucanget.biz -u "${{ secrets.CI_REGISTRY_USER }}" --password-stdin
|
|
|
|
- name: Prepare environment
|
|
run: |
|
|
cat > .env << 'EOF'
|
|
# Database & Redis
|
|
DATABASE_URL="${{ secrets.DATABASE_URL }}"
|
|
REDIS_URL=redis://redis:6379/0
|
|
|
|
# Playout
|
|
PLAYLIST_PATH=/app/playlist/playlist.txt
|
|
HLS_SEGMENT_DURATION=2
|
|
HLS_WINDOW_SIZE=12
|
|
LOG_LEVEL=info
|
|
|
|
# Cloudflare R2 (required for HLS segment sync)
|
|
R2_ACCESS_KEY_ID="${{ secrets.R2_ACCESS_KEY_ID }}"
|
|
R2_SECRET_ACCESS_KEY="${{ secrets.R2_SECRET_ACCESS_KEY }}"
|
|
R2_BUCKET_NAME="${{ secrets.R2_BUCKET_NAME }}"
|
|
R2_ENDPOINT="${{ secrets.R2_ENDPOINT }}"
|
|
R2_REGION=auto
|
|
|
|
# Cloudflare
|
|
CLOUDFLARE_API_TOKEN="${{ secrets.CLOUDFLARE_API_TOKEN }}"
|
|
CLOUDFLARE_ACCOUNT_ID="${{ secrets.CLOUDFLARE_ACCOUNT_ID }}"
|
|
MEDIA_DOMAIN=${{ secrets.MEDIA_DOMAIN }}
|
|
|
|
# Frontend public URLs
|
|
VITE_PUBLIC_APP_URL=${{ secrets.VITE_PUBLIC_APP_URL }}
|
|
VITE_PUBLIC_WS_URL=${{ secrets.VITE_PUBLIC_WS_URL }}
|
|
VITE_PUBLIC_HLS_URL=${{ secrets.VITE_PUBLIC_HLS_URL }}
|
|
VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR=${{ secrets.VITE_PUBLIC_ADMIN_DEFAULT_MODERATOR }}
|
|
VITE_PUBLIC_ADMIN_DEFAULT_ROOM=${{ secrets.VITE_PUBLIC_ADMIN_DEFAULT_ROOM }}
|
|
VITE_PUBLIC_ADMIN_MODERATION_LOG_LIMIT=50
|
|
|
|
# Admin auth
|
|
ADMIN_USERNAME="${{ secrets.ADMIN_USERNAME }}"
|
|
ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}"
|
|
ADMIN_SESSION_TTL_HOURS=12
|
|
|
|
# Nginx
|
|
NGINX_PORT=80
|
|
EOF
|
|
|
|
- name: Seed Prometheus config on remote host
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
docker run --rm \
|
|
-v linearvoid_prometheus-config:/etc/prometheus \
|
|
alpine:latest sh -c '
|
|
cat > /etc/prometheus/prometheus.yml << "PROMEOF"
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
|
|
rule_files:
|
|
- "/etc/prometheus/alerts.yml"
|
|
|
|
scrape_configs:
|
|
- job_name: "chat"
|
|
static_configs:
|
|
- targets: ["chat:3000"]
|
|
- job_name: "playout"
|
|
static_configs:
|
|
- targets: ["playout:8081"]
|
|
- job_name: "nginx"
|
|
static_configs:
|
|
- targets: ["nginx-exporter:9113"]
|
|
- job_name: "redis"
|
|
static_configs:
|
|
- targets: ["redis-exporter:9121"]
|
|
- job_name: "postgres"
|
|
static_configs:
|
|
- targets: ["postgres-exporter:9187"]
|
|
- job_name: "node"
|
|
static_configs:
|
|
- targets: ["node-exporter:9100"]
|
|
- job_name: "prometheus"
|
|
static_configs:
|
|
- targets: ["localhost:9090"]
|
|
PROMEOF
|
|
cat > /etc/prometheus/alerts.yml << "ALERTSEOF"
|
|
groups:
|
|
- name: linearvoid
|
|
rules:
|
|
- alert: ChatDown
|
|
expr: up{job="chat"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Chat service is down"
|
|
- alert: PlayoutDown
|
|
expr: up{job="playout"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Playout service is down"
|
|
- alert: RedisDown
|
|
expr: linearvoid_chat_redis_up == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Redis is unreachable"
|
|
- alert: PlayoutSlateActive
|
|
expr: linearvoid_playout_mode == 1
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Playout in slate fallback"
|
|
- alert: HighChatLatency
|
|
expr: rate(linearvoid_chat_messages_total[5m]) < 0.01
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Unusually low chat message rate"
|
|
- alert: PrometheusWALFailing
|
|
expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Prometheus WAL writes failing"
|
|
- alert: HighCPU
|
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High CPU usage"
|
|
- alert: LowDiskSpace
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Low disk space"
|
|
ALERTSEOF
|
|
'
|
|
|
|
- name: Seed Grafana provisioning on remote host
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
docker run --rm \
|
|
-v linearvoid_grafana-provisioning:/etc/grafana/provisioning \
|
|
alpine:latest sh -c '
|
|
mkdir -p /etc/grafana/provisioning/datasources /etc/grafana/provisioning/dashboards
|
|
cat > /etc/grafana/provisioning/datasources/datasource.yml << "DSEOF"
|
|
apiVersion: 1
|
|
datasources:
|
|
- name: Prometheus
|
|
type: prometheus
|
|
access: proxy
|
|
url: http://prometheus:9090
|
|
isDefault: true
|
|
editable: true
|
|
DSEOF
|
|
cat > /etc/grafana/provisioning/dashboards/dashboard.yml << "PROVEOF"
|
|
apiVersion: 1
|
|
providers:
|
|
- name: "default"
|
|
orgId: 1
|
|
folder: ""
|
|
type: file
|
|
disableDeletion: false
|
|
updateIntervalSeconds: 10
|
|
allowUiUpdates: true
|
|
options:
|
|
path: /etc/grafana/provisioning/dashboards
|
|
PROVEOF
|
|
cat > /etc/grafana/provisioning/dashboards/linearvoid-app.json << "DBEOF"
|
|
{
|
|
"title": "Linear Void Monitoring",
|
|
"panels": [
|
|
{
|
|
"title": "Concurrent Viewers",
|
|
"type": "timeseries",
|
|
"targets": [{ "expr": "linearvoid_chat_connected_clients" }]
|
|
},
|
|
{
|
|
"title": "Stored Messages",
|
|
"type": "stat",
|
|
"targets": [{ "expr": "linearvoid_chat_messages_total" }]
|
|
},
|
|
{
|
|
"title": "Redis Reachable",
|
|
"type": "stat",
|
|
"targets": [{ "expr": "linearvoid_chat_redis_up" }]
|
|
},
|
|
{
|
|
"title": "Playout Mode",
|
|
"type": "stat",
|
|
"targets": [{ "expr": "linearvoid_playout_mode" }]
|
|
}
|
|
]
|
|
}
|
|
DBEOF
|
|
'
|
|
|
|
- name: Deploy with docker-compose
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
docker compose -f docker-compose.prod.yml pull
|
|
docker compose -f docker-compose.prod.yml up -d --remove-orphans --force-recreate
|
|
|
|
- name: Health check
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
# Wait for services to stabilize
|
|
sleep 30
|
|
|
|
# Check chat health (via docker exec — IP-agnostic, works on any DOCKER_HOST)
|
|
echo "Checking chat health..."
|
|
if ! docker exec linearvoid-chat curl -sf http://localhost:3000/health; then
|
|
echo "Chat health check failed"
|
|
docker logs linearvoid-chat --tail 50
|
|
exit 1
|
|
fi
|
|
|
|
# Check nginx health
|
|
echo "Checking nginx health..."
|
|
if ! docker exec linearvoid-nginx curl -sf http://localhost/health; then
|
|
echo "Nginx health check failed"
|
|
docker logs linearvoid-nginx --tail 50
|
|
exit 1
|
|
fi
|
|
|
|
# Retry playout health check since it may still be initializing
|
|
echo "Checking playout health..."
|
|
for i in $(seq 1 6); do
|
|
if docker exec linearvoid-playout curl -sf http://localhost:8081/health; then
|
|
echo "Playout is healthy"
|
|
break
|
|
fi
|
|
echo "Playout not ready yet (attempt $i/6), waiting 10s..."
|
|
sleep 10
|
|
done
|
|
if ! docker exec linearvoid-playout curl -sf http://localhost:8081/health; then
|
|
echo "Playout failed to become healthy after 6 attempts"
|
|
docker logs linearvoid-playout --tail 50
|
|
exit 1
|
|
fi
|
|
|
|
echo "All services healthy"
|
|
|
|
- name: Cleanup old images
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
docker image prune -f --filter "until=24h"
|