175 lines
5.8 KiB
YAML
175 lines
5.8 KiB
YAML
# CD workflow — TEST LAB DEPLOYMENT
|
|
# Deploys to test lab host 192.168.88.40:2375.
|
|
# Production Hetzner target TBD — switch DOCKER_HOST when provisioned.
|
|
# See DEPLOYMENT.md Production Deployment section for cutover plan.
|
|
|
|
name: CD
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
workflow_dispatch:
|
|
|
|
jobs:
|
|
deploy:
|
|
runs-on: [ubuntu-latest]
|
|
steps:
|
|
- name: Checkout code
|
|
uses: actions/checkout@v4
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Set up Docker Buildx
|
|
uses: docker/setup-buildx-action@v3
|
|
|
|
- name: Prepare environment
|
|
run: |
|
|
cat > .env << 'EOF'
|
|
DATABASE_URL="${{ secrets.DATABASE_URL }}"
|
|
REDIS_URL=redis://redis:6379/0
|
|
PLAYLIST_PATH=/app/playlist/playlist.txt
|
|
HLS_SEGMENT_DURATION=6
|
|
HLS_WINDOW_SIZE=24
|
|
LOG_LEVEL=info
|
|
ADMIN_USERNAME="${{ secrets.ADMIN_USERNAME }}"
|
|
ADMIN_PASSWORD="${{ secrets.ADMIN_PASSWORD }}"
|
|
ADMIN_SESSION_TTL_HOURS=12
|
|
NGINX_PORT=80
|
|
EOF
|
|
|
|
- name: Seed Prometheus config
|
|
run: |
|
|
mkdir -p monitoring/prometheus
|
|
cat > monitoring/prometheus/prometheus.yml << 'EOF'
|
|
global:
|
|
scrape_interval: 15s
|
|
evaluation_interval: 15s
|
|
|
|
rule_files:
|
|
- "/etc/prometheus/alerts.yml"
|
|
|
|
scrape_configs:
|
|
- job_name: 'chat'
|
|
static_configs:
|
|
- targets: ['chat:3000']
|
|
- job_name: 'playout'
|
|
static_configs:
|
|
- targets: ['playout:8081']
|
|
- job_name: 'nginx'
|
|
static_configs:
|
|
- targets: ['nginx-exporter:9113']
|
|
- job_name: 'redis'
|
|
static_configs:
|
|
- targets: ['redis-exporter:9121']
|
|
- job_name: 'postgres'
|
|
static_configs:
|
|
- targets: ['postgres-exporter:9187']
|
|
- job_name: 'node'
|
|
static_configs:
|
|
- targets: ['node-exporter:9100']
|
|
- job_name: 'prometheus'
|
|
static_configs:
|
|
- targets: ['localhost:9090']
|
|
EOF
|
|
cat > monitoring/prometheus/alerts.yml << 'EOF'
|
|
groups:
|
|
- name: linearvoid
|
|
rules:
|
|
- alert: ChatDown
|
|
expr: up{job="chat"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Chat service is down"
|
|
- alert: PlayoutDown
|
|
expr: up{job="playout"} == 0
|
|
for: 2m
|
|
labels:
|
|
severity: critical
|
|
annotations:
|
|
summary: "Playout service is down"
|
|
- alert: RedisDown
|
|
expr: linearvoid_chat_redis_up == 0
|
|
for: 2m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Redis is unreachable"
|
|
- alert: PlayoutSlateActive
|
|
expr: linearvoid_playout_mode == 1
|
|
for: 1m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Playout in slate fallback"
|
|
- alert: HighCPU
|
|
expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
|
|
for: 10m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "High CPU usage"
|
|
- alert: LowDiskSpace
|
|
expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
|
|
for: 5m
|
|
labels:
|
|
severity: warning
|
|
annotations:
|
|
summary: "Low disk space"
|
|
EOF
|
|
|
|
- name: Deploy with docker-compose
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
docker compose -f docker-compose.prod.yml pull
|
|
docker compose -f docker-compose.prod.yml up -d --remove-orphans
|
|
|
|
- name: Health check
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
# Wait for services to stabilize
|
|
sleep 20
|
|
|
|
# Check chat health (via docker exec — IP-agnostic, works on any DOCKER_HOST)
|
|
echo "Checking chat health..."
|
|
if ! docker exec linearvoid-chat curl -sf http://localhost:3000/health; then
|
|
echo "Chat health check failed"
|
|
docker logs linearvoid-chat --tail 50
|
|
exit 1
|
|
fi
|
|
|
|
# Check nginx health
|
|
echo "Checking nginx health..."
|
|
if ! docker exec linearvoid-nginx curl -sf http://localhost/health; then
|
|
echo "Nginx health check failed"
|
|
docker logs linearvoid-nginx --tail 50
|
|
exit 1
|
|
fi
|
|
|
|
# Retry playout health check since it may still be initializing
|
|
echo "Checking playout health..."
|
|
for i in $(seq 1 6); do
|
|
if docker exec linearvoid-playout curl -sf http://localhost:8081/health; then
|
|
echo "Playout is healthy"
|
|
break
|
|
fi
|
|
echo "Playout not ready yet (attempt $i/6), waiting 5s..."
|
|
sleep 5
|
|
done
|
|
if ! docker exec linearvoid-playout curl -sf http://localhost:8081/health; then
|
|
echo "Playout failed to become healthy after 6 attempts"
|
|
docker logs linearvoid-playout --tail 50
|
|
exit 1
|
|
fi
|
|
|
|
echo "All services healthy"
|
|
|
|
- name: Cleanup old images
|
|
env:
|
|
DOCKER_HOST: tcp://192.168.88.40:2375
|
|
run: |
|
|
docker image prune -f --filter "until=24h"
|