feat: monitoring exporters, alerting, rate limiting, backups, incident runbook
- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100) - Create 8 alerting rules in monitoring/prometheus/alerts.yml - Update prometheus.yml with all scrape targets + rule_files - WS connection rate limiting: max 5/IP/60s (Redis INCR) - Admin login rate limiting: max 5/IP/5min (Redis INCR) - 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test - docs/incident-runbook.md with 5-tier triage + escalation - Update CD workflow: full prometheus config + alerts seeding
This commit is contained in:
zwitschi committed
1 parent
b5710548b1
commit
0f81a81029
11 files changed
+683
-73
No files matched your search
@@ -0,0 +1,45 @@
|
||||
#!/bin/bash
|
||||
# Daily PostgreSQL backup to R2 (14-day retention)
|
||||
# Schedule: cron daily. Requires: pg_dump, aws CLI or rclone for R2.
|
||||
set -e
|
||||
|
||||
BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}"
|
||||
RETENTION_DAYS="${RETENTION_DAYS:-14}"
|
||||
R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
|
||||
DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}"
|
||||
|
||||
TIMESTAMP=$(date +%Y%m%d)
|
||||
BACKUP_FILE="$BACKUP_DIR/daily/db_${TIMESTAMP}.dump"
|
||||
|
||||
mkdir -p "$(dirname "$BACKUP_FILE")"
|
||||
|
||||
echo "[$(date -Iseconds)] Starting daily backup..."
|
||||
|
||||
# Extract connection details from DATABASE_URL
|
||||
DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
|
||||
DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
|
||||
DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p')
|
||||
DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p')
|
||||
DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p')
|
||||
|
||||
export PGPASSWORD="${DB_PASS}"
|
||||
|
||||
# Dump
|
||||
pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \
|
||||
-Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}"
|
||||
|
||||
echo " Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
|
||||
|
||||
# Upload to R2
|
||||
if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then
|
||||
aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/daily/db_${TIMESTAMP}.dump" \
|
||||
--endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \
|
||||
echo " Uploaded to R2: daily/db_${TIMESTAMP}.dump" || \
|
||||
echo " WARNING: R2 upload failed"
|
||||
fi
|
||||
|
||||
# Cleanup old backups
|
||||
find "$BACKUP_DIR/daily" -name "db_*.dump" -mtime "+${RETENTION_DAYS}" -delete 2>/dev/null || true
|
||||
echo " Cleaned up backups older than ${RETENTION_DAYS} days"
|
||||
|
||||
echo "[$(date -Iseconds)] Daily backup complete."
|
||||
@@ -0,0 +1,48 @@
|
||||
#!/bin/bash
|
||||
# Weekly full PostgreSQL backup to R2 (8-week retention)
|
||||
# Schedule: cron weekly (runs backup-db.sh with weekly prefix)
|
||||
set -e
|
||||
|
||||
BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}"
|
||||
RETENTION_WEEKS="${RETENTION_WEEKS:-8}"
|
||||
R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
|
||||
DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}"
|
||||
|
||||
TIMESTAMP=$(date +%Y%m%d)
|
||||
BACKUP_FILE="$BACKUP_DIR/weekly/db_${TIMESTAMP}.dump"
|
||||
|
||||
mkdir -p "$(dirname "$BACKUP_FILE")"
|
||||
|
||||
echo "[$(date -Iseconds)] Starting weekly full backup..."
|
||||
|
||||
DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
|
||||
DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
|
||||
DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p')
|
||||
DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p')
|
||||
DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p')
|
||||
|
||||
export PGPASSWORD="${DB_PASS}"
|
||||
|
||||
pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \
|
||||
-Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}"
|
||||
|
||||
echo " Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
|
||||
|
||||
# Verify integrity
|
||||
pg_restore --list "$BACKUP_FILE" > /dev/null 2>&1 && \
|
||||
echo " Integrity check: PASSED" || \
|
||||
echo " WARNING: Integrity check FAILED"
|
||||
|
||||
# Upload
|
||||
if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then
|
||||
aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/weekly/db_${TIMESTAMP}.dump" \
|
||||
--endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \
|
||||
echo " Uploaded to R2: weekly/db_${TIMESTAMP}.dump" || \
|
||||
echo " WARNING: R2 upload failed"
|
||||
fi
|
||||
|
||||
# Cleanup
|
||||
find "$BACKUP_DIR/weekly" -name "db_*.dump" -mtime "+$((RETENTION_WEEKS * 7))" -delete 2>/dev/null || true
|
||||
echo " Cleaned up backups older than ${RETENTION_WEEKS} weeks"
|
||||
|
||||
echo "[$(date -Iseconds)] Weekly backup complete."
|
||||
@@ -0,0 +1,63 @@
|
||||
#!/bin/bash
|
||||
# Weekly backup integrity verification.
|
||||
# Downloads latest backup from R2, restores into temp container, runs sanity checks.
|
||||
# Schedule: cron weekly (after weekly backup).
|
||||
set -e
|
||||
|
||||
R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
|
||||
TEMP_DIR=$(mktemp -d)
|
||||
trap "rm -rf $TEMP_DIR" EXIT
|
||||
|
||||
echo "[$(date -Iseconds)] Starting backup integrity verification..."
|
||||
|
||||
# Download latest weekly backup from R2
|
||||
if [ -z "$R2_ENDPOINT" ] || [ -z "$R2_ACCESS_KEY_ID" ]; then
|
||||
echo " SKIP: R2 credentials not configured (R2_ENDPOINT, R2_ACCESS_KEY_ID)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
LATEST=$(aws s3 ls "s3://${R2_BUCKET}/weekly/" --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null | sort | tail -1 | awk '{print $4}')
|
||||
if [ -z "$LATEST" ]; then
|
||||
echo " SKIP: No weekly backups found in R2"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo " Downloading: $LATEST"
|
||||
aws s3 cp "s3://${R2_BUCKET}/weekly/${LATEST}" "$TEMP_DIR/backup.dump" \
|
||||
--endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null
|
||||
|
||||
echo " File size: $(du -h "$TEMP_DIR/backup.dump" | cut -f1)"
|
||||
|
||||
# Restore into temp PostgreSQL container
|
||||
echo " Starting temp PostgreSQL..."
|
||||
docker run -d --name lv-backup-test-$$ \
|
||||
-e POSTGRES_PASSWORD=test \
|
||||
-e POSTGRES_DB=linearvoid \
|
||||
-p 15432:5432 \
|
||||
postgres:15-alpine 2>/dev/null
|
||||
|
||||
sleep 5
|
||||
|
||||
# Wait for PG to be ready
|
||||
for i in $(seq 1 10); do
|
||||
if docker exec lv-backup-test-$$ pg_isready -U postgres 2>/dev/null; then
|
||||
break
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
|
||||
# Restore
|
||||
echo " Restoring backup..."
|
||||
docker cp "$TEMP_DIR/backup.dump" "lv-backup-test-$$:/tmp/backup.dump"
|
||||
docker exec lv-backup-test-$$ pg_restore -U postgres -d linearvoid /tmp/backup.dump 2>/dev/null && \
|
||||
echo " Restore: OK" || { echo " Restore: FAILED"; docker rm -f lv-backup-test-$$ 2>/dev/null; exit 1; }
|
||||
|
||||
# Sanity checks
|
||||
MSG_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM messages;" 2>/dev/null || echo "0")
|
||||
MOD_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM moderation_logs;" 2>/dev/null || echo "0")
|
||||
echo " Messages: $MSG_COUNT, Moderation: $MOD_COUNT"
|
||||
|
||||
# Cleanup
|
||||
docker rm -f lv-backup-test-$$ 2>/dev/null
|
||||
|
||||
echo "[$(date -Iseconds)] Backup verification complete. OK."
|
||||
Reference in new issue
Block a user