zwitschi
0f81a81029
- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100) - Create 8 alerting rules in monitoring/prometheus/alerts.yml - Update prometheus.yml with all scrape targets + rule_files - WS connection rate limiting: max 5/IP/60s (Redis INCR) - Admin login rate limiting: max 5/IP/5min (Redis INCR) - 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test - docs/incident-runbook.md with 5-tier triage + escalation - Update CD workflow: full prometheus config + alerts seeding
49 lines
1.8 KiB
Bash
49 lines
1.8 KiB
Bash
#!/bin/bash
|
|
# Weekly full PostgreSQL backup to R2 (8-week retention)
|
|
# Schedule: cron weekly (runs backup-db.sh with weekly prefix)
|
|
set -e
|
|
|
|
BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}"
|
|
RETENTION_WEEKS="${RETENTION_WEEKS:-8}"
|
|
R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
|
|
DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}"
|
|
|
|
TIMESTAMP=$(date +%Y%m%d)
|
|
BACKUP_FILE="$BACKUP_DIR/weekly/db_${TIMESTAMP}.dump"
|
|
|
|
mkdir -p "$(dirname "$BACKUP_FILE")"
|
|
|
|
echo "[$(date -Iseconds)] Starting weekly full backup..."
|
|
|
|
DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
|
|
DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
|
|
DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p')
|
|
DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p')
|
|
DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p')
|
|
|
|
export PGPASSWORD="${DB_PASS}"
|
|
|
|
pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \
|
|
-Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}"
|
|
|
|
echo " Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
|
|
|
|
# Verify integrity
|
|
pg_restore --list "$BACKUP_FILE" > /dev/null 2>&1 && \
|
|
echo " Integrity check: PASSED" || \
|
|
echo " WARNING: Integrity check FAILED"
|
|
|
|
# Upload
|
|
if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then
|
|
aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/weekly/db_${TIMESTAMP}.dump" \
|
|
--endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \
|
|
echo " Uploaded to R2: weekly/db_${TIMESTAMP}.dump" || \
|
|
echo " WARNING: R2 upload failed"
|
|
fi
|
|
|
|
# Cleanup
|
|
find "$BACKUP_DIR/weekly" -name "db_*.dump" -mtime "+$((RETENTION_WEEKS * 7))" -delete 2>/dev/null || true
|
|
echo " Cleaned up backups older than ${RETENTION_WEEKS} weeks"
|
|
|
|
echo "[$(date -Iseconds)] Weekly backup complete."
|