zwitschi
0f81a81029
- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100) - Create 8 alerting rules in monitoring/prometheus/alerts.yml - Update prometheus.yml with all scrape targets + rule_files - WS connection rate limiting: max 5/IP/60s (Redis INCR) - Admin login rate limiting: max 5/IP/5min (Redis INCR) - 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test - docs/incident-runbook.md with 5-tier triage + escalation - Update CD workflow: full prometheus config + alerts seeding
64 lines
2.2 KiB
Bash
64 lines
2.2 KiB
Bash
#!/bin/bash
|
|
# Weekly backup integrity verification.
|
|
# Downloads latest backup from R2, restores into temp container, runs sanity checks.
|
|
# Schedule: cron weekly (after weekly backup).
|
|
set -e
|
|
|
|
R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
|
|
TEMP_DIR=$(mktemp -d)
|
|
trap "rm -rf $TEMP_DIR" EXIT
|
|
|
|
echo "[$(date -Iseconds)] Starting backup integrity verification..."
|
|
|
|
# Download latest weekly backup from R2
|
|
if [ -z "$R2_ENDPOINT" ] || [ -z "$R2_ACCESS_KEY_ID" ]; then
|
|
echo " SKIP: R2 credentials not configured (R2_ENDPOINT, R2_ACCESS_KEY_ID)"
|
|
exit 0
|
|
fi
|
|
|
|
LATEST=$(aws s3 ls "s3://${R2_BUCKET}/weekly/" --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null | sort | tail -1 | awk '{print $4}')
|
|
if [ -z "$LATEST" ]; then
|
|
echo " SKIP: No weekly backups found in R2"
|
|
exit 0
|
|
fi
|
|
|
|
echo " Downloading: $LATEST"
|
|
aws s3 cp "s3://${R2_BUCKET}/weekly/${LATEST}" "$TEMP_DIR/backup.dump" \
|
|
--endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null
|
|
|
|
echo " File size: $(du -h "$TEMP_DIR/backup.dump" | cut -f1)"
|
|
|
|
# Restore into temp PostgreSQL container
|
|
echo " Starting temp PostgreSQL..."
|
|
docker run -d --name lv-backup-test-$$ \
|
|
-e POSTGRES_PASSWORD=test \
|
|
-e POSTGRES_DB=linearvoid \
|
|
-p 15432:5432 \
|
|
postgres:15-alpine 2>/dev/null
|
|
|
|
sleep 5
|
|
|
|
# Wait for PG to be ready
|
|
for i in $(seq 1 10); do
|
|
if docker exec lv-backup-test-$$ pg_isready -U postgres 2>/dev/null; then
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
# Restore
|
|
echo " Restoring backup..."
|
|
docker cp "$TEMP_DIR/backup.dump" "lv-backup-test-$$:/tmp/backup.dump"
|
|
docker exec lv-backup-test-$$ pg_restore -U postgres -d linearvoid /tmp/backup.dump 2>/dev/null && \
|
|
echo " Restore: OK" || { echo " Restore: FAILED"; docker rm -f lv-backup-test-$$ 2>/dev/null; exit 1; }
|
|
|
|
# Sanity checks
|
|
MSG_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM messages;" 2>/dev/null || echo "0")
|
|
MOD_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM moderation_logs;" 2>/dev/null || echo "0")
|
|
echo " Messages: $MSG_COUNT, Moderation: $MOD_COUNT"
|
|
|
|
# Cleanup
|
|
docker rm -f lv-backup-test-$$ 2>/dev/null
|
|
|
|
echo "[$(date -Iseconds)] Backup verification complete. OK."
|