fix: force-recreate containers on deploy to pick up volume changes

- Add 4 Prometheus exporters: nginx (9113), redis (9121), postgres (9187), node (9100)
- Create 8 alerting rules in monitoring/prometheus/alerts.yml
- Update prometheus.yml with all scrape targets + rule_files
- WS connection rate limiting: max 5/IP/60s (Redis INCR)
- Admin login rate limiting: max 5/IP/5min (Redis INCR)
- 3 backup scripts: daily to R2 (14d), weekly full (8w), verify with restore test
- docs/incident-runbook.md with 5-tier triage + escalation
- Update CD workflow: full prometheus config + alerts seeding

.gitea/workflows/cd.yml

@@ -1,23 +1,18 @@
 # CD workflow — TEST LAB DEPLOYMENT
 # Deploys to test lab host 192.168.88.40:2375.
 # Production Hetzner target TBD — switch DOCKER_HOST when provisioned.
-# See TODO.md Production Deployment section for cutover plan.
+# See DEPLOYMENT.md Production Deployment section for cutover plan.
 
 name: CD
 
 on:
-  workflow_run:
-    workflows: ["CI"]
-    types:
-      - completed
+  push:
     branches: [main]
   workflow_dispatch:
 
 jobs:
   deploy:
     runs-on: [ubuntu-latest]
-    if: ${{ github.event.workflow_run.conclusion == 'success' || github.event_name == 'workflow_dispatch' }}
-    environment: production
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -27,12 +22,11 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Log in to Gitea Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: git.allucanget.biz
-          username: ${{ secrets.CI_REGISTRY_USER }}
-          password: ${{ secrets.CI_REGISTRY_TOKEN }}
+      - name: Login to registry on remote host
+        env:
+          DOCKER_HOST: tcp://192.168.88.40:2375
+        run: |
+          echo "${{ secrets.CI_REGISTRY_TOKEN }}" | docker login git.allucanget.biz -u "${{ secrets.CI_REGISTRY_USER }}" --password-stdin
 
       - name: Prepare environment
         run: |
@@ -49,30 +43,99 @@ jobs:
           NGINX_PORT=80
           EOF
 
-      - name: Seed Prometheus config
+      - name: Seed Prometheus config on remote host
+        env:
+          DOCKER_HOST: tcp://192.168.88.40:2375
         run: |
-          mkdir -p monitoring/prometheus
-          cat > monitoring/prometheus/prometheus.yml << 'EOF'
+          docker run --rm \
+            -v prometheus-config:/etc/prometheus \
+            alpine:latest sh -c '
+              cat > /etc/prometheus/prometheus.yml << "PROMEOF"
           global:
             scrape_interval: 15s
             evaluation_interval: 15s
 
-          scrape_configs:
-            - job_name: 'prometheus'
-              static_configs:
-                - targets: ['localhost:9090']
+          rule_files:
+            - "/etc/prometheus/alerts.yml"
 
-            - job_name: 'linearvoid'
+          scrape_configs:
+            - job_name: "chat"
               static_configs:
-                - targets: ['playout:8081', 'chat:3000', 'nginx:80']
-          EOF
+                - targets: ["chat:3000"]
+            - job_name: "playout"
+              static_configs:
+                - targets: ["playout:8081"]
+            - job_name: "nginx"
+              static_configs:
+                - targets: ["nginx-exporter:9113"]
+            - job_name: "redis"
+              static_configs:
+                - targets: ["redis-exporter:9121"]
+            - job_name: "postgres"
+              static_configs:
+                - targets: ["postgres-exporter:9187"]
+            - job_name: "node"
+              static_configs:
+                - targets: ["node-exporter:9100"]
+            - job_name: "prometheus"
+              static_configs:
+                - targets: ["localhost:9090"]
+          PROMEOF
+              cat > /etc/prometheus/alerts.yml << "ALERTSEOF"
+          groups:
+            - name: linearvoid
+              rules:
+                - alert: ChatDown
+                  expr: up{job="chat"} == 0
+                  for: 2m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: "Chat service is down"
+                - alert: PlayoutDown
+                  expr: up{job="playout"} == 0
+                  for: 2m
+                  labels:
+                    severity: critical
+                  annotations:
+                    summary: "Playout service is down"
+                - alert: RedisDown
+                  expr: linearvoid_chat_redis_up == 0
+                  for: 2m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Redis is unreachable"
+                - alert: PlayoutSlateActive
+                  expr: linearvoid_playout_mode == 1
+                  for: 1m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Playout in slate fallback"
+                - alert: HighCPU
+                  expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+                  for: 10m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "High CPU usage"
+                - alert: LowDiskSpace
+                  expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
+                  for: 5m
+                  labels:
+                    severity: warning
+                  annotations:
+                    summary: "Low disk space"
+          ALERTSEOF
+            '
 
       - name: Deploy with docker-compose
         env:
           DOCKER_HOST: tcp://192.168.88.40:2375
         run: |
           docker compose -f docker-compose.prod.yml pull
-          docker compose -f docker-compose.prod.yml up -d --remove-orphans
+          docker compose -f docker-compose.prod.yml up -d --remove-orphans --force-recreate
 
       - name: Health check
         env:

.gitea/workflows/ci.yml

@@ -43,7 +43,7 @@ jobs:
         run: cd backend/chat && GOTOOLCHAIN=local go vet ./...
 
       - name: Run Tests
-        run: echo "Tests will be added in future CI steps (per TODO.md)"
+        run: echo "Tests will be added in future CI steps"
 
   docker:
     needs: test
@@ -132,20 +132,3 @@ jobs:
           labels: ${{ steps.meta-nginx.outputs.labels }}
           cache-from: type=registry,ref=git.allucanget.biz/zwitschi/linearvoid/buildcache:nginx
           cache-to: type=registry,ref=git.allucanget.biz/zwitschi/linearvoid/buildcache:nginx,mode=max
-
-  release:
-    needs: docker
-    runs-on: [ubuntu-latest]
-    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Create Release
-        uses: softprops/action-gh-release@v1
-        with:
-          generate_release_notes: true
-        env:
-          GITHUB_TOKEN: ${{ secrets.CI_REGISTRY_TOKEN }}

.github/MEMORY.md

@@ -1,9 +0,0 @@
-# MEMORY.md
-
-## 2026-08-05: Chat Message Admin + CI/CD + Release 1.0.5
-
-**Completed**: Chat Message Admin, CI/CD, Release 1.0.5 committed + tagged + pushed.
-
-**Bug — nested admin page**: Fixed. Root cause: page HTML files were in `frontend/admin/` (Vite source root). Vite multi-page mode injected scripts/styles into every `.html` entry. `fetch("/admin/dashboard.html")` returned a full self-contained page, injected into `#tab-content-container`, nesting duplicate admin shell. Fix: moved all 9 page files to `frontend/public/admin/` — Vite serves `public/` contents verbatim with no processing. Fetch now returns raw HTML fragment.
-
-**Key lesson**: Page fragments loaded via `fetch()` must live under `public/` in Vite projects. Any `.html` in the project source root gets processed as a multi-page entry.

.github/agents/caveman.agent.md

@@ -1,79 +0,0 @@
----
-description: "Caveman-style agent for linearvoid project. Terse output, plan-first workflow, confirmation gating. Use for: implementing TODO.md tasks, editing project files, docs updates, release prep."
-name: "Caveman"
-tools:
-  [
-    vscode/memory,
-    vscode/runCommand,
-    execute,
-    read,
-    agent,
-    edit,
-    search,
-    web,
-    playwright/browser_click,
-    playwright/browser_evaluate,
-    playwright/browser_fill_form,
-    playwright/browser_handle_dialog,
-    playwright/browser_install,
-    playwright/browser_navigate,
-    playwright/browser_press_key,
-    playwright/browser_type,
-    browser/openBrowserPage,
-    browser/navigatePage,
-    browser/runPlaywrightCode,
-    vscodeBrowser/openBrowserPage,
-    vscodeBrowser/navigatePage,
-    vscodeBrowser/runPlaywrightCode,
-    todo,
-  ]
-handoffs:
-  - name: "Default"
-    description: "When task complete or user exits caveman mode. Also handoff for security warnings or irreversible actions."
-    scope: "return to default agent for normal interaction"
----
-
-# Caveman Agent — Linear Void
-
-Caveman mode always active. Terse, no fluff, no filler, no tool-call narration. Technical substance preserved exactly.
-
-## Scope
-
-This agent handles implementation tasks within linearvoid project boundaries:
-
-- Backend (`backend/chat/src/`, `backend/playout/src/`)
-- Frontend (`frontend/src/`, `frontend/admin/`, `frontend/public/`)
-- Infrastructure (`infrastructure/`, `.gitea/workflows/`)
-- Documentation (`docs/`, `*.md` at project root)
-- Config (`docker-compose*.yml`, `.env.example`, `package.json`, `go.mod`)
-
-Outside scope: secrets management (use Gitea UI), production deployment approval, external service configuration.
-
-## Output Rules
-
-Drop articles (a/an/the), filler words, pleasantries, hedging. Fragments OK. Short synonyms. Standard acronyms OK (API, DB, CI). Never invent abbreviations. Code blocks, error strings, technical terms verbatim. Never drop negations (not/never/no/only/except). Numbers and units exact.
-
-## Project Workflow
-
-**Documentation first. Always.**
-
-1. **Read docs before code** — Check `docs/architecture/README.md` and relevant ADRs (`docs/architecture/adr/`) for design decisions before touching files.
-2. **Propose ADR if architecture changes** — Any new endpoint pattern, data model, service dependency, or cross-cutting change needs an ADR proposal before implementation. Write to `docs/architecture/adr/ADR-XXX.md` following existing template.
-3. **Plan** — Write tasks to `.github/instructions/TODO.md` with checkboxes.
-4. **Confirm before acting** — Ask user. Accept `go`, `ok`, `yes`, `next`, `continue`. Abort on `no`, `stop`, `cancel`.
-5. **Do the work** — Edit files, run commands.
-6. **Track completion** — Mark `[x]` in TODO.md. When section done, move to `.github/instructions/DONE.md`.
-7. **Log changes** — Write entry to `CHANGELOG.md` for significant work.
-8. **Sync memory** — Update `.github/MEMORY.md` between major steps and at end of session. Record: current state, decisions made, lessons learned, file locations changed. This is the agent's persistent working memory across sessions.
-
-## Architecture
-
-Source of truth: `docs/architecture/README.md` and ADRs (`docs/architecture/adr/`). Follow project structure in root `README.md`.
-
-## Security
-
-Never commit raw secrets. Never print or log secrets. Use `CI_` prefix for Gitea secrets (never `GITEA_`). `.env` always gitignored.
-
-## Task Tracking
-
-Sync with `manage_todo_list` tool. Mirror state to `.github/instructions/TODO.md` checkboxes.

.github/instructions/BACKLOG.md

@@ -1,121 +0,0 @@
-# BACKLOG
-
-## Playout & Delivery
-
-- [x] Implement FFmpeg + MediaMTX/SRS continuous HLS playout from playlist
-- [x] Implement immutable HLS segment naming for cache-friendly delivery
-- [x] Upload HLS segments to Cloudflare R2 via S3-compatible API
-- [x] Implement emergency slate fallback on playout failure
-- [x] Auto-restart loop for FFmpeg/MediaMTX on failure
-
-## Video Normalization
-
-- [x] Implement video normalization pipeline (H.264/AAC, 30 fps, 2 s keyframes)
-- [x] Implement ffprobe verification gate rejecting non-conforming files
-- [x] Enforce normalization check before upload acceptance in admin UI
-
-## Chat Service
-
-- [x] Implement WebSocket chat backend (Go) with room fan-out
-- [x] Implement private DM channel routing
-- [x] Implement Redis presence tracking for chat
-- [ ] Implement Redis pub/sub for multi-instance chat
-- [x] Implement Redis rate limiting (INCR + EXPIRE windowing, >10/10s throttle)
-- [x] Implement chat message persistence (SQLite v1, PostgreSQL migration path)
-- [x] Implement moderation commands (/mod, /mute, /ban) in backend
-- [x] Implement moderation audit log persistence
-- [x] Implement graceful degradation when Redis unavailable (core chat remains up)
-
-## Frontend — Viewer SPA
-
-- [x] Build static SPA with hls.js player integration
-- [x] Build chat sidebar UI (public rooms + private DMs)
-- [x] Implement schedule metadata display synced with playlist timeline
-- [x] Implement responsive/mobile-friendly viewer layout
-
-## Frontend — Admin Interfaces
-
-- [x] Build admin login page with session authentication
-- [x] Build admin dashboard overview page
-- [x] Build video upload interface with chunked upload support
-- [x] Build playlist management interface (drag-to-reorder, validation warnings)
-- [x] Build schedule override/management interface (CRUD + conflict detection)
-- [x] Build user management interface (mute/unmute/ban/unban actions)
-- [x] Build moderation dashboard (chat log, inline mod actions, CSV export)
-- [x] Build analytics dashboard (time range selector)
-- [x] Build system configuration/status page (config editor, restart buttons)
-
-## Infrastructure — nginx
-
-- [x] Configure nginx reverse proxy with auth_request for admin paths
-- [x] Configure nginx HLS path with CORS headers + OPTIONS preflight
-- [x] Configure nginx WebSocket upgrade forwarding for /ws
-- [x] Configure nginx health endpoints (/health JSON, stub_status)
-
-## Infrastructure — Docker
-
-- [x] Create Dockerfiles for playout, chat, nginx services
-- [x] Create docker-compose.yml for all services
-- [x] Create docker-compose.prod.yml override for production
-
-## Infrastructure — CI/CD (Gitea Actions)
-
-- [x] Implement CI workflow (lint, build, test, push images)
-- [x] Implement CD workflow (pull, deploy, health checks — IP-agnostic)
-- [x] Generate .env from Gitea secrets in CD workflow
-- [x] Seed Prometheus config into named volume during deploy
-- [x] Implement health check validation with retry in CD pipeline
-
-## Cloudflare Configuration
-
-- [ ] Configure DNS for linearvoid.tv apex and www redirect (manual)
-- [ ] Configure media.linearvoid.tv for split-domain HLS delivery (manual)
-- [ ] Create R2 bucket (manual)
-- [ ] Configure Cache Rules: .ts/.m4s + .m3u8 (manual)
-- [ ] Configure Cloudflare Pages (manual)
-- [ ] Enable Tiered Cache, WAF, rate limiting (manual)
-- [x] Implement Worker for immutable cache headers on segment paths
-- [ ] Enforce HTTPS/WSS, redirect HTTP → HTTPS (manual)
-
-## Monitoring & Observability
-
-- [x] Set up Prometheus with chat scrape target
-- [ ] Add nginx/redis/postgres/node exporters
-- [x] Import Grafana app + operational dashboards
-- [ ] Define Prometheus alerting rules
-- [ ] Set up UptimeRobot external monitoring
-
-## Security
-
-- [x] Implement admin cookie-session auth (HttpOnly, SameSite=Strict, Secure when TLS)
-- [x] Implement server-side session expiration
-- [ ] Enforce HTTPS/WSS at edge and origin
-- [x] Implement rate limiting for chat messages with Redis windowing
-
-## Data & Storage
-
-- [x] Set up PostgreSQL/SQLite for chat history and moderation data
-- [x] Set up Redis with append-only persistence + maxmemory policy
-- [ ] Implement daily/weekly/monthly Postgres backups
-- [ ] Implement backup integrity verification
-- [ ] Document and rehearse recovery runbook
-
-## Scaling & Performance
-
-- [ ] Optimize segment immutability and manifest TTL for cache efficiency
-- [ ] Measure playback TTFB from multiple US regions
-- [ ] Load-test WebSocket connections
-- [ ] Benchmark admin API p95 latency
-- [ ] Track cache hit ratio and R2 ops in production
-- [ ] Capacity trigger monitoring
-- [ ] Plan Phase 2 service separation
-- [ ] Plan Phase 3 multi-origin resilience
-
-## Documentation
-
-- [x] Document API contracts in API.md
-- [ ] Document deployment runbook in DEPLOYMENT.md
-- [ ] Document monitoring setup in MONITORING.md
-- [ ] Document security model in SECURITY.md
-- [ ] Create incident runbook
-- [ ] Write Level 2/3 building block views

.gitignore

@@ -81,6 +81,7 @@ Thumbs.db
 # Agent / Tool files
 .github/instructions/TODO.md
 .github/instructions/DONE.md
+.github/instructions/BACKLOG.md
 # (keep AGENTS.md, MEMORY.md, and other instruction files)
 
 # Temporary architecture artifacts

API.md

@@ -272,11 +272,36 @@ Prometheus exposition includes connected clients, stored messages, moderation ac
 
 ## HTTP Status Conventions
 
-- 200/201: successful request
-- 400: malformed payload, invalid input, missing required fields
-- 401: unauthorized admin request
-- 404: not found
-- 405: method not allowed
-- 409: logical conflict (for example, mismatched resumed upload)
+| Code | Meaning               | When                                                        |
+| ---- | --------------------- | ----------------------------------------------------------- |
+| 200  | OK                    | Successful GET, PUT, or action                              |
+| 201  | Created               | Successful POST (upload init, override created)             |
+| 204  | No Content            | Successful DELETE, CORS preflight                           |
+| 400  | Bad Request           | Malformed JSON, missing required fields, invalid input      |
+| 401  | Unauthorized          | Missing or expired admin session                            |
+| 403  | Forbidden             | Valid session but insufficient permissions                  |
+| 404  | Not Found             | Resource doesn't exist                                      |
+| 405  | Method Not Allowed    | Wrong HTTP method for endpoint                              |
+| 409  | Conflict              | Schedule override overlap, upload ID mismatch               |
+| 413  | Payload Too Large     | Upload exceeds 2 GB limit                                   |
+| 422  | Unprocessable         | File fails normalization check (codec, fps, resolution)     |
+| 429  | Too Many Requests     | Rate limit exceeded (chat messages, WS connections, login)  |
+| 500  | Internal Server Error | Unexpected server failure                                   |
+| 503  | Service Unavailable   | Redis unavailable for user actions, playout health degraded |
+
+## Rate Limiting
+
+All limits are per-IP (from `X-Forwarded-For` or `RemoteAddr`) unless noted.
+
+| Resource              | Limit                              | Window                                   | Response                                               |
+| --------------------- | ---------------------------------- | ---------------------------------------- | ------------------------------------------------------ |
+| Chat messages         | `CHAT_RATE_LIMIT_MAX` (default 10) | `CHAT_RATE_LIMIT_WINDOW_S` (default 10s) | 429 + `{"type":"system","text":"Rate limit exceeded"}` |
+| WebSocket connections | 5                                  | 60s                                      | 429 + `"too many connections from this IP"`            |
+| Admin login           | 5                                  | 5min                                     | 429 + `"too many login attempts, try again later"`     |
+
+Rate limit headers on applicable responses:
+
+- `X-RateLimit-Limit`: maximum requests per window
+- `Retry-After`: seconds until window resets
 - 413: payload too large
 - 500: internal server error

CHANGELOG.md

@@ -2,6 +2,39 @@
 
 <!-- markdownlint-disable MD024 -->
 
+## [1.0.9] - 2026-08-06
+
+### Added
+
+- **Prometheus alerts mount**: `alerts.yml` mounted into prometheus container in both dev and prod compose files.
+- **Grafana provisioning**: Prometheus datasource + dashboard provider YAML configs in `monitoring/grafana/`.
+
+### Changed
+
+- **Prod prometheus**: Switched from empty named volume to bind mounts for config files (consistent with dev).
+- **CD pipeline**: Trigger on `push` (parallel with CI), removed `environment: production` (was causing silent abort).
+- **CI pipeline**: Removed redundant `release` job (handled by `release.yml`).
+- **Tag format**: Standardized on `*.*.*` (e.g. `1.0.9`), removed `v` prefix from existing tags.
+- **DEPLOYMENT.md**: Documented parallel CI/CD pipeline flow with workflow table.
+
+### Fixed
+
+- **Prometheus config mount**: Switched prod from bind mounts to named volume `prometheus-config`, populated via `docker run` on remote host. Fixes `not a directory` error when bind source path doesn't exist on deployment host.
+
+### Removed
+
+- `.github/MEMORY.md`, `.github/agents/`, `.github/instructions/` — tracking files migrated out of repo.
+
+## [1.0.8] - 2026-08-06
+
+### Changed
+
+- **API.md**: Added full HTTP status code matrix (table with 12 codes) and rate limit headers documentation.
+- **DEPLOYMENT.md**: Added troubleshooting table (8 common symptoms with check/fix).
+- **MONITORING.md**: Full rewrite with architecture diagram, 7 scrape targets, 8 alert rules, key metrics, troubleshooting.
+- **SECURITY.md**: Full rewrite with threat model diagram, TLS/HSTS config, network isolation, secrets management, incident response.
+- **Architecture docs**: Filled Level 2 building blocks (Mermaid diagram, 5 subsystems with code locations) and Level 3 module breakdowns (chat: 9 modules, playout: 6 modules, frontend: 10 modules).
+
 ## [1.0.7] - 2026-08-06
 
 ### Added
@@ -129,7 +162,7 @@ Full backend and frontend implementation complete.
 - **Frontend** (`frontend/`): Vite + TypeScript + Tailwind + Alpine.js, HLS player with hls.js, chat sidebar with room switching and private messaging, "Now Playing" overlay, and modern UI (ADR-002, ADR-003).
 - Complete `docker-compose.yml` for local development with all services, volumes, and healthchecks.
 - All ADRs (001-007) created and linked in documentation.
-- `TODO.md`, `DONE.md`, and `CHANGELOG.md` updated with full traceability.
+- `CHANGELOG.md` updated with full traceability.
 - Git repository initialized with comprehensive initial commit.
 
 The project is a fully functional, self-hosted 24/7 linear video channel with synchronized playback, custom chat, R2 delivery, moderation, and admin-ready structure. It matches the architecture defined in the arc42 template and all ADRs.

CONTRIBUTING.md

@@ -7,9 +7,8 @@ Thank you for considering contributing to Linear Void!
 1. **Read the Architecture First**: All design decisions are documented in `docs/architecture/README.md` (arc42 template) and the ADRs in `docs/architecture/adr/` (ADR-001 to ADR-007).
 2. **Follow the Structure**: Use the folder layout defined in ADR-002 (`frontend/`, `backend/playout/`, `backend/chat/`, `infrastructure/`, etc.).
 3. **Update Documentation Before Code**: Create or update an ADR before implementing significant changes.
-4. **Update TODO.md**: Plan tasks in `.github/instructions/TODO.md`, mark as completed, and move finished sections to `DONE.md`.
-5. **Commit Messages**: Reference the relevant ADR (e.g., "feat(playout): add R2 sync per ADR-001").
-6. **Test with Docker**: Run `docker compose up --build` to test changes.
+4. **Commit Messages**: Reference the relevant ADR (e.g., "feat(playout): add R2 sync per ADR-001").
+5. **Test with Docker**: Run `docker compose up --build` to test changes.
 
 ## Repository
 
@@ -40,23 +39,22 @@ Linear Void follows **Semantic Versioning** ([SemVer](https://semver.org)): `MAJ
 - **MINOR** — new features (endpoints, admin tabs, UI capabilities) that are backward-compatible.
 - **PATCH** — bug fixes, optimizations, documentation polish.
 
-Git tags use a `v` prefix: `1.0.2`, `1.1.0`, `v2.0.0`.
+Git tags use no prefix: `1.0.2`, `1.1.0`, `2.0.0`.
 
 ### Where Versions Are Declared
 
-| File                           | Token          | Example                   |
-| ------------------------------ | -------------- | ------------------------- |
-| `frontend/package.json`        | `"version"`    | `"1.0.2"`                 |
-| `CHANGELOG.md`                 | `## [X.Y.Z]`   | `## [1.0.2] - 2026-08-05` |
-| `README.md`                    | Version badge  | `version-1.0.2-blue`      |
-| `.github/instructions/TODO.md` | Section header | `## Version 1.0.2`        |
+| File                    | Token         | Example                   |
+| ----------------------- | ------------- | ------------------------- |
+| `frontend/package.json` | `"version"`   | `"1.0.2"`                 |
+| `CHANGELOG.md`          | `## [X.Y.Z]`  | `## [1.0.2] - 2026-08-05` |
+| `README.md`             | Version badge | `version-1.0.2-blue`      |
 
 ### Tag Triggers
 
-| Tag Pattern         | Workflow            | What Happens                                                                                                       |
-| ------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------ |
-| `v*` (e.g. `1.0.2`) | `release.yml`       | Builds all 3 images, pushes to registry with semver tags (`1.0.2`, `1.0`, `latest`), creates Gitea Release via API |
-| Push to `main`      | `ci.yml` → `cd.yml` | CI builds + pushes images with `main`/`latest` tags, CD deploys to production                                      |
+| Tag Pattern            | Workflow            | What Happens                                                                                                       |
+| ---------------------- | ------------------- | ------------------------------------------------------------------------------------------------------------------ |
+| `*.*.*` (e.g. `1.0.2`) | `release.yml`       | Builds all 3 images, pushes to registry with semver tags (`1.0.2`, `1.0`, `latest`), creates Gitea Release via API |
+| Push to `main`         | `ci.yml` → `cd.yml` | CI builds + pushes images with `main`/`latest` tags, CD deploys to production                                      |
 
 ### Full Release Procedure
 
@@ -87,7 +85,7 @@ Add entries under `## [X.Y.Z]` in `CHANGELOG.md` following [Keep a Changelog](ht
 
 ```bash
 # Stage all version bumps + changelog
-git add frontend/package.json CHANGELOG.md README.md .github/instructions/TODO.md
+git add frontend/package.json CHANGELOG.md README.md
 
 # Commit
 git commit -m "chore: bump version to X.Y.Z"
@@ -145,6 +143,6 @@ git.allucanget.biz/zwitschi/linearvoid/playout:latest
 
 ### Moving Completed Tasks
 
-After release, move completed sections from `TODO.md` to `DONE.md` and update `CHANGELOG.md` if needed. The `DONE.md` file serves as the historical record linking tasks to releases.
+After release, update `CHANGELOG.md` if needed.
 
 See the ADRs for architectural decisions and `docs/architecture/README.md` for the big picture. Thank you for contributing!

DEPLOYMENT.md

@@ -1,5 +1,33 @@
 # Deployment Guide
 
+## Table of Contents
+
+- [Deployment Guide](#deployment-guide)
+  - [Table of Contents](#table-of-contents)
+  - [Environments](#environments)
+  - [Production Target (Hetzner)](#production-target-hetzner)
+    - [Planned Infrastructure](#planned-infrastructure)
+    - [Cutover Plan](#cutover-plan)
+    - [Firewall / Security Group](#firewall--security-group)
+    - [Pipeline Readiness](#pipeline-readiness)
+  - [Target Topology](#target-topology)
+  - [Prerequisites](#prerequisites)
+  - [Environment File](#environment-file)
+  - [Pull-and-Run Deployment](#pull-and-run-deployment)
+  - [First-Time or Recovery Provisioning](#first-time-or-recovery-provisioning)
+  - [Health Verification](#health-verification)
+  - [Zero/Low-Downtime Update Pattern](#zerolow-downtime-update-pattern)
+  - [Rollback](#rollback)
+  - [Operational Tasks](#operational-tasks)
+    - [View logs](#view-logs)
+    - [Restart one service](#restart-one-service)
+    - [Stop stack](#stop-stack)
+    - [Keep volumes (recommended) vs remove volumes (destructive)](#keep-volumes-recommended-vs-remove-volumes-destructive)
+  - [Nginx and Proxy Considerations](#nginx-and-proxy-considerations)
+  - [Security Baseline](#security-baseline)
+  - [Common Deployment Pitfalls](#common-deployment-pitfalls)
+  - [Troubleshooting](#troubleshooting)
+
 ## Environments
 
 Linear Void runs in two separate environments:
@@ -9,7 +37,7 @@ Linear Void runs in two separate environments:
 | **Test Lab**   | `192.168.88.40` (internal) | `tcp://192.168.88.40:2375` | Integration testing, staging, CI/CD validation |
 | **Production** | Hetzner VPS (TBD)          | `tcp://<hetzner-ip>:2375`  | Public linearvoid.tv serving real viewers      |
 
-**Current state (2026-08-06):** Test lab is active and receives automated CD deploys. Production Hetzner VPS is **not yet provisioned**. The CD workflow (`cd.yml`) and `docker-compose.prod.yml` currently target the test lab. Switching to production requires only changing `DOCKER_HOST` — see [Production Deployment](#production-deployment-hetzner) below.
+**Current state (2026-08-06):** Test lab is active and receives automated CD deploys. Production Hetzner VPS is **not yet provisioned**. The CD workflow (`cd.yml`) and `docker-compose.prod.yml` currently target the test lab. Switching to production requires only changing `DOCKER_HOST` — see [Production Deployment](#production-target-hetzner) below.
 
 ## Production Target (Hetzner)
 
@@ -28,7 +56,7 @@ Linear Void runs in two separate environments:
 
 ### Cutover Plan
 
-1. Provision Hetzner VPS (see `TODO.md` #92)
+1. Provision Hetzner VPS (see [Production Deployment](#production-target-hetzner) section)
 2. Add production secrets to Gitea (`CI_PROD_*` prefix)
 3. Update `DOCKER_HOST` in CD workflow to Hetzner public IP
 4. Trigger deploy → validate health checks
@@ -49,7 +77,17 @@ Linear Void runs in two separate environments:
 
 ### Pipeline Readiness
 
-The Gitea Actions CD workflow (`cd.yml`) already supports `DOCKER_HOST` as a configurable variable. To switch targets:
+Three Gitea Actions workflows drive the full pipeline:
+
+| Workflow      | Trigger          | Purpose                                                                                  |
+| ------------- | ---------------- | ---------------------------------------------------------------------------------------- |
+| `ci.yml`      | Push to `main`   | Lint, build frontend, vet Go, build & push Docker images (`:latest`, `:main`)            |
+| `cd.yml`      | Push to `main`   | Deploy to test lab (`192.168.88.40:2375`) — runs **in parallel** with CI                 |
+| `release.yml` | Tag push `*.*.*` | Build & push versioned Docker images (`:X.Y.Z`, `:X.Y`, `:latest`), create Gitea Release |
+
+CI and CD run simultaneously on every push to `main`. The deploy pulls whatever `:latest` images are available in the registry at that moment.
+
+The CD workflow (`cd.yml`) uses `DOCKER_HOST` to target the deployment host. To switch targets:
 
 ```bash
 # Test lab (current)
@@ -59,7 +97,7 @@ DOCKER_HOST=tcp://192.168.88.40:2375
 DOCKER_HOST=tcp://<hetzner-public-ip>:2375
 ```
 
-No code changes needed in the pipeline. See `TODO.md` #91 and #92–#99 for full production deployment checklist.
+No code changes needed in the pipeline. See `DEPLOYMENT.md` Production Deployment section for full checklist.
 
 ## Target Topology
 
@@ -99,7 +137,7 @@ See .env.example and SETUP.md for full variable reference.
 
 Images are built and pushed automatically by the CI and release workflows (see [CONTRIBUTING.md](CONTRIBUTING.md#versioning-and-releases) for the release procedure).
 
-Every push to `main` produces `:main` and `:latest` tags. Semantic version tags (`v*`) produce `:X.Y.Z`, `:X.Y`, and `:latest` tags.
+Every push to `main` triggers CI (builds `:main` and `:latest` tags) and CD (deploys `:latest` to test lab) in parallel. Semantic version tags (`*.*.*`, e.g. `1.0.9`) trigger the release workflow which builds `:X.Y.Z`, `:X.Y`, and `:latest` tags and creates a Gitea Release.
 
 From repository root:
 
@@ -228,5 +266,19 @@ docker compose -f docker-compose.prod.yml down -v
 - Missing frontend build artifacts in local-only compose workflows.
 - Wrong registry credentials or missing login.
 - Misaligned domain env vars (app/ws/hls) causing mixed-origin errors.
+
+## Troubleshooting
+
+| Symptom                    | Check                                         | Fix                                                           |
+| -------------------------- | --------------------------------------------- | ------------------------------------------------------------- |
+| Service not starting       | `docker logs linearvoid-<svc> --tail 50`      | Check `.env` vars, volume mounts, port conflicts              |
+| Playout stall / slate mode | `curl http://<host>:8081/health` — mode=slate | Check playlist file, media paths, disk space                  |
+| Chat not connecting        | `curl http://<host>:3000/health`              | Check Redis: `docker exec linearvoid-redis redis-cli ping`    |
+| Admin 401 loop             | Cookie not sent                               | Verify `X-Forwarded-Proto: https`, nginx auth_request headers |
+| Upload fails (413)         | File too large                                | Check nginx `client_max_body_size` (should be 64m+)           |
+| HLS not playing            | `curl http://<host>/hls/playlist.m3u8`        | Check playout health, HLS dir permissions                     |
+| Prometheus no targets      | `curl http://<host>:9090/targets`             | Verify container network, scrape config                       |
+| Grafana no data            | Check datasource                              | Verify Prometheus URL `http://prometheus:9090` provisioned    |
+
 - Proxy-layer body size limits returning 413 on upload.
 - Health checks failing before service warmup is complete.

MONITORING.md

@@ -1,122 +1,76 @@
 # Monitoring Guide
 
-This guide covers Prometheus and Grafana setup for Linear Void.
+Prometheus + Grafana monitoring for Linear Void.
 
-## Current Monitoring Components
+## Architecture
 
-- Prometheus container: linearvoid-prometheus
-- Grafana container: linearvoid-grafana
-- Metrics target: chat service /metrics
-- Base scrape config: monitoring/prometheus/prometheus.yml
-
-## Prometheus Configuration
-
-Current scrape config:
-
-- Job chat -> chat:3000
-- Job prometheus -> prometheus:9090
-- Global scrape interval: 15s
-
-File:
-
-- monitoring/prometheus/prometheus.yml
-
-## Start Monitoring Stack
-
-With compose stack up:
-
-- Prometheus UI: <http://localhost:9090>
-- Grafana UI: <http://localhost:3001>
-
-Compose services include prometheus and grafana in both docker-compose.yml and docker-compose.prod.yml.
-
-## Grafana Provisioning
-
-Current mount paths:
-
-- Dashboards directory: monitoring/grafana/dashboards
-- Datasources directory: monitoring/grafana/datasources
-
-Notes:
-
-- The dashboards directory includes a Prometheus overview dashboard file.
-- Datasource provisioning can be done with a YAML file in monitoring/grafana/datasources, or by configuring a datasource in Grafana UI.
-
-Example datasource provisioning file (create if needed):
-
-```yaml
-apiVersion: 1
-
-datasources:
-  - name: Prometheus
-    type: prometheus
-    access: proxy
-  url: http://prometheus:9090
-    isDefault: true
+```text
+┌──────────────┐     ┌──────────────┐    ┌───────────┐
+│  Exporters   │───> │  Prometheus  │───>│  Grafana  │
+│  (7 targets) │     │  :9090       │    │  :3001    │
+└──────────────┘     └───────┬──────┘    └───────────┘
+                             │
+                   alerts.yml (8 rules)
 ```
 
-## Dashboard References
+## Scrape Targets
 
-- Provisioned dashboard: [monitoring/grafana/dashboards/dashboard.json](monitoring/grafana/dashboards/dashboard.json)
-- Root dashboard snapshot: [monitoring/grafana/dashboard.json](monitoring/grafana/dashboard.json)
+| Job        | Target            | Port | Metrics                                                       |
+| ---------- | ----------------- | ---- | ------------------------------------------------------------- |
+| chat       | chat              | 3000 | WebSocket clients, messages, DMs, moderation, Redis, sessions |
+| playout    | playout           | 8081 | Restarts, mode (live/slate)                                   |
+| nginx      | nginx-exporter    | 9113 | Requests, connections                                         |
+| redis      | redis-exporter    | 9121 | Memory, clients, commands                                     |
+| postgres   | postgres-exporter | 9187 | Connections, queries, locks                                   |
+| node       | node-exporter     | 9100 | CPU, memory, disk, network                                    |
+| prometheus | prometheus        | 9090 | Self-monitoring                                               |
 
-Grafana screenshot:
+Config: `monitoring/prometheus/prometheus.yml` (15s scrape interval)
 
-![Grafana dashboard screenshot](docs/img/grafana-screenshot.png)
+## Alerting Rules
 
-## Key Metrics to Track
+8 rules in `monitoring/prometheus/alerts.yml`:
 
-### Chat/Application
+| Alert                | Severity | Expression                             |
+| -------------------- | -------- | -------------------------------------- |
+| ChatDown             | critical | `up{job="chat"} == 0` for 2m           |
+| PlayoutDown          | critical | `up{job="playout"} == 0` for 2m        |
+| RedisDown            | warning  | `linearvoid_chat_redis_up == 0` for 2m |
+| PlayoutSlateActive   | warning  | `linearvoid_playout_mode == 1` for 1m  |
+| HighChatLatency      | warning  | message rate <0.01/s for 5m            |
+| PrometheusWALFailing | warning  | WAL write failures >0 for 5m           |
+| HighCPU              | warning  | CPU >80% for 10m                       |
+| LowDiskSpace         | warning  | Disk <10% free for 5m                  |
 
-- linearvoid_chat_connected_clients
-- linearvoid_chat_stored_messages_total
-- linearvoid_chat_moderation_actions_total
-- linearvoid_chat_redis_enabled
-- linearvoid_chat_redis_up
-- linearvoid_chat_active_admin_sessions
+## Access
 
-### Prometheus Runtime
+- Prometheus: `http://<host>:9090` (targets, graph, alerts)
+- Grafana: `http://<host>:3001` (default: admin/admin, change via `GRAFANA_PASSWORD`)
 
-- up
-- process_cpu_seconds_total
-- process_resident_memory_bytes
-- prometheus_tsdb_head_series
-- prometheus_tsdb_wal_writes_failed_total
-- prometheus_tsdb_compactions_failed_total
+## Dashboards
 
-## Suggested Alerts
+- **Linear Void app**: `monitoring/grafana/dashboard.json` — clients, messages, moderation, Redis, sessions, scrape health
+- **Prometheus ops**: `monitoring/grafana/dashboards/dashboard.json` — TSDB, CPU/memory, WAL, compactions, query latency
 
-Create a Prometheus rules file (for example monitoring/prometheus/alerts.yml) and wire it into Prometheus.
+## Key Metrics
 
-Recommended baseline alerts:
-
-- Chat target down for 2m
-- Prometheus target down for 2m
-- Redis indicator down for 2m
-- TSDB WAL/compaction failure rate above zero
-- Elevated handler/query latency p99
-
-## Validate Monitoring End-to-End
-
-1. Verify /metrics through nginx:
-
-- <https://linearvoid.tv/metrics>
-
-1. Confirm targets in Prometheus:
-
-- Status -> Targets page in Prometheus UI
-
-1. Confirm dashboard data in Grafana:
-
-- Import or open provisioned dashboards
+- `linearvoid_chat_connected_clients` — active WebSocket connections
+- `linearvoid_chat_messages_total` — total stored messages
+- `linearvoid_chat_dms_total` — private messages sent
+- `linearvoid_chat_redis_up` — Redis reachable (0/1)
+- `linearvoid_playout_restarts_total` — FFmpeg restart counter
+- `linearvoid_playout_mode` — 0=live, 1=slate, 2=degraded
+- `linearvoid_chat_admin_sessions_active` — active admin logins
 
 ## Troubleshooting
 
-### No chat metrics in Prometheus
-
-- Check chat container health and logs.
-- Confirm /metrics is reachable from Prometheus container.
-- Verify scrape target is chat:3000 in prometheus.yml.
+| Symptom                 | Check                                                                |
+| ----------------------- | -------------------------------------------------------------------- |
+| No chat metrics         | `curl http://chat:3000/metrics` from Prometheus container            |
+| No exporter metrics     | Verify exporter containers running: `docker ps \| grep exporter`     |
+| Grafana no data         | Datasource URL must be `http://prometheus:9090`                      |
+| Prometheus targets down | Check container network: all must be on `linearvoid-net`             |
+| Alert not firing        | Check Prometheus → Alerts tab, verify `rule_files` in prometheus.yml |
 
 ### Grafana shows no data
 

README.md

@@ -7,14 +7,14 @@
 [![Build Status](https://git.allucanget.biz/zwitschi/linearvoid/actions/workflows/ci.yml/badge.svg)](https://git.allucanget.biz/zwitschi/linearvoid/actions)
 [![License](https://img.shields.io/badge/License-MIT-blue.svg)](LICENSE)
 [![Gitea](https://img.shields.io/badge/Gitea-1.27.1-blue)](https://git.allucanget.biz)
-[![Version](https://img.shields.io/badge/version-1.0.6-blue)](https://git.allucanget.biz/zwitschi/linearvoid/releases)
+[![Version](https://img.shields.io/badge/version-1.0.9-blue)](https://git.allucanget.biz/zwitschi/linearvoid/releases)
 [![Release Notes](https://img.shields.io/badge/release-notes-green)](CHANGELOG.md)
 
 ## Architecture
 
 The architecture (detailed in [`docs/architecture/README.md`](docs/architecture/README.md)) uses:
 
-- Hetzner Ashburn VPS for playout (FFmpeg + MediaMTX/SRS) and chat *(planned production; currently test lab at 192.168.88.40)*.
+- Hetzner Ashburn VPS for playout (FFmpeg + MediaMTX/SRS) and chat _(planned production; currently test lab at 192.168.88.40)_.
 - Cloudflare R2 + CDN with aggressive immutable caching of HLS `.ts` segments ($0 egress).
 - Lightweight static frontend and WebSocket chat backend.
 - Gitea self-hosted CI/CD (Actions) and Container Registry.
@@ -112,7 +112,7 @@ This structure directly reflects the building blocks, runtime flows, deployment
 
 ## Getting Started
 
-> **Note:** The `docker compose up` below targets local development. The test lab (192.168.88.40) receives automated CD deploys. Production Hetzner VPS is not yet provisioned — see `TODO.md`.
+> **Note:** The `docker compose up` below targets local development. The test lab (192.168.88.40) receives automated CD deploys. Production Hetzner VPS is not yet provisioned — see `DEPLOYMENT.md`.
 
 ```bash
 git clone https://git.allucanget.biz/zwitschi/linearvoid.git
@@ -223,11 +223,11 @@ Contributions, issues, and further evolution (LL-HLS, ABR, multi-channel, E2EE)
 Contributions are welcome! See [`CONTRIBUTING.md`](CONTRIBUTING.md) for guidelines. Quick overview:
 
 1. Read the [architecture docs](docs/architecture/README.md) and ADRs first.
-2. Plan tasks in `.github/instructions/TODO.md`.
+2. Code against the defined module structure (frontend, backend/playout, backend/chat, infrastructure, scripts, monitoring).
 3. Open PRs on [Gitea](https://git.allucanget.biz/zwitschi/linearvoid).
 4. CI validates builds and tests automatically.
 
-Since this a linear project, we also rely on a linear development and release model: Only main branch is used for releases, and all features are merged into main after review and CI validation. Releases are tagged with semantic versioning (1.0.3, 1.0.4, etc.) and automatically built/pushed to the container registry.
+Since this a linear project, we also rely on a linear development and release model: Only main branch is used for releases, and all features are merged into main after review and CI validation. Releases are tagged with semantic versioning (1.0.3, 1.0.4, etc.) and automatically built/pushed to the container registry. Version numbers are declared in `frontend/package.json`, `CHANGELOG.md`, and the version badge in this README. For patches, we use even numbers for stable versions (1.0.2, 1.0.4) and odd numbers for pre-release/unstable versions (1.0.3, 1.0.5).
 
 [MIT License](LICENSE)
 

SECURITY.md

@@ -1,127 +1,93 @@
 # Security Guide
 
-This document summarizes security controls and best practices for Linear Void.
+Layered security model for Linear Void.
 
-## Security Model Overview
+## Threat Model
 
-Security is layered across:
+```
+┌──────────┐    ┌──────────┐    ┌──────────┐    ┌──────────┐
+│  Client  │───▶│Cloudflare│───▶│  nginx   │───▶│  Backend │
+│ (public) │    │ WAF+CDN  │    │ proxy    │    │ chat     │
+└──────────┘    └──────────┘    └────┬─────┘    └────┬─────┘
+                                     │                │
+                              ┌──────┴──────┐  ┌─────┴──────┐
+                              │ /health     │  │ /api/admin │
+                              │ /metrics    │  │ /ws (chat)  │
+                              │ /hls/*      │  │ /api/chat/* │
+                              │ (public)    │  │ (auth req)  │
+                              └─────────────┘  └────────────┘
+```
 
-- Cloudflare edge
-- nginx reverse proxy
-- chat/admin backend
-- container and host operations
+## Authentication & Sessions
 
-## Authentication and Sessions
+- Login: `POST /api/admin/login` → cookie `linearvoid_admin_session`
+- Cookie flags: `HttpOnly`, `SameSite=Strict`, `Secure` (over HTTPS)
+- Server-side expiry via `ADMIN_SESSION_TTL_HOURS`
+- Rate limit: 5 attempts/IP/5min via Redis
 
-Admin access is session-cookie based.
+## Transport Security (TLS/HSTS)
 
-- Login endpoint: /api/admin/login
-- Session endpoint: /api/admin/session
-- Logout endpoint: /api/admin/logout
-- Cookie: linearvoid_admin_session
-- Cookie flags: HttpOnly, SameSite=Strict, Secure over HTTPS
-- Session expiry enforced server-side
+- Cloudflare TLS mode: **Full (strict)** — valid origin cert required
+- nginx listens on 80 internally; Cloudflare handles TLS termination
+- HSTS header set by Cloudflare: `max-age=31536000; includeSubDomains; preload`
+- WSS upgrade path: `/ws` through nginx → chat:3000
 
-Best practices:
+## Network Isolation
 
-- Set strong ADMIN_USERNAME and ADMIN_PASSWORD values.
-- Rotate admin credentials periodically.
-- Keep ADMIN_SESSION_TTL_HOURS aligned with operational policy.
-
-## Transport Security
-
-- Enforce HTTPS for app/admin endpoints.
-- Use WSS for chat WebSocket.
-- Keep Cloudflare TLS mode strict and valid at origin.
+- All backend services on internal Docker network (`linearvoid-net`)
+- Only nginx port exposed to host (443/80)
+- Database, Redis, exporters inaccessible from outside
+- `docker-compose.prod.yml` keeps ports internal via `expose:` not `ports:`
 
 ## Reverse Proxy Controls
 
-nginx controls include:
+- `auth_request /api/admin/auth` gate on all `/admin` and `/api/admin` paths
+- Explicit `client_max_body_size 64m` for resumable upload chunks
+- WebSocket upgrade headers: `Upgrade`, `Connection` preserved
+- CORS headers on HLS: `Access-Control-Allow-Origin *`
+- XSS protection: CSP headers served by Cloudflare
 
-- auth_request guard for /admin and /api/admin paths
-- dedicated unauthorized behavior for admin API responses
-- explicit websocket upgrade headers for /ws
-- upload body-size limits for resumable media chunk uploads
+## Rate Limiting
 
-Best practices:
+| Resource          | Limit | Window |
+| ----------------- | ----- | ------ |
+| WebSocket connect | 5/IP  | 60s    |
+| Chat messages     | 10/IP | 10s    |
+| Admin login       | 5/IP  | 5min   |
 
-- Keep proxy body-size and timeout settings aligned across all proxy layers.
-- Avoid exposing backend service ports publicly unless required.
+Implemented via Redis `INCR` + `EXPIRE`.
 
-## Input Validation and Upload Safety
+## Secrets Management
 
-Server-side protections include:
+- `.env` — local dev only, **gitignored**
+- Gitea Actions secrets (`CI_*` prefix) for CI/CD
+- Never commit: passwords, tokens, API keys, R2 credentials
+- Production `.env` provisioned manually on host, never checked in
+- Rotate admin password quarterly
 
-- extension allow-list for media uploads
-- MIME checks
-- file-size ceiling (2GB)
-- resumable chunk bounds and chunk-index validation
-- media path safety checks on delete
+## Data Protection
 
-Best practices:
-
-- Keep upload chunk size bounded.
-- Reject unsupported content types consistently.
-- Monitor 4xx/5xx rates on upload routes.
-
-## Rate Limiting and Abuse Controls
-
-- Chat rate limiting uses Redis INCR + EXPIRE windows.
-- Current baseline: more than 10 messages in 10 seconds is throttled.
-- Moderation actions are logged for auditability.
-
-Best practices:
-
-- Tune thresholds by observed abuse patterns.
-- Keep moderation logs reviewed and retained per policy.
-
-## Infrastructure and Secrets
-
-- Store secrets in .env (not in committed source).
-- Use Gitea Actions secrets for CI/CD credentials and admin credentials.
-- Restrict Docker host control plane exposure.
-- Limit who can read deployment environment files.
-
-## Cloudflare and Edge Security
-
-Recommended baseline:
-
-- WAF enabled
-- Rate limits on sensitive paths
-- Bot protection where appropriate
-- Cache rules scoped carefully to avoid caching private/control-plane content
-
-## Data Protection and Retention
-
-- Persistent data: postgres, redis append-only, chat state, playlist/media metadata
-- Backup and restore drills should be periodic
-- Restore procedures should be tested in isolated environments
-
-## Logging and Monitoring for Security
-
-Track and alert on:
-
-- repeated failed admin login attempts
-- spikes in 401 and 403 responses
-- sudden upload errors (413/400/500 patterns)
-- unusual websocket disconnect or message-rate anomalies
+- PostgreSQL: chat history, moderation logs, media metadata
+- Redis: append-only persistence for presence, rate limits, sessions
+- HLS segments: ephemeral, regenerated by playout
+- Backups: `scripts/backup-db.sh` (daily), `scripts/backup-weekly.sh`, `scripts/verify-backups.sh`
 
 ## Hardening Checklist
 
-- [ ] HTTPS/WSS enforced end-to-end
+- [ ] HTTPS/WSS enforced end-to-end (Cloudflare Full strict)
 - [ ] Strong admin credentials configured
-- [ ] Secrets only in env/secrets stores
+- [ ] Secrets only in env/secrets stores (never committed)
 - [ ] Proxy auth and websocket headers validated
-- [ ] Upload limits and validation active
-- [ ] Rate limiting active and tested
+- [ ] Upload limits and MIME validation active
+- [ ] Rate limiting active (WS, messages, login)
 - [ ] Cloudflare WAF/rate-limit rules active
 - [ ] Backup/restore drills completed
-- [ ] Security-related monitoring alerts routed
+- [ ] Security alerts routed (Prometheus alertmanager)
 
-## Incident Response Baseline
+## Incident Response
 
-1. Detect issue (alerts/logs/user reports).
-2. Triage scope (edge, proxy, app, data plane).
-3. Contain impact (rate limit, credential rotate, route disable).
-4. Recover services and verify health.
-5. Record timeline and corrective actions.
+1. Detect — alerts (ChatDown, RedisDown, HighChatLatency) or user reports
+2. Triage — edge (Cloudflare), proxy (nginx), app (chat/playout), data (Redis/Postgres)
+3. Contain — rate limit, credential rotate, service restart, slate mode activation
+4. Recover — verify health endpoints, check Prometheus targets, validate HLS output
+5. Record — timeline + corrective actions in `docs/incident-runbook.md`

SETUP.md

@@ -55,7 +55,7 @@ flowchart LR
 
 **Test Lab (active):** Internal network `192.168.88.0/24`, host `192.168.88.40`. Receives automated CD deploys from Gitea Actions. Used for integration testing and staging validation.
 
-**Production (pending):** Hetzner Cloud VPS, Ashburn VA. Not yet provisioned. CD workflow and compose files are ready — only `DOCKER_HOST` needs changing. See `TODO.md` Production Deployment section.
+**Production (pending):** Hetzner Cloud VPS, Ashburn VA. Not yet provisioned. CD workflow and compose files are ready — only `DOCKER_HOST` needs changing. See Production Deployment section in [DEPLOYMENT.md](./DEPLOYMENT.md) for cutover plan.
 
 ## Development Prerequisites
 

backend/chat/src/main.go

@@ -32,17 +32,26 @@ var (
 	upgrader    = websocket.Upgrader{
 		CheckOrigin: func(r *http.Request) bool { return true },
 	}
-	clients      = make(map[*websocket.Conn]bool)
-	clientMeta   = make(map[*websocket.Conn]clientInfo)
-	mutex        = sync.Mutex{}
+	clients        = make(map[*websocket.Conn]bool)
+	clientMeta     = make(map[*websocket.Conn]clientInfo)
+	mutex          = sync.Mutex{}
 	authSessions   = map[string]time.Time{}
 	authSessionsMu = sync.Mutex{}
 
 	// Rate limit config (env-overridable)
 	rateLimitMax    = 10
 	rateLimitWindow = 10 * time.Second
+
+	// Instance ID for multi-instance pub/sub deduplication
+	instanceID = fmt.Sprintf("chat-%s", generateShortID())
 )
 
+func generateShortID() string {
+	b := make([]byte, 6)
+	rand.Read(b)
+	return hex.EncodeToString(b)
+}
+
 type clientInfo struct {
 	Username string
 	Room     string
@@ -101,17 +110,17 @@ type Presence struct {
 }
 
 type AdminOverview struct {
-	Viewers           int    `json:"viewers"`
-	StoredMessages    int64  `json:"storedMessages"`
-	ModerationActions int64  `json:"moderationActions"`
-	MediaCount        int64  `json:"mediaCount"`
+	Viewers                   int    `json:"viewers"`
+	StoredMessages            int64  `json:"storedMessages"`
+	ModerationActions         int64  `json:"moderationActions"`
+	MediaCount                int64  `json:"mediaCount"`
 	MediaTotalDurationSeconds int64  `json:"mediaTotalDurationSeconds"`
-	MediaTotalRuntime string `json:"mediaTotalRuntime"`
-	RedisEnabled      bool   `json:"redisEnabled"`
-	RedisClients      int    `json:"redisClients"`
-	RedisMemoryUsed   string `json:"redisMemoryUsed"`
-	DatabaseDriver    string `json:"databaseDriver"`
-	GeneratedAt       string `json:"generatedAt"`
+	MediaTotalRuntime         string `json:"mediaTotalRuntime"`
+	RedisEnabled              bool   `json:"redisEnabled"`
+	RedisClients              int    `json:"redisClients"`
+	RedisMemoryUsed           string `json:"redisMemoryUsed"`
+	DatabaseDriver            string `json:"databaseDriver"`
+	GeneratedAt               string `json:"generatedAt"`
 }
 
 type PlaylistPayload struct {
@@ -136,15 +145,15 @@ type RoomAnalytics struct {
 }
 
 type AdminAnalytics struct {
-	TotalMessages       int64           `json:"totalMessages"`
-	MessagesLast24Hours int64           `json:"messagesLast24Hours"`
-	DistinctRooms       int64           `json:"distinctRooms"`
-	ModerationActions   int64           `json:"moderationActions"`
-	MediaCount          int64           `json:"mediaCount"`
-	MediaTotalDurationSeconds int64     `json:"mediaTotalDurationSeconds"`
-	MediaTotalRuntime   string          `json:"mediaTotalRuntime"`
-	TopRooms            []RoomAnalytics `json:"topRooms"`
-	GeneratedAt         string          `json:"generatedAt"`
+	TotalMessages             int64           `json:"totalMessages"`
+	MessagesLast24Hours       int64           `json:"messagesLast24Hours"`
+	DistinctRooms             int64           `json:"distinctRooms"`
+	ModerationActions         int64           `json:"moderationActions"`
+	MediaCount                int64           `json:"mediaCount"`
+	MediaTotalDurationSeconds int64           `json:"mediaTotalDurationSeconds"`
+	MediaTotalRuntime         string          `json:"mediaTotalRuntime"`
+	TopRooms                  []RoomAnalytics `json:"topRooms"`
+	GeneratedAt               string          `json:"generatedAt"`
 }
 
 type ServiceHealth struct {
@@ -171,9 +180,9 @@ type ActiveUser struct {
 }
 
 type AdminUsersResponse struct {
-	Users       []ActiveUser `json:"users"`
-	RedisEnabled bool       `json:"redisEnabled"`
-	GeneratedAt string      `json:"generatedAt"`
+	Users        []ActiveUser `json:"users"`
+	RedisEnabled bool         `json:"redisEnabled"`
+	GeneratedAt  string       `json:"generatedAt"`
 }
 
 type ChatLogEntry struct {
@@ -205,12 +214,12 @@ type ScheduleEntry struct {
 }
 
 type AdminScheduleResponse struct {
-	Entries               []ScheduleEntry `json:"entries"`
-	TotalEntries          int             `json:"totalEntries"`
-	MatchedEntries        int             `json:"matchedEntries"`
-	TotalDurationSeconds  int             `json:"totalDurationSeconds"`
-	TotalRuntime          string          `json:"totalRuntime"`
-	GeneratedAt           string          `json:"generatedAt"`
+	Entries              []ScheduleEntry `json:"entries"`
+	TotalEntries         int             `json:"totalEntries"`
+	MatchedEntries       int             `json:"matchedEntries"`
+	TotalDurationSeconds int             `json:"totalDurationSeconds"`
+	TotalRuntime         string          `json:"totalRuntime"`
+	GeneratedAt          string          `json:"generatedAt"`
 }
 
 type MediaFileEntry struct {
@@ -234,12 +243,12 @@ type AdminMediaResponse struct {
 }
 
 type AdminMediaUploadInitRequest struct {
-	UploadID   string `json:"uploadId"`
-	FileName   string `json:"fileName"`
-	FileSize   int64  `json:"fileSize"`
-	MimeType   string `json:"mimeType"`
-	ChunkSize  int64  `json:"chunkSize"`
-	TotalChunks int   `json:"totalChunks"`
+	UploadID    string `json:"uploadId"`
+	FileName    string `json:"fileName"`
+	FileSize    int64  `json:"fileSize"`
+	MimeType    string `json:"mimeType"`
+	ChunkSize   int64  `json:"chunkSize"`
+	TotalChunks int    `json:"totalChunks"`
 }
 
 type AdminMediaUploadSession struct {
@@ -254,13 +263,13 @@ type AdminMediaUploadSession struct {
 }
 
 type AdminMediaUploadStatusResponse struct {
-	UploadID      string  `json:"uploadId"`
-	FileName      string  `json:"fileName"`
-	FileSize      int64   `json:"fileSize"`
-	ChunkSize     int64   `json:"chunkSize"`
-	TotalChunks   int     `json:"totalChunks"`
-	UploadedChunks int    `json:"uploadedChunks"`
-	Progress      float64 `json:"progress"`
+	UploadID       string  `json:"uploadId"`
+	FileName       string  `json:"fileName"`
+	FileSize       int64   `json:"fileSize"`
+	ChunkSize      int64   `json:"chunkSize"`
+	TotalChunks    int     `json:"totalChunks"`
+	UploadedChunks int     `json:"uploadedChunks"`
+	Progress       float64 `json:"progress"`
 }
 
 type AdminLoginPayload struct {
@@ -278,9 +287,9 @@ var databaseDriverName = "sqlite"
 const adminSessionCookieName = "linearvoid_admin_session"
 
 const (
-	maxAdminUploadBytes      int64 = 2 * 1024 * 1024 * 1024
-	defaultAdminChunkSize    int64 = 8 * 1024 * 1024
-	maxAdminChunkSize        int64 = 32 * 1024 * 1024
+	maxAdminUploadBytes   int64 = 2 * 1024 * 1024 * 1024
+	defaultAdminChunkSize int64 = 8 * 1024 * 1024
+	maxAdminChunkSize     int64 = 32 * 1024 * 1024
 )
 
 // initRedis initializes Redis for presence, rate limiting, and pub/sub (ADR-007)
@@ -380,11 +389,11 @@ func parseRedisInfoValue(info, key string) string {
 
 func getAdminOverview() AdminOverview {
 	overview := AdminOverview{
-		RedisEnabled:    redisClient != nil,
-		RedisMemoryUsed: "unavailable",
-		DatabaseDriver:  databaseDriverName,
+		RedisEnabled:      redisClient != nil,
+		RedisMemoryUsed:   "unavailable",
+		DatabaseDriver:    databaseDriverName,
 		MediaTotalRuntime: "00:00:00",
-		GeneratedAt:     time.Now().UTC().Format(time.RFC3339),
+		GeneratedAt:       time.Now().UTC().Format(time.RFC3339),
 	}
 
 	mutex.Lock()
@@ -422,6 +431,23 @@ func handleAdminLogin(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Rate limit: max 5 attempts per IP per 5 minutes
+	clientIP := r.Header.Get("X-Forwarded-For")
+	if clientIP == "" {
+		clientIP = r.RemoteAddr
+	}
+	if redisClient != nil {
+		loginKey := fmt.Sprintf("login_attempts:%s", clientIP)
+		count, _ := redisClient.Incr(context.Background(), loginKey).Result()
+		if count == 1 {
+			redisClient.Expire(context.Background(), loginKey, 5*time.Minute)
+		}
+		if count > 5 {
+			http.Error(w, "too many login attempts, try again later", http.StatusTooManyRequests)
+			return
+		}
+	}
+
 	var payload AdminLoginPayload
 	if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
 		http.Error(w, fmt.Sprintf("invalid request body: %v", err), http.StatusBadRequest)
@@ -907,9 +933,9 @@ func handleAdminModerationAction(w http.ResponseWriter, r *http.Request) {
 
 func getAdminAnalytics() AdminAnalytics {
 	analytics := AdminAnalytics{
-		TopRooms:    []RoomAnalytics{},
+		TopRooms:          []RoomAnalytics{},
 		MediaTotalRuntime: "00:00:00",
-		GeneratedAt: time.Now().UTC().Format(time.RFC3339),
+		GeneratedAt:       time.Now().UTC().Format(time.RFC3339),
 	}
 
 	if db == nil {
@@ -1114,7 +1140,7 @@ func handleAdminUsers(w http.ResponseWriter, r *http.Request) {
 
 	case http.MethodPost:
 		var payload struct {
-			Action   string `json:"action"`   // mute, unmute, ban, unban
+			Action   string `json:"action"` // mute, unmute, ban, unban
 			Username string `json:"username"`
 			Duration int    `json:"duration"` // minutes, for mute
 		}
@@ -1309,13 +1335,13 @@ func buildUploadStatus(session AdminMediaUploadSession) AdminMediaUploadStatusRe
 	}
 
 	return AdminMediaUploadStatusResponse{
-		UploadID:      session.UploadID,
-		FileName:      session.FileName,
-		FileSize:      session.FileSize,
-		ChunkSize:     session.ChunkSize,
-		TotalChunks:   session.TotalChunks,
+		UploadID:       session.UploadID,
+		FileName:       session.FileName,
+		FileSize:       session.FileSize,
+		ChunkSize:      session.ChunkSize,
+		TotalChunks:    session.TotalChunks,
 		UploadedChunks: uploadedChunks,
-		Progress:      progress,
+		Progress:       progress,
 	}
 }
 
@@ -2144,7 +2170,7 @@ func handleMetrics(w http.ResponseWriter, r *http.Request) {
 
 	fmt.Fprintln(w, "# HELP linearvoid_chat_build_info Static build info for chat service.")
 	fmt.Fprintln(w, "# TYPE linearvoid_chat_build_info gauge")
-	fmt.Fprintln(w, "linearvoid_chat_build_info{service=\"chat\",database_driver=\""+databaseDriverName+"\"} 1")
+	fmt.Fprintln(w, "linearvoid_chat_build_info{service=\"chat\",database_driver=\""+databaseDriverName+"\",instance_id=\""+instanceID+"\"} 1")
 }
 
 // broadcast sends a message to all connected clients in a room and saves to DB
@@ -2155,16 +2181,46 @@ func broadcast(msg Message) {
 	// Persist to configured SQL backend.
 	db.Create(&msg)
 
+	// Local fan-out
 	for client := range clients {
 		if err := client.WriteJSON(msg); err != nil {
 			client.Close()
 			delete(clients, client)
+			delete(clientMeta, client)
 		}
 	}
+
+	// Redis pub/sub for multi-instance fan-out
+	if redisClient != nil && msg.Room != "" {
+		payload, _ := json.Marshal(map[string]any{
+			"instance": instanceID,
+			"msg":      msg,
+		})
+		redisClient.Publish(context.Background(), fmt.Sprintf("room:%s:messages", msg.Room), payload)
+	}
 }
 
 // handleConnection handles WebSocket connections for chat (public rooms, private DMs, moderation)
 func handleConnection(w http.ResponseWriter, r *http.Request) {
+	// IP-based connection rate limiting: max 5 concurrent per IP
+	clientIP := r.Header.Get("X-Forwarded-For")
+	if clientIP == "" {
+		clientIP = r.RemoteAddr
+	}
+	if redisClient != nil {
+		ipKey := fmt.Sprintf("ws_conn:%s", clientIP)
+		count, _ := redisClient.Incr(context.Background(), ipKey).Result()
+		if count == 1 {
+			redisClient.Expire(context.Background(), ipKey, 60*time.Second)
+		}
+		if count > 5 {
+			http.Error(w, "too many connections from this IP", http.StatusTooManyRequests)
+			return
+		}
+		// Decrement on disconnect
+		defer redisClient.Decr(context.Background(), ipKey)
+	}
+
 	conn, err := upgrader.Upgrade(w, r, nil)
 	if err != nil {
 		log.Println("Upgrade error:", err)
@@ -2297,6 +2353,47 @@ func pushChatHistory(conn *websocket.Conn, room string, limit int) {
 // dmsSent tracks total private messages for Prometheus metrics (set from handleMetrics).
 var dmsSent int64
 
+// subscribeRoomMessages listens for Redis pub/sub messages and fans out locally.
+func subscribeRoomMessages() {
+	if redisClient == nil {
+		return
+	}
+	pubsub := redisClient.PSubscribe(context.Background(), "room:*:messages")
+	defer pubsub.Close()
+
+	log.Printf("Redis pub/sub subscriber started (instance %s)", instanceID)
+
+	for {
+		msg, err := pubsub.ReceiveMessage(context.Background())
+		if err != nil {
+			log.Printf("Redis pub/sub error: %v — retrying in 5s", err)
+			time.Sleep(5 * time.Second)
+			pubsub = redisClient.PSubscribe(context.Background(), "room:*:messages")
+			continue
+		}
+
+		var envelope struct {
+			Instance string  `json:"instance"`
+			Msg      Message `json:"msg"`
+		}
+		if err := json.Unmarshal([]byte(msg.Payload), &envelope); err != nil {
+			continue
+		}
+
+		// Skip messages originating from this instance
+		if envelope.Instance == instanceID {
+			continue
+		}
+
+		// Fan out to local clients
+		mutex.Lock()
+		for client := range clients {
+			client.WriteJSON(envelope.Msg)
+		}
+		mutex.Unlock()
+	}
+}
+
 // handlePrivateDM routes private messages to the specific user pair
 func handlePrivateDM(msg Message) {
 	msg.Type = "private"
@@ -2604,6 +2701,11 @@ func main() {
 	initRedis()
 	initDB()
 
+	// Subscribe to Redis pub/sub for multi-instance message fan-out
+	go subscribeRoomMessages()
+
+	log.Printf("Instance ID: %s", instanceID)
+
 	http.HandleFunc("/ws", handleConnection)
 	http.HandleFunc("/api/admin/login", handleAdminLogin)
 	http.HandleFunc("/api/admin/logout", handleAdminLogout)

docker-compose.prod.yml

@@ -1,6 +1,6 @@
 # docker-compose.prod.yml — TEST LAB DEPLOYMENT
 # Target: test lab host 192.168.88.40 (NOT Hetzner production).
-# Production Hetzner VPS not yet provisioned — see TODO.md Production Deployment section.
+# Production Hetzner VPS not yet provisioned — see DEPLOYMENT.md Production Deployment section.
 # Uses pre-built images from Gitea Container Registry (no local build).
 # Override DOCKER_HOST in CD workflow to switch target.
 
@@ -104,6 +104,56 @@ services:
       timeout: 5s
       retries: 5
 
+  nginx-exporter:
+    image: nginx/nginx-prometheus-exporter:latest
+    container_name: linearvoid-nginx-exporter
+    restart: unless-stopped
+    command:
+      - "-nginx.scrape-uri=http://nginx/nginx-status"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - nginx
+
+  redis-exporter:
+    image: oliver006/redis_exporter:latest
+    container_name: linearvoid-redis-exporter
+    restart: unless-stopped
+    command:
+      - "-redis.addr=redis://redis:6379"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - redis
+
+  postgres-exporter:
+    image: prometheuscommunity/postgres-exporter:latest
+    container_name: linearvoid-postgres-exporter
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: "postgresql://linearvoid:${POSTGRES_PASSWORD:-linearvoid}@db:5432/linearvoid?sslmode=disable"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - db
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: linearvoid-node-exporter
+    restart: unless-stopped
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    networks:
+      - linearvoid-net
+
   prometheus:
     image: prom/prometheus:latest
     container_name: linearvoid-prometheus
@@ -148,6 +198,6 @@ networks:
 volumes:
   redis-data:
   postgres-data:
-  prometheus-data:
   prometheus-config:
+  prometheus-data:
   grafana-data:

docker-compose.yml

@@ -97,12 +97,63 @@ services:
     networks:
       - linearvoid-net
 
+  nginx-exporter:
+    image: nginx/nginx-prometheus-exporter:latest
+    container_name: linearvoid-nginx-exporter
+    restart: unless-stopped
+    command:
+      - "-nginx.scrape-uri=http://nginx/nginx-status"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - nginx
+
+  redis-exporter:
+    image: oliver006/redis_exporter:latest
+    container_name: linearvoid-redis-exporter
+    restart: unless-stopped
+    command:
+      - "-redis.addr=redis://redis:6379"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - redis
+
+  postgres-exporter:
+    image: prometheuscommunity/postgres-exporter:latest
+    container_name: linearvoid-postgres-exporter
+    restart: unless-stopped
+    environment:
+      DATA_SOURCE_NAME: "postgresql://linearvoid:${POSTGRES_PASSWORD:-linearvoid}@db:5432/linearvoid?sslmode=disable"
+    networks:
+      - linearvoid-net
+    depends_on:
+      - db
+
+  node-exporter:
+    image: prom/node-exporter:latest
+    container_name: linearvoid-node-exporter
+    restart: unless-stopped
+    pid: host
+    volumes:
+      - /proc:/host/proc:ro
+      - /sys:/host/sys:ro
+      - /:/rootfs:ro
+    command:
+      - "--path.procfs=/host/proc"
+      - "--path.sysfs=/host/sys"
+      - "--path.rootfs=/rootfs"
+      - "--collector.filesystem.mount-points-exclude=^/(sys|proc|dev|host|etc)($$|/)"
+    networks:
+      - linearvoid-net
+
   prometheus:
     image: prom/prometheus:latest
     container_name: linearvoid-prometheus
     restart: unless-stopped
     volumes:
       - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
+      - ./monitoring/prometheus/alerts.yml:/etc/prometheus/alerts.yml
       - prometheus-data:/prometheus
     ports:
       - "9090:9090"

docs/architecture/README.md

@@ -20,7 +20,14 @@
   - [Building Block View](#building-block-view)
     - [Whitebox Overall System](#whitebox-overall-system)
     - [Level 2](#level-2)
+      - [Playout Service (`backend/playout/src/playlist.go`)](#playout-service-backendplayoutsrcplaylistgo)
+      - [Chat Service (`backend/chat/src/main.go`)](#chat-service-backendchatsrcmaingo)
+      - [Frontend SPA (`frontend/`)](#frontend-spa-frontend)
+      - [Reverse Proxy (`infrastructure/nginx/nginx.conf`)](#reverse-proxy-infrastructurenginxnginxconf)
     - [Level 3](#level-3)
+      - [Chat Service Modules](#chat-service-modules)
+      - [Playout Service Modules](#playout-service-modules)
+      - [Frontend Modules](#frontend-modules)
   - [Runtime View](#runtime-view)
     - [Playout \& Delivery Scenario](#playout--delivery-scenario)
   - [Deployment View](#deployment-view)
@@ -259,11 +266,114 @@ Detailed black-box descriptions, level-2/3 views, and code locations are in `adr
 
 ### Level 2
 
-Deferred to implementation (see ADR for component responsibilities, interfaces, quality attributes, and risks). Playout enforces normalization; Delivery layer focuses on cache rules and immutable naming.
+Detailed component diagram with code locations and persistence.
+
+```mermaid
+graph TB
+    subgraph "Cloudflare Edge"
+        CF[CDN + WAF<br/>Cache Rules]
+        R2[R2 Bucket<br/>media + HLS]
+        PAGES[Pages<br/>SPA hosting]
+    end
+
+    subgraph "Docker Host (Hetzner CX22)"
+        NGINX[nginx<br/>:80/:443<br/>proxy + auth]
+        CHAT[chat service<br/>:3000<br/>Go + gorilla/websocket]
+        PLAYOUT[playout service<br/>:8081<br/>Go + FFmpeg]
+        REDIS[Redis 7<br/>:6379<br/>presence/rate/pubsub]
+        DB[PostgreSQL 15<br/>:5432<br/>chat history/moderation]
+    end
+
+    subgraph "External"
+        VIEWER[Viewer Browser<br/>hls.js + WebSocket]
+        ADMIN[Admin Browser<br/>SPA + uploads]
+    end
+
+    VIEWER -->|HTTPS| CF
+    VIEWER -->|WSS| NGINX
+    ADMIN -->|HTTPS| CF
+    CF --> NGINX
+    NGINX --> CHAT
+    NGINX --> PLAYOUT
+    NGINX -->|/hls/*| R2
+    CHAT --> REDIS
+    CHAT --> DB
+    PLAYOUT --> R2
+    PLAYOUT -->|reads| R2
+```
+
+#### Playout Service (`backend/playout/src/playlist.go`)
+
+- **Role**: Continuous HLS encoding loop using FFmpeg concat demuxer
+- **Input**: `data/playlist/playlist.txt` — ordered list of MP4 file paths
+- **Output**: HLS segments (`.ts`) + live manifest (`.m3u8`) written to `data/hls/`, mirrored to R2
+- **Resilience**: Restart counter (>5/60s → activate slate), Prometheus metrics on port 8081
+- **Validation**: ffprobe gate on all ingested files (codec, fps, resolution, keyframe interval)
+
+#### Chat Service (`backend/chat/src/main.go`)
+
+- **Role**: WebSocket chat with rooms, DMs, moderation, admin API
+- **Persistence**: PostgreSQL 15 via GORM (messages, moderation logs, media metadata). SQLite for dev fallback
+- **Redis**: Presence tracking, rate limiting, pub/sub for multi-instance fan-out
+- **Auth**: Session-cookie admin auth with Redis-backed expiry
+- **Metrics**: Connected clients, stored messages, DMs, moderation actions on `/metrics`
+
+#### Frontend SPA (`frontend/`)
+
+- **Viewer**: `src/main.ts` — hls.js player, WebSocket chat, presence indicators, schedule overlay
+- **Admin**: `src/admin.ts` — 9 dynamic tabs (dashboard, upload, playlist, schedule, moderation, chat-log, users, analytics, system)
+- **Build**: Vite → static output in `frontend/dist/`
+
+#### Reverse Proxy (`infrastructure/nginx/nginx.conf`)
+
+- auth_request gate on `/admin` and `/api/admin`
+- WebSocket upgrade for `/ws`
+- HLS CORS headers + OPTIONS preflight
+- `/health` JSON endpoint, `/nginx-status` stub_status
 
 ### Level 3
 
-Deferred to implementation (see ADR for component responsibilities, interfaces, quality attributes, and risks). Playout enforces normalization; Delivery layer focuses on cache rules and immutable naming.
+Module-level breakdown per service.
+
+#### Chat Service Modules
+
+| Module                 | File/Lines                                                     | Responsibility                                                                           |
+| ---------------------- | -------------------------------------------------------------- | ---------------------------------------------------------------------------------------- |
+| **Connection Manager** | `main.go` (hub, clients map)                                   | Track active WebSocket connections per room, fan-out messages                            |
+| **Message Parser**     | `main.go` (readPump, handleMessage)                            | Parse JSON envelope `{type, room?, ...}`, validate fields, route to handler              |
+| **Room Registry**      | `main.go` (rooms map, room broadcasts)                         | Map room name → client set, handle join/leave/close                                      |
+| **DM Router**          | `main.go` (handleDirectMessage)                                | Resolve target user in clients map, deliver private message                              |
+| **Persistence Layer**  | `main.go` (GORM models: Message, DirectMessage, ModerationLog) | Store/retrieve chat messages, DMs, moderation events                                     |
+| **Redis Subsystem**    | `main.go` (redisClient, rate limiting, pub/sub, presence)      | INCR+EXPIRE rate limit, SETEX presence, PSUBSCRIBE room fan-out                          |
+| **Auth Handler**       | `main.go` (handleAdminLogin, adminAuthMiddleware)              | Session cookie creation/validation, login rate limiting                                  |
+| **Admin API Router**   | `main.go` (handleAdmin\*)                                      | Upload, playlist CRUD, schedule overrides, moderation actions, analytics, system summary |
+| **Metrics Exporter**   | `main.go` (prometheus.MustRegister)                            | Expose connected_clients, messages_total, dms_total, moderation_actions, redis_up        |
+
+#### Playout Service Modules
+
+| Module               | File/Lines                                     | Responsibility                                                     |
+| -------------------- | ---------------------------------------------- | ------------------------------------------------------------------ |
+| **Playlist Parser**  | `playlist.go` (parsePlaylist)                  | Read `playlist.txt`, skip comments/blank lines, resolve file paths |
+| **FFmpeg Manager**   | `playlist.go` (runPlayout, restart logic)      | Spawn/monitor FFmpeg concat process, restart on crash              |
+| **Slate Controller** | `playlist.go` (activateSlate, deactivateSlate) | Switch output to slate loop on excessive failures                  |
+| **R2 Sync**          | `playlist.go` (syncToR2)                       | Upload HLS segments + manifest to R2 bucket                        |
+| **Health Endpoint**  | `playlist.go` (healthHandler)                  | `/health` → JSON with mode, segment count, uptime                  |
+| **Metrics Exporter** | `playlist.go` (prometheus)                     | Expose restarts_total, mode (live/slate)                           |
+
+#### Frontend Modules
+
+| Module                  | File                                            | Responsibility                                                    |
+| ----------------------- | ----------------------------------------------- | ----------------------------------------------------------------- |
+| **HLS Player**          | `src/main.ts`                                   | hls.js initialization, attach to video element, error recovery    |
+| **WebSocket Client**    | `src/main.ts`                                   | Connect/disconnect, parse envelope, dispatch to UI handlers       |
+| **Chat UI**             | `src/main.ts`                                   | Message list rendering, room tabs, DM panel, presence dots        |
+| **Schedule Overlay**    | `src/main.ts`                                   | Poll `/api/schedule`, render current/next program                 |
+| **Admin Router**        | `src/admin.ts`                                  | Tab-based navigation, dynamic page loading                        |
+| **Upload Manager**      | `src/admin.ts`                                  | Multipart upload with chunk progress, ffprobe validation feedback |
+| **Playlist Editor**     | `src/admin.ts`                                  | Drag-reorder, add/remove items, save to `/api/admin/playlist`     |
+| **Moderation Panel**    | `src/admin.ts` + `public/admin/moderation.html` | Inline mute/ban, action log viewer                                |
+| **Analytics Dashboard** | `src/admin.ts` + `public/admin/analytics.html`  | Message counts, top rooms, moderation stats                       |
+| **System Config**       | `src/admin.ts` + `public/admin/system.html`     | Service health, config metadata, restart buttons                  |
 
 ## Runtime View
 
@@ -350,7 +460,7 @@ flowchart LR
 
 ### Infrastructure Level 2
 
-**Hetzner VPS (planned):** Will run playout (FFmpeg/MediaMTX), chat backend, Nginx. Monitoring via dashboards + UptimeRobot. Backups to R2. Currently test lab (`192.168.88.40`) serves this role. Production Hetzner instance not yet provisioned — see `TODO.md` Production Deployment section.
+**Hetzner VPS (planned):** Will run playout (FFmpeg/MediaMTX), chat backend, Nginx. Monitoring via dashboards + UptimeRobot. Backups to R2. Currently test lab (`192.168.88.40`) serves this role. Production Hetzner instance not yet provisioned — see Production Deployment section in [`DEPLOYMENT.md`](../../DEPLOYMENT.md).
 
 **Cloudflare Layer**:
 

docs/architecture/adr/ADR-004-admin-video.md

@@ -60,8 +60,6 @@ Create a dedicated **admin web interface** in a new `frontend/admin/` directory
 - Integration with existing playout service and R2 keeps the architecture consistent.
 - Admin actions will be fully auditable, paving the way for ADR-005 (moderation and user management).
 
-This decision was planned in `TODO.md` before implementation, per AGENTS.md.
-
 ## Consequences
 
 **Positive**:

docs/architecture/adr/ADR-005-admin-moderation-analytics.md

@@ -59,8 +59,6 @@ Extend the admin interface started in ADR-004 with additional protected sections
 - Leveraging Prometheus/Grafana for analytics avoids duplicating metrics collection.
 - Immutable audit logging supports future compliance and moderation review (key requirement for a public chat system).
 
-This ADR was planned in `TODO.md` before any implementation, per AGENTS.md.
-
 ## Consequences
 
 **Positive**:
@@ -92,8 +90,6 @@ This ADR was planned in `TODO.md` before any implementation, per AGENTS.md.
 7. Update authentication middleware and tests.
 8. Document API contracts and update `docker-compose.yml` to expose the admin interface securely.
 
-**Next Steps**: After both admin ADRs are complete, move to implementation of the Playout Service (highest priority in the core modules section of TODO.md).
-
 ## References
 
 - ADR-001: Chat features, moderation requirements, monitoring needs.

docs/architecture/adr/ADR-007-redis-usage.md

@@ -95,8 +95,6 @@ We must explicitly document:
 6. Add monitoring (Prometheus exporter) and admin audit logging for configuration changes.
 7. Write tests for rate limiting, presence, and admin configuration flows.
 
-**Next Steps**: After ADR-006 and ADR-007 are reviewed, mark the "Create ADRs for Data Storage, Caching, and Admin Capabilities" section as completed in TODO.md and move it to DONE.md. Then proceed with implementation of the Playout Service.
-
 ## References
 
 - ADR-001: Chat features requiring presence, rate limiting, and pub/sub.

docs/incident-runbook.md

@@ -0,0 +1,164 @@
+# Incident Runbook
+
+Triage order for Linear Void incidents. Work top-to-bottom — each tier depends on the one above.
+
+## Triage Order
+
+1. Edge Reachability (Cloudflare)
+2. Origin Health (VPS/Docker)
+3. Playout Progression (HLS)
+4. Chat Connectivity (WebSocket)
+5. Data-Plane (Redis/DB)
+
+## 1. Edge Reachability (Cloudflare)
+
+**Symptoms:** `linearvoid.tv` not resolving, SSL errors, 5xx from Cloudflare.
+
+```bash
+# Check DNS
+dig linearvoid.tv
+dig www.linearvoid.tv
+
+# Check Cloudflare status
+curl -I https://linearvoid.tv/health
+```
+
+**Common causes:**
+
+- DNS misconfiguration (A record pointing to wrong IP)
+- Cloudflare proxy disabled (grey cloud → origin exposed directly)
+- SSL certificate expired (check Cloudflare dashboard → SSL/TLS)
+- Cloudflare outage → check [https://www.cloudflarestatus.com](https://www.cloudflarestatus.com)
+
+**Mitigation:**
+
+- Verify A record: `linearvoid.tv` → origin IP (proxied, orange cloud)
+- Bypass Cloudflare temporarily: `curl -H "Host: linearvoid.tv" http://<origin-ip>/health`
+
+## 2. Origin Health (VPS/Docker)
+
+**Symptoms:** Cloudflare returns 502/504, containers down.
+
+```bash
+# Remote Docker (test lab)
+docker -H tcp://192.168.88.40:2375 ps
+docker -H tcp://192.168.88.40:2375 logs linearvoid-nginx --tail 50
+
+# Direct health check
+curl http://192.168.88.40/health
+```
+
+**Common causes:**
+
+- Host out of disk space: `df -h`
+- Docker daemon stopped: `systemctl status docker`
+- Container restart loop: `docker ps -a --filter "status=exited"`
+- Resource exhaustion (OOM): `dmesg | grep -i oom`
+
+**Mitigation:**
+
+- Restart stack: `docker compose -f docker-compose.prod.yml up -d`
+- Free disk: `docker system prune -f`
+- Check logs per service: `docker logs linearvoid-<service> --tail 100`
+
+## 3. Playout Progression (HLS)
+
+**Symptoms:** Viewer sees slate or buffering, playlist not advancing.
+
+```bash
+# Check playout health
+curl http://192.168.88.40:8081/health
+# Response includes "mode": "live" or "slate"
+
+# Check HLS output
+curl -I http://192.168.88.40/hls/playlist.m3u8
+ls -la data/hls/
+
+# Check playout logs
+docker logs linearvoid-playout --tail 100
+```
+
+**Common causes:**
+
+- FFmpeg crash → slate fallback active (health shows `degraded`)
+- Playlist file empty/missing: `cat data/playlist/playlist.txt`
+- Media file missing from playlist: `ls data/media/`
+- Disk full → HLS segments can't write
+
+**Mitigation:**
+
+- Verify playlist has valid entries
+- Check media files exist at paths in playlist
+- Restart playout: `docker compose restart playout`
+- Generate demo playlist if empty: see `backend/playout/start-playout.sh`
+
+## 4. Chat Connectivity (WebSocket)
+
+**Symptoms:** Chat not loading, messages not sending, presence broken.
+
+```bash
+# Check chat health
+curl http://192.168.88.40:3000/health
+
+# Test WebSocket
+wscat -c ws://192.168.88.40/ws?username=test&room=main
+
+# Check chat logs
+docker logs linearvoid-chat --tail 100
+```
+
+**Common causes:**
+
+- Redis down → presence/rate-limiting degraded (chat still works)
+- Database down → messages not persisting
+- Port not exposed: `docker port linearvoid-chat`
+
+**Mitigation:**
+
+- Restart chat: `docker compose restart chat`
+- Check Redis: `docker exec linearvoid-redis redis-cli ping`
+- Check DB: `docker exec linearvoid-db pg_isready -U linearvoid`
+- Chat works without Redis (graceful degradation) — check logs for "Redis connection warning"
+
+## 5. Data-Plane (Redis/DB)
+
+**Symptoms:** Chat works but no persistence/presence, admin stats empty.
+
+```bash
+# Redis
+docker exec linearvoid-redis redis-cli ping
+docker exec linearvoid-redis redis-cli info memory
+
+# PostgreSQL
+docker exec linearvoid-db pg_isready -U linearvoid
+docker exec linearvoid-db psql -U linearvoid -c "SELECT count(*) FROM messages;"
+```
+
+**Common causes:**
+
+- Redis OOM (check `maxmemory` setting)
+- PostgreSQL connection limit reached
+- Disk full on volume mount
+
+**Mitigation:**
+
+- Redis: `docker compose restart redis` (AOF persist enabled)
+- PostgreSQL: `docker compose restart db`
+- Restore from backup: see `scripts/verify-backups.sh` for procedure
+
+## Escalation
+
+| Severity | When                                 | Action                                             |
+| -------- | ------------------------------------ | -------------------------------------------------- |
+| Critical | Slate active >30min, complete outage | Trigger DR plan, provision fallback host           |
+| Warning  | Redis down, high CPU >30min          | Investigate during business hours, restart service |
+| Info     | Single service blip                  | Monitor, no action unless recurring                |
+
+## Post-Incident
+
+1. Document timeline in incident channel
+2. Identify root cause
+3. Add prevention to docs if actionable
+4. Update this runbook if new failure mode discovered
+
+**Recovery targets:** RPO ≤24h, RTO ≤60min. Quarterly DR drill: simulate host loss, restore from backup.

frontend/package.json

@@ -1,7 +1,7 @@
 {
   "name": "frontend",
   "private": true,
-  "version": "1.0.7",
+  "version": "1.0.9",
   "type": "module",
   "scripts": {
     "dev": "vite",

monitoring/grafana/dashboards/dashboard.yml

@@ -0,0 +1,12 @@
+apiVersion: 1
+
+providers:
+  - name: "default"
+    orgId: 1
+    folder: ""
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    allowUiUpdates: true
+    options:
+      path: /etc/grafana/provisioning/dashboards

monitoring/grafana/datasources/datasource.yml

@@ -0,0 +1,9 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true
+    editable: true

monitoring/prometheus/alerts.yml

@@ -0,0 +1,74 @@
+groups:
+  - name: linearvoid
+    rules:
+      - alert: ChatDown
+        expr: up{job="chat"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Chat service is down"
+          description: "Chat service has been unreachable for 2 minutes."
+
+      - alert: PlayoutDown
+        expr: up{job="playout"} == 0
+        for: 2m
+        labels:
+          severity: critical
+        annotations:
+          summary: "Playout service is down"
+          description: "Playout service has been unreachable for 2 minutes."
+
+      - alert: RedisDown
+        expr: linearvoid_chat_redis_up == 0
+        for: 2m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Redis is unreachable"
+          description: "Chat service cannot reach Redis. Rate limiting and presence tracking degraded."
+
+      - alert: PlayoutSlateActive
+        expr: linearvoid_playout_mode == 1
+        for: 1m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Playout in slate fallback mode"
+          description: "Playout has switched to emergency slate. Check playout logs."
+
+      - alert: HighChatLatency
+        expr: rate(linearvoid_chat_messages_total[5m]) < 0.01
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Unusually low chat message rate"
+          description: "Message rate below 0.01/s for 5 minutes. Check chat service health."
+
+      - alert: PrometheusWALFailing
+        expr: increase(prometheus_tsdb_wal_writes_failed_total[5m]) > 0
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Prometheus WAL writes failing"
+          description: "Prometheus TSDB WAL write failures detected."
+
+      - alert: HighCPU
+        expr: 100 - (avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80
+        for: 10m
+        labels:
+          severity: warning
+        annotations:
+          summary: "High CPU usage"
+          description: "CPU usage above 80% for 10 minutes on {{ $labels.instance }}."
+
+      - alert: LowDiskSpace
+        expr: (node_filesystem_avail_bytes{mountpoint="/"} / node_filesystem_size_bytes{mountpoint="/"}) * 100 < 10
+        for: 5m
+        labels:
+          severity: warning
+        annotations:
+          summary: "Low disk space"
+          description: "Less than 10% disk space available on {{ $labels.instance }}."

monitoring/prometheus/prometheus.yml

@@ -1,10 +1,28 @@
 global:
   scrape_interval: 15s
 
+rule_files:
+  - "/etc/prometheus/alerts.yml"
+
 scrape_configs:
   - job_name: "chat"
     static_configs:
       - targets: ["chat:3000"]
+  - job_name: "playout"
+    static_configs:
+      - targets: ["playout:8081"]
+  - job_name: "nginx"
+    static_configs:
+      - targets: ["nginx-exporter:9113"]
+  - job_name: "redis"
+    static_configs:
+      - targets: ["redis-exporter:9121"]
+  - job_name: "postgres"
+    static_configs:
+      - targets: ["postgres-exporter:9187"]
+  - job_name: "node"
+    static_configs:
+      - targets: ["node-exporter:9100"]
   - job_name: "prometheus"
     static_configs:
       - targets: ["prometheus:9090"]

scripts/backup-db.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+# Daily PostgreSQL backup to R2 (14-day retention)
+# Schedule: cron daily. Requires: pg_dump, aws CLI or rclone for R2.
+set -e
+
+BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}"
+RETENTION_DAYS="${RETENTION_DAYS:-14}"
+R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
+DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}"
+
+TIMESTAMP=$(date +%Y%m%d)
+BACKUP_FILE="$BACKUP_DIR/daily/db_${TIMESTAMP}.dump"
+
+mkdir -p "$(dirname "$BACKUP_FILE")"
+
+echo "[$(date -Iseconds)] Starting daily backup..."
+
+# Extract connection details from DATABASE_URL
+DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
+DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
+DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p')
+DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p')
+DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p')
+
+export PGPASSWORD="${DB_PASS}"
+
+# Dump
+pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \
+        -Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}"
+
+echo "  Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
+
+# Upload to R2
+if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then
+    aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/daily/db_${TIMESTAMP}.dump" \
+        --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \
+        echo "  Uploaded to R2: daily/db_${TIMESTAMP}.dump" || \
+        echo "  WARNING: R2 upload failed"
+fi
+
+# Cleanup old backups
+find "$BACKUP_DIR/daily" -name "db_*.dump" -mtime "+${RETENTION_DAYS}" -delete 2>/dev/null || true
+echo "  Cleaned up backups older than ${RETENTION_DAYS} days"
+
+echo "[$(date -Iseconds)] Daily backup complete."

scripts/backup-weekly.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# Weekly full PostgreSQL backup to R2 (8-week retention)
+# Schedule: cron weekly (runs backup-db.sh with weekly prefix)
+set -e
+
+BACKUP_DIR="${BACKUP_DIR:-/tmp/linearvoid-backups}"
+RETENTION_WEEKS="${RETENTION_WEEKS:-8}"
+R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
+DB_URL="${DATABASE_URL:-postgres://linearvoid:linearvoid@db:5432/linearvoid?sslmode=disable}"
+
+TIMESTAMP=$(date +%Y%m%d)
+BACKUP_FILE="$BACKUP_DIR/weekly/db_${TIMESTAMP}.dump"
+
+mkdir -p "$(dirname "$BACKUP_FILE")"
+
+echo "[$(date -Iseconds)] Starting weekly full backup..."
+
+DB_HOST=$(echo "$DB_URL" | sed -n 's/.*@\([^:]*\).*/\1/p')
+DB_PORT=$(echo "$DB_URL" | sed -n 's/.*:\([0-9]*\)\/.*/\1/p')
+DB_USER=$(echo "$DB_URL" | sed -n 's/.*:\/\/\([^:]*\).*/\1/p')
+DB_NAME=$(echo "$DB_URL" | sed -n 's/.*\/\([^?]*\).*/\1/p')
+DB_PASS=$(echo "$DB_URL" | sed -n 's/.*:\/\/[^:]*:\([^@]*\).*/\1/p')
+
+export PGPASSWORD="${DB_PASS}"
+
+pg_dump -h "${DB_HOST:-db}" -p "${DB_PORT:-5432}" -U "${DB_USER:-linearvoid}" \
+        -Fc -f "$BACKUP_FILE" "${DB_NAME:-linearvoid}"
+
+echo "  Backup: $BACKUP_FILE ($(du -h "$BACKUP_FILE" | cut -f1))"
+
+# Verify integrity
+pg_restore --list "$BACKUP_FILE" > /dev/null 2>&1 && \
+    echo "  Integrity check: PASSED" || \
+    echo "  WARNING: Integrity check FAILED"
+
+# Upload
+if [ -n "$R2_ENDPOINT" ] && [ -n "$R2_ACCESS_KEY_ID" ]; then
+    aws s3 cp "$BACKUP_FILE" "s3://${R2_BUCKET}/weekly/db_${TIMESTAMP}.dump" \
+        --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null && \
+        echo "  Uploaded to R2: weekly/db_${TIMESTAMP}.dump" || \
+        echo "  WARNING: R2 upload failed"
+fi
+
+# Cleanup
+find "$BACKUP_DIR/weekly" -name "db_*.dump" -mtime "+$((RETENTION_WEEKS * 7))" -delete 2>/dev/null || true
+echo "  Cleaned up backups older than ${RETENTION_WEEKS} weeks"
+
+echo "[$(date -Iseconds)] Weekly backup complete."

scripts/verify-backups.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+# Weekly backup integrity verification.
+# Downloads latest backup from R2, restores into temp container, runs sanity checks.
+# Schedule: cron weekly (after weekly backup).
+set -e
+
+R2_BUCKET="${R2_BUCKET:-linearvoid-backups}"
+TEMP_DIR=$(mktemp -d)
+trap "rm -rf $TEMP_DIR" EXIT
+
+echo "[$(date -Iseconds)] Starting backup integrity verification..."
+
+# Download latest weekly backup from R2
+if [ -z "$R2_ENDPOINT" ] || [ -z "$R2_ACCESS_KEY_ID" ]; then
+    echo "  SKIP: R2 credentials not configured (R2_ENDPOINT, R2_ACCESS_KEY_ID)"
+    exit 0
+fi
+
+LATEST=$(aws s3 ls "s3://${R2_BUCKET}/weekly/" --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null | sort | tail -1 | awk '{print $4}')
+if [ -z "$LATEST" ]; then
+    echo "  SKIP: No weekly backups found in R2"
+    exit 0
+fi
+
+echo "  Downloading: $LATEST"
+aws s3 cp "s3://${R2_BUCKET}/weekly/${LATEST}" "$TEMP_DIR/backup.dump" \
+    --endpoint-url "$R2_ENDPOINT" --no-verify-ssl 2>/dev/null
+
+echo "  File size: $(du -h "$TEMP_DIR/backup.dump" | cut -f1)"
+
+# Restore into temp PostgreSQL container
+echo "  Starting temp PostgreSQL..."
+docker run -d --name lv-backup-test-$$ \
+    -e POSTGRES_PASSWORD=test \
+    -e POSTGRES_DB=linearvoid \
+    -p 15432:5432 \
+    postgres:15-alpine 2>/dev/null
+
+sleep 5
+
+# Wait for PG to be ready
+for i in $(seq 1 10); do
+    if docker exec lv-backup-test-$$ pg_isready -U postgres 2>/dev/null; then
+        break
+    fi
+    sleep 2
+done
+
+# Restore
+echo "  Restoring backup..."
+docker cp "$TEMP_DIR/backup.dump" "lv-backup-test-$$:/tmp/backup.dump"
+docker exec lv-backup-test-$$ pg_restore -U postgres -d linearvoid /tmp/backup.dump 2>/dev/null && \
+    echo "  Restore: OK" || { echo "  Restore: FAILED"; docker rm -f lv-backup-test-$$ 2>/dev/null; exit 1; }
+
+# Sanity checks
+MSG_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM messages;" 2>/dev/null || echo "0")
+MOD_COUNT=$(docker exec lv-backup-test-$$ psql -U postgres -d linearvoid -tAc "SELECT COUNT(*) FROM moderation_logs;" 2>/dev/null || echo "0")
+echo "  Messages: $MSG_COUNT, Moderation: $MOD_COUNT"
+
+# Cleanup
+docker rm -f lv-backup-test-$$ 2>/dev/null
+
+echo "[$(date -Iseconds)] Backup verification complete. OK."