feat: Add production and development Docker Compose configurations, health check endpoint, and update documentation

2025-10-27 20:57:36 +01:00
parent a6a5f630cc
commit dcb08ab1b8
7 changed files with 318 additions and 4 deletions
--- a/README.md
+++ b/README.md
@@ -78,7 +78,19 @@ docker run --rm -p 8000:8000 ^

 ### Orchestrated Deployment

-Use `docker compose` or an orchestrator of your choice to co-locate PostgreSQL/Redis alongside the app when needed. The image expects migrations to be applied before startup.
+Use `docker compose` or an orchestrator of your choice to co-locate PostgreSQL/Redis/Traefik alongside the app when needed. The image expects migrations to be applied before startup.
+
+### Production docker-compose workflow
+
+`docker-compose.prod.yml` covers the API plus optional Traefik (`reverse-proxy` profile) and on-host Postgres (`local-db` profile). Commands, health checks, and environment variables are documented in [docs/quickstart.md](docs/quickstart.md#compose-driven-production-stack) and expanded in [docs/architecture/07_deployment_view.md](docs/architecture/07_deployment_view.md).
+
+### Development docker-compose workflow
+
+`docker-compose.dev.yml` runs FastAPI (with reload) and Postgres in a single stack. See [docs/quickstart.md](docs/quickstart.md#compose-driven-development-stack) for lifecycle commands and troubleshooting, plus the architecture chapter ([docs/architecture/15_development_setup.md](docs/architecture/15_development_setup.md)) for deeper context.
+
+### Test docker-compose workflow
+
+`docker-compose.test.yml` mirrors the CI pipeline: it provisions Postgres, runs the database bootstrap script, and executes pytest. Usage examples live in [docs/quickstart.md](docs/quickstart.md#compose-driven-test-stack).

 ## CI/CD expectations

--- a/backups/.gitkeep
+++ b/backups/.gitkeep
--- a/config/setup_production.env.example
+++ b/config/setup_production.env.example
@@ -0,0 +1,35 @@
+# Copy this file to config/setup_production.env and replace values with production secrets
+
+# Container image and runtime configuration
+CALMINER_IMAGE=registry.example.com/calminer/api:latest
+CALMINER_DOMAIN=calminer.example.com
+TRAEFIK_ACME_EMAIL=ops@example.com
+CALMINER_API_PORT=8000
+UVICORN_WORKERS=4
+UVICORN_LOG_LEVEL=info
+CALMINER_NETWORK=calminer_backend
+API_LIMIT_CPUS=1.0
+API_LIMIT_MEMORY=1g
+API_RESERVATION_MEMORY=512m
+TRAEFIK_LIMIT_CPUS=0.5
+TRAEFIK_LIMIT_MEMORY=512m
+POSTGRES_LIMIT_CPUS=1.0
+POSTGRES_LIMIT_MEMORY=2g
+POSTGRES_RESERVATION_MEMORY=1g
+
+# Application database connection
+DATABASE_DRIVER=postgresql+psycopg2
+DATABASE_HOST=production-db.internal
+DATABASE_PORT=5432
+DATABASE_NAME=calminer
+DATABASE_USER=calminer_app
+DATABASE_PASSWORD=ChangeMe123!
+DATABASE_SCHEMA=public
+
+# Optional consolidated SQLAlchemy URL (overrides granular settings when set)
+# DATABASE_URL=postgresql+psycopg2://calminer_app:ChangeMe123!@production-db.internal:5432/calminer
+
+# Superuser credentials used by scripts/setup_database.py for migrations/seed data
+DATABASE_SUPERUSER=postgres
+DATABASE_SUPERUSER_PASSWORD=ChangeMeSuper123!
+DATABASE_SUPERUSER_DB=postgres
--- a/docker-compose.prod.yml
+++ b/docker-compose.prod.yml
@@ -0,0 +1,130 @@
+services:
+  api:
+    image: ${CALMINER_IMAGE:-calminer-api:latest}
+    build:
+      context: .
+      dockerfile: Dockerfile
+    restart: unless-stopped
+    env_file:
+      - config/setup_production.env
+    environment:
+      UVICORN_WORKERS: ${UVICORN_WORKERS:-2}
+      UVICORN_LOG_LEVEL: ${UVICORN_LOG_LEVEL:-info}
+    command:
+      [
+        "sh",
+        "-c",
+        "uvicorn main:app --host 0.0.0.0 --port 8000 --workers ${UVICORN_WORKERS:-2} --log-level ${UVICORN_LOG_LEVEL:-info}",
+      ]
+    ports:
+      - "${CALMINER_API_PORT:-8000}:8000"
+    deploy:
+      resources:
+        limits:
+          cpus: ${API_LIMIT_CPUS:-1.0}
+          memory: ${API_LIMIT_MEMORY:-1g}
+        reservations:
+          memory: ${API_RESERVATION_MEMORY:-512m}
+    healthcheck:
+      test:
+        - "CMD-SHELL"
+        - 'python -c "import urllib.request; urllib.request.urlopen(''http://127.0.0.1:8000/health'').read()"'
+      interval: 30s
+      timeout: 10s
+      retries: 5
+      start_period: 30s
+    networks:
+      - calminer_backend
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+    labels:
+      - "traefik.enable=true"
+      - "traefik.http.routers.calminer.rule=Host(`${CALMINER_DOMAIN}`)"
+      - "traefik.http.routers.calminer.entrypoints=websecure"
+      - "traefik.http.routers.calminer.tls.certresolver=letsencrypt"
+      - "traefik.http.services.calminer.loadbalancer.server.port=8000"
+
+  traefik:
+    image: traefik:v3.1
+    restart: unless-stopped
+    command:
+      - "--providers.docker=true"
+      - "--providers.docker.exposedbydefault=false"
+      - "--entrypoints.web.address=:80"
+      - "--entrypoints.websecure.address=:443"
+      - "--certificatesresolvers.letsencrypt.acme.tlschallenge=true"
+      - "--certificatesresolvers.letsencrypt.acme.email=${TRAEFIK_ACME_EMAIL:?TRAEFIK_ACME_EMAIL not set}"
+      - "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
+    ports:
+      - "80:80"
+      - "443:443"
+    deploy:
+      resources:
+        limits:
+          cpus: ${TRAEFIK_LIMIT_CPUS:-0.5}
+          memory: ${TRAEFIK_LIMIT_MEMORY:-512m}
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - traefik_letsencrypt:/letsencrypt
+    networks:
+      - calminer_backend
+    profiles:
+      - reverse-proxy
+    healthcheck:
+      test:
+        - "CMD"
+        - "traefik"
+        - "healthcheck"
+        - "--entrypoints=web"
+        - "--entrypoints=websecure"
+      interval: 30s
+      timeout: 10s
+      retries: 5
+
+  postgres:
+    image: postgres:16
+    profiles:
+      - local-db
+    restart: unless-stopped
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-calminer}
+      POSTGRES_USER: ${POSTGRES_USER:-calminer}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme}
+      LANG: en_US.UTF-8
+      LC_ALL: en_US.UTF-8
+      POSTGRES_INITDB_ARGS: --encoding=UTF8 --locale=en_US.UTF-8
+    ports:
+      - "${CALMINER_DB_PORT:-5432}:5432"
+    deploy:
+      resources:
+        limits:
+          cpus: ${POSTGRES_LIMIT_CPUS:-1.0}
+          memory: ${POSTGRES_LIMIT_MEMORY:-2g}
+        reservations:
+          memory: ${POSTGRES_RESERVATION_MEMORY:-1g}
+    volumes:
+      - pg_data_prod:/var/lib/postgresql/data
+      - ./backups:/backups
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "pg_isready -U ${POSTGRES_USER:-calminer} -d ${POSTGRES_DB:-calminer}",
+        ]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    networks:
+      - calminer_backend
+
+networks:
+  calminer_backend:
+    name: ${CALMINER_NETWORK:-calminer_backend}
+    driver: bridge
+
+volumes:
+  pg_data_prod:
+  traefik_letsencrypt:
--- a/docs/architecture/07_deployment_view.md
+++ b/docs/architecture/07_deployment_view.md
@@ -1,6 +1,6 @@
 ---
-title: "07 — Deployment View"
-description: "Describe deployment topology, infrastructure components, and environments (dev/stage/prod)."
+title: '07 — Deployment View'
+description: 'Describe deployment topology, infrastructure components, and environments (dev/stage/prod).'
 status: draft
 ---

@@ -85,6 +85,14 @@ The development environment is set up for local development and testing. It incl
 - Local PostgreSQL instance (docker compose recommended, script available at `docker-compose.postgres.yml`)
 - FastAPI server running in debug mode

+`docker-compose.dev.yml` encapsulates this topology:
+
+- `api` service mounts the repository for live reloads (`uvicorn --reload`) and depends on the database health check.
+- `db` service uses the Debian-based `postgres:16` image with UTF-8 locale configuration and persists data in `pg_data_dev`.
+- A shared `calminer_backend` bridge network keeps traffic contained; ports 8000/5432 are published for local tooling.
+
+See [docs/quickstart.md](../quickstart.md#compose-driven-development-stack) for command examples and volume maintenance tips.
+
 ### Testing Environment

 The testing environment is set up for automated testing and quality assurance. It includes:
@@ -93,6 +101,14 @@ The testing environment is set up for automated testing and quality assurance. I
 - FastAPI server running in testing mode
 - Automated test suite (e.g., pytest) for running unit and integration tests

+`docker-compose.test.yml` provisions an ephemeral CI-like stack:
+
+- `tests` service builds the application image, installs `requirements-test.txt`, runs the database setup script (dry-run + apply), then executes pytest.
+- `api` service is available on port 8001 for manual verification against the test database.
+- `postgres` service seeds a disposable Postgres 16 instance with health checks and named volumes (`pg_data_test`, `pip_cache_test`).
+
+Typical commands mirror the CI workflow (`docker compose -f docker-compose.test.yml run --rm tests`); the [quickstart](../quickstart.md#compose-driven-test-stack) lists variations and teardown steps.
+
 ### Production Environment

 The production environment is set up for serving live traffic and includes:
@@ -102,6 +118,22 @@ The production environment is set up for serving live traffic and includes:
 - Load balancer (Traefik) for distributing incoming requests
 - Monitoring and logging tools for tracking application performance

+#### Production docker compose topology
+
+- `docker-compose.prod.yml` defines the runtime topology for operator-managed deployments.
+- `api` service runs the FastAPI image with resource limits (`API_LIMIT_CPUS`, `API_LIMIT_MEMORY`) and a `/health` probe consumed by Traefik and the Compose health check.
+- `traefik` service (enabled via the `reverse-proxy` profile) terminates TLS using the ACME resolver configured by `TRAEFIK_ACME_EMAIL` and routes `CALMINER_DOMAIN` traffic to the API.
+- `postgres` service (enabled via the `local-db` profile) exists for edge deployments without managed PostgreSQL and persists data in the `pg_data_prod` volume while mounting `./backups` for operator snapshots.
+- All services join the configurable `CALMINER_NETWORK` (defaults to `calminer_backend`) to keep traffic isolated from host networks.
+
+Deployment workflow:
+
+1. Copy `config/setup_production.env.example` to `config/setup_production.env` and populate domain, registry image tag, database credentials, and resource budgets.
+2. Launch the stack with `docker compose --env-file config/setup_production.env -f docker-compose.prod.yml --profile reverse-proxy up -d` (append `--profile local-db` when hosting Postgres locally).
+3. Run database migrations and seeding using `docker compose --env-file config/setup_production.env -f docker-compose.prod.yml run --rm api python scripts/setup_database.py --run-migrations --seed-data`.
+4. Monitor container health via `docker compose -f docker-compose.prod.yml ps` or Traefik dashboards; the API health endpoint returns `{ "status": "ok" }` when ready.
+5. Shut down with `docker compose -f docker-compose.prod.yml down` (volumes persist unless `-v` is supplied).
+
 ## Containerized Deployment Flow

 The Docker-based deployment path aligns with the solution strategy documented in [Solution Strategy](04_solution_strategy.md) and the CI practices captured in [Testing & CI](07_deployment/07_01_testing_ci.md.md).
--- a/docs/quickstart.md
+++ b/docs/quickstart.md
@@ -4,6 +4,13 @@ This document contains the expanded development, usage, testing, and migration g

 ## Development

+### Prerequisites
+
+- Python 3.10+
+- Node.js 20+ (for Playwright-driven E2E tests)
+- Docker (optional, required for containerized workflows)
+- Git
+
 To get started locally:

 ```powershell
@@ -47,6 +54,99 @@ docker run --rm -p 8000:8000 ^

 If you maintain a Postgres or Redis dependency locally, consider authoring a `docker compose` stack that pairs them with the app container. The Docker image expects the database to be reachable and migrations executed before serving traffic.

+### Compose-driven development stack
+
+The repository ships with `docker-compose.dev.yml`, wiring the API and database into a single development stack. It defaults to the Debian-based `postgres:16` image so UTF-8 locales are available without additional tooling and mounts persistent data in the `pg_data_dev` volume.
+
+Typical workflow (run from the repository root):
+
+```powershell
+# Build images and ensure dependencies are cached
+docker compose -f docker-compose.dev.yml build
+
+# Start FastAPI and Postgres in the background
+docker compose -f docker-compose.dev.yml up -d
+
+# Tail logs for both services
+docker compose -f docker-compose.dev.yml logs -f
+
+# Stop services but keep the database volume for reuse
+docker compose -f docker-compose.dev.yml down
+
+# Remove the persistent Postgres volume when you need a clean slate
+docker volume rm calminer_pg_data_dev  # optional; confirm exact name with `docker volume ls`
+```
+
+Environment variables used by the containers live directly in the compose file (`DATABASE_HOST=db`, `DATABASE_NAME=calminer_dev`, etc.), so no extra `.env` file is required. Adjust or override them via `docker compose ... -e VAR=value` if necessary.
+
+For a deeper walkthrough (including volume naming conventions, port mappings, and how the stack fits into the broader architecture), cross-check `docs/architecture/15_development_setup.md`. That chapter mirrors the compose defaults captured here so both documents stay in sync.
+
+### Compose-driven test stack
+
+Use `docker-compose.test.yml` to spin up a Postgres 16 container and execute the Python test suite in a disposable worker container:
+
+```powershell
+# Build images used by the test workflow
+docker compose -f docker-compose.test.yml build
+
+# Run the default target (unit tests)
+docker compose -f docker-compose.test.yml run --rm tests
+
+# Run a specific target (e.g., full suite)
+docker compose -f docker-compose.test.yml run --rm -e PYTEST_TARGET=tests tests
+
+# Tear everything down and drop the test database volume
+docker compose -f docker-compose.test.yml down -v
+```
+
+The `tests` service prepares the database via `scripts/setup_database.py` before invoking pytest, ensuring migrations and seed data mirror CI behaviour. Named volumes (`pip_cache_test`, `pg_data_test`) cache dependencies and data between runs; remove them with `down -v` whenever you want a pristine environment. An `api` service is available on `http://localhost:8001` for spot-checking API responses against the same test database.
+
+### Compose-driven production stack
+
+Use `docker-compose.prod.yml` for operator-managed deployments. The file defines:
+
+- `api`: FastAPI container with configurable CPU/memory limits and a `/health` probe.
+- `traefik`: Optional (enable with the `reverse-proxy` profile) to terminate TLS and route traffic based on `CALMINER_DOMAIN`.
+- `postgres`: Optional (enable with the `local-db` profile) when a managed database is unavailable; persists data in `pg_data_prod` and mounts `./backups`.
+
+Commands (run from the repository root):
+
+```powershell
+# Prepare environment variables once per environment
+copy config\setup_production.env.example config\setup_production.env
+
+# Start API behind Traefik
+docker compose ^
+  --env-file config/setup_production.env ^
+  -f docker-compose.prod.yml ^
+  --profile reverse-proxy ^
+  up -d
+
+# Add the local Postgres profile when running without managed DB
+docker compose ^
+  --env-file config/setup_production.env ^
+  -f docker-compose.prod.yml ^
+  --profile reverse-proxy --profile local-db ^
+  up -d
+
+# Apply migrations/seed data
+docker compose ^
+  --env-file config/setup_production.env ^
+  -f docker-compose.prod.yml ^
+  run --rm api ^
+  python scripts/setup_database.py --run-migrations --seed-data
+
+# Check health (FastAPI exposes /health)
+docker compose -f docker-compose.prod.yml ps
+
+# Stop services (volumes persist unless -v is supplied)
+docker compose -f docker-compose.prod.yml down
+```
+
+Key environment variables (documented in `config/setup_production.env.example`): container image tag, domain/ACME email, published ports, network name, and resource limits (`API_LIMIT_CPUS`, `API_LIMIT_MEMORY`, etc.).
+
+For deployment topology diagrams and operational sequencing, see [docs/architecture/07_deployment_view.md](architecture/07_deployment_view.md#production-docker-compose-topology).
+
 ## Usage Overview

 - **API base URL**: `http://localhost:8000/api`
@@ -98,7 +198,7 @@ python scripts/setup_database.py --run-migrations --seed-data
 The dry-run invocation reports which steps would execute without making changes. The live run applies the baseline (if not already recorded in `schema_migrations`) and seeds the reference data relied upon by the UI and API.

 > ℹ️ When `--seed-data` is supplied without `--run-migrations`, the bootstrap script automatically applies any pending SQL migrations first so the `application_setting` table (and future settings-backed features) are present before seeding.
-
+>
 > ℹ️ The application still accepts `DATABASE_URL` as a fallback if the granular variables are not set.

 ## Database bootstrap workflow
--- a/main.py
+++ b/main.py
@@ -32,6 +32,11 @@ async def json_validation(
    return await validate_json(request, call_next)


+@app.get("/health", summary="Container health probe")
+async def health() -> dict[str, str]:
+    return {"status": "ok"}
+
+
 app.mount("/static", StaticFiles(directory="static"), name="static")

 # Include API routers