feat: Refactor database configuration to use granular environment variables; update Docker and CI/CD workflows accordingly

.env.example

@@ -1,4 +1,14 @@
 # Example environment variables for CalMiner
 
-# PostgreSQL database connection URL
+# PostgreSQL connection settings
-database_url=postgresql://<user>:<password>@localhost:5432/calminer
+DATABASE_DRIVER=postgresql
+DATABASE_HOST=localhost
+DATABASE_PORT=5432
+DATABASE_USER=<user>
+DATABASE_PASSWORD=<password>
+DATABASE_NAME=calminer
+# Optional: set a schema (comma-separated for multiple entries)
+# DATABASE_SCHEMA=public
+
+# Legacy fallback (still supported, but granular settings are preferred)
+# DATABASE_URL=postgresql://<user>:<password>@localhost:5432/calminer

.gitea/workflows/build-and-push.yml

@@ -9,17 +9,17 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
 
       - name: Login to Gitea Registry
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ${{ secrets.REGISTRY_URL }}
           username: ${{ secrets.REGISTRY_USERNAME }}
           password: ${{ secrets.REGISTRY_PASSWORD }}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v5
         with:
           context: .
           push: true

.gitea/workflows/deploy.yml

@@ -18,4 +18,12 @@ jobs:
             docker pull ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest
             docker stop calminer || true
             docker rm calminer || true
-            docker run -d --name calminer -p 8000:8000 ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest
+            docker run -d --name calminer -p 8000:8000 \
+              -e DATABASE_DRIVER=${{ secrets.DATABASE_DRIVER }} \
+              -e DATABASE_HOST=${{ secrets.DATABASE_HOST }} \
+              -e DATABASE_PORT=${{ secrets.DATABASE_PORT }} \
+              -e DATABASE_USER=${{ secrets.DATABASE_USER }} \
+              -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
+              -e DATABASE_NAME=${{ secrets.DATABASE_NAME }} \
+              -e DATABASE_SCHEMA=${{ secrets.DATABASE_SCHEMA }} \
+              ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest

.gitea/workflows/test.yml.bak

@@ -6,13 +6,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
           python-version: "3.10"
       - name: Cache pip
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.cache/pip
           key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}

Dockerfile

@@ -5,11 +5,11 @@ FROM python:3.10-slim AS builder
 WORKDIR /app
 COPY requirements.txt /app/requirements.txt
 RUN apt-get update \
-	&& apt-get install -y --no-install-recommends build-essential gcc libpq-dev \
+    && apt-get install -y --no-install-recommends build-essential gcc libpq-dev \
-	&& python -m pip install --upgrade pip \
+    && python -m pip install --upgrade pip \
-	&& pip install --no-cache-dir --prefix=/install -r /app/requirements.txt \
+    && pip install --no-cache-dir --prefix=/install -r /app/requirements.txt \
-	&& apt-get purge -y --auto-remove build-essential gcc \
+    && apt-get purge -y --auto-remove build-essential gcc \
-	&& rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/*
 
 FROM python:3.10-slim
 WORKDIR /app
@@ -19,7 +19,14 @@ COPY --from=builder /install /usr/local
 
 # Production environment variables
 ENV PYTHONDONTWRITEBYTECODE=1 \
-	PYTHONUNBUFFERED=1
+	PYTHONUNBUFFERED=1 \
+	DATABASE_DRIVER=postgresql \
+	DATABASE_HOST=localhost \
+	DATABASE_PORT=5432 \
+	DATABASE_USER=calminer \
+	DATABASE_PASSWORD=changeme \
+	DATABASE_NAME=calminer \
+	DATABASE_SCHEMA=public
 
 # Copy application code
 COPY . /app

README.md

@@ -45,8 +45,16 @@ docker build -t calminer:latest .
 # Run the container (exposes FastAPI on http://localhost:8000)
 docker run --rm -p 8000:8000 calminer:latest
 
-# Provide environment variables (e.g., database URL)
+# Provide database configuration via granular environment variables
-docker run --rm -p 8000:8000 -e DATABASE_URL="postgresql://user:pass@host/db" calminer:latest
+docker run --rm -p 8000:8000 ^
+  -e DATABASE_DRIVER="postgresql" ^
+  -e DATABASE_HOST="db.host" ^
+  -e DATABASE_PORT="5432" ^
+  -e DATABASE_USER="calminer" ^
+  -e DATABASE_PASSWORD="s3cret" ^
+  -e DATABASE_NAME="calminer" ^
+  -e DATABASE_SCHEMA="public" ^
+  calminer:latest
 ```
 
 Use `docker compose` or an orchestrator of your choice to co-locate PostgreSQL/Redis alongside the app when needed. The image expects migrations to be applied before startup.
@@ -59,4 +67,4 @@ CalMiner uses Gitea Actions workflows stored in `.gitea/workflows/`:
 - `build-and-push.yml` builds the Docker image, reuses cached layers, and pushes to the configured registry.
 - `deploy.yml` pulls the pushed image on the target host and restarts the container.
 
-Pipelines assume the following secrets are provisioned in the Gitea instance: `GITEA_USERNAME`, `GITEA_PASSWORD`, `GITEA_REGISTRY`, `SSH_HOST`, `SSH_USERNAME`, and `SSH_PRIVATE_KEY`.
+Pipelines assume the following secrets are provisioned in the Gitea instance: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, and `SSH_PRIVATE_KEY`.

config/database.py

@@ -1,12 +1,63 @@
 from sqlalchemy import create_engine
+from sqlalchemy.engine import URL
 from sqlalchemy.orm import declarative_base, sessionmaker
 import os
 from dotenv import load_dotenv
 
+
 load_dotenv()
-DATABASE_URL = os.environ.get("DATABASE_URL")
+
-if not DATABASE_URL:
+
-    raise RuntimeError("DATABASE_URL environment variable is not set")
+def _build_database_url() -> str:
+    """Construct the SQLAlchemy database URL from granular environment vars.
+
+    Falls back to `DATABASE_URL` for backward compatibility.
+    """
+
+    legacy_url = os.environ.get("DATABASE_URL")
+    if legacy_url:
+        return legacy_url
+
+    driver = os.environ.get("DATABASE_DRIVER", "postgresql")
+    host = os.environ.get("DATABASE_HOST")
+    port = os.environ.get("DATABASE_PORT", "5432")
+    user = os.environ.get("DATABASE_USER")
+    password = os.environ.get("DATABASE_PASSWORD")
+    database = os.environ.get("DATABASE_NAME")
+    schema = os.environ.get("DATABASE_SCHEMA", "public")
+
+    missing = [
+        var_name
+        for var_name, value in (
+            ("DATABASE_HOST", host),
+            ("DATABASE_USER", user),
+            ("DATABASE_NAME", database),
+        )
+        if not value
+    ]
+
+    if missing:
+        raise RuntimeError(
+            "Missing database configuration: set DATABASE_URL or provide "
+            f"granular variables ({', '.join(missing)})"
+        )
+
+    url = URL.create(
+        drivername=driver,
+        username=user,
+        password=password,
+        host=host,
+        port=int(port) if port else None,
+        database=database,
+    )
+
+    if schema:
+        url = url.set(query={"options": f"-csearch_path={schema}"})
+
+    return str(url)
+
+
+DATABASE_URL = _build_database_url()
 
 engine = create_engine(DATABASE_URL, echo=True, future=True)
 SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

docs/architecture/01_introduction_and_goals.md

@@ -38,7 +38,7 @@ Frontend components are server-rendered Jinja2 templates, with Chart.js powering
 
 ### Current implementation status (summary)
 
-- Currency normalization, simulation scaffold, and reporting service exist; see [quickstart](docs/quickstart.md) for full status and migration instructions.
+- Currency normalization, simulation scaffold, and reporting service exist; see [quickstart](../quickstart.md) for full status and migration instructions.
 
 ## MVP Features (migrated)
 

docs/architecture/04_solution_strategy/04_01_implementation_plan_20251020.md

@@ -6,7 +6,7 @@ This file contains the implementation plan (MVP features, steps, and estimates).
 
 1. Connect to PostgreSQL database with schema `calminer`.
 1. Create and activate a virtual environment and install dependencies via `requirements.txt`.
-1. Define environment variables in `.env`, including `DATABASE_URL`.
+1. Define database environment variables in `.env` (e.g., `DATABASE_DRIVER`, `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME`, optional `DATABASE_SCHEMA`).
 1. Configure FastAPI entrypoint in `main.py` to include routers.
 
 ## Feature: Scenario Management
@@ -107,4 +107,4 @@ This file contains the implementation plan (MVP features, steps, and estimates).
 1. Write unit tests in `tests/unit/test_reporting.py`.
 1. Enhance UI in `components/Dashboard.html` with charts.
 
-See [UI and Style](docs/architecture/13_ui_and_style.md) for the UI template audit, layout guidance, and next steps.
+See [UI and Style](../13_ui_and_style.md) for the UI template audit, layout guidance, and next steps.

docs/architecture/05_building_block_view.md

@@ -8,13 +8,13 @@ status: draft
 
 ## Architecture overview
 
-This overview complements [architecture](docs/architecture/README.md) with a high-level map of CalMiner's module layout and request flow.
+This overview complements [architecture](README.md) with a high-level map of CalMiner's module layout and request flow.
 
 Refer to the detailed architecture chapters in `docs/architecture/`:
 
-- Module map & components: [Building Block View](docs/architecture/05_building_block_view.md)
+- Module map & components: [Building Block View](05_building_block_view.md)
-- Request flow & runtime interactions: [Runtime View](docs/architecture/06_runtime_view.md)
+- Request flow & runtime interactions: [Runtime View](06_runtime_view.md)
-- Simulation roadmap & strategy: [Solution Strategy](docs/architecture/04_solution_strategy.md)
+- Simulation roadmap & strategy: [Solution Strategy](04_solution_strategy.md)
 
 ## System Components
 

docs/architecture/07_deployment_view.md

@@ -64,8 +64,8 @@ The Docker-based deployment path aligns with the solution strategy documented in
 ### Image Build
 
 - The multi-stage `Dockerfile` installs dependencies in a builder layer (including system compilers and Python packages) and copies only the required runtime artifacts to the final image.
-- Build arguments are minimal; environment configuration (e.g., `DATABASE_URL`) is supplied at runtime. Secrets and configuration should be passed via environment variables or an orchestrator.
+- Build arguments are minimal; database configuration is supplied at runtime via granular variables (`DATABASE_DRIVER`, `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME`, optional `DATABASE_SCHEMA`). Secrets and configuration should be passed via environment variables or an orchestrator.
-- The resulting image exposes port `8000` and starts `uvicorn main:app` (s. [README.md](../README.md)).
+- The resulting image exposes port `8000` and starts `uvicorn main:app` (s. [README.md](../../README.md)).
 
 ### Runtime Environment
 
@@ -79,7 +79,7 @@ The Docker-based deployment path aligns with the solution strategy documented in
 - `test.yml` executes the pytest suite using cached pip dependencies.
 - `build-and-push.yml` logs into the container registry, rebuilds the Docker image using GitHub Actions cache-backed layers, and pushes `latest` (and additional tags as required).
 - `deploy.yml` connects to the target host via SSH, pulls the pushed tag, stops any existing container, and launches the new version.
-- Required secrets: `GITEA_REGISTRY`, `GITEA_USERNAME`, `GITEA_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
+- Required secrets: `REGISTRY_URL`, `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
 - Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [14 — Testing & CI](14_testing_ci.md) up to date.
 
 ## Integrations and Future Work (deployment-related)

docs/architecture/14_testing_ci.md

@@ -24,7 +24,7 @@ CalMiner uses a combination of unit, integration, and end-to-end tests to ensure
 - `test.yml` runs on every push with cached Python dependencies via `actions/cache@v3`.
 - `build-and-push.yml` builds the Docker image with `docker/build-push-action@v2`, reusing GitHub Actions cache-backed layers, and pushes to the Gitea registry.
 - `deploy.yml` connects to the target host (via `appleboy/ssh-action`) to pull the freshly pushed image and restart the container.
-- Mandatory secrets: `GITEA_USERNAME`, `GITEA_PASSWORD`, `GITEA_REGISTRY`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
+- Mandatory secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
 - Run tests on pull requests to shared branches; enforce coverage target ≥80% (pytest-cov).
 
 ### Running Tests

docs/architecture/15_development_setup.md

@@ -10,7 +10,7 @@ This document outlines the local development environment and steps to get the pr
 
 ## Clone and Project Setup
 
-```powershell
+````powershell
 # Clone the repository
 git clone https://git.allucanget.biz/allucanget/calminer.git
 cd calminer
@@ -36,28 +36,34 @@ pip install -r requirements.txt
 
 ```sql
 CREATE USER calminer_user WITH PASSWORD 'your_password';
-```
+````
 
 1. Create database:
 
-```sql
+````sql
 CREATE DATABASE calminer;
 ```python
 
 ## Environment Variables
 
 1. Copy `.env.example` to `.env` at project root.
-1. Edit `.env` to set database connection string:
+1. Edit `.env` to set database connection details:
 
 ```dotenv
-DATABASE_URL=postgresql://<user>:<password>@localhost:5432/calminer
+DATABASE_DRIVER=postgresql
-```
+DATABASE_HOST=localhost
+DATABASE_PORT=5432
+DATABASE_USER=calminer_user
+DATABASE_PASSWORD=your_password
+DATABASE_NAME=calminer
+DATABASE_SCHEMA=public
+````
 
-1. The application uses `python-dotenv` to load these variables.
+1. The application uses `python-dotenv` to load these variables. A legacy `DATABASE_URL` value is still accepted if the granular keys are omitted.
 
 ## Running the Application
 
-```powershell
+````powershell
 # Start the FastAPI server
 uvicorn main:app --reload
 ```python
@@ -66,6 +72,6 @@ uvicorn main:app --reload
 
 ```powershell
 pytest
-```
+````
 
 E2E tests use Playwright and a session-scoped `live_server` fixture that starts the app at `http://localhost:8001` for browser-driven tests.

docs/quickstart.md

@@ -34,7 +34,15 @@ docker build -t calminer:latest .
 docker run --rm -p 8000:8000 calminer:latest
 
 # Supply environment variables (e.g., Postgres connection)
-docker run --rm -p 8000:8000 -e DATABASE_URL="postgresql://user:pass@host/db" calminer:latest
+docker run --rm -p 8000:8000 ^
+  -e DATABASE_DRIVER="postgresql" ^
+  -e DATABASE_HOST="db.host" ^
+  -e DATABASE_PORT="5432" ^
+  -e DATABASE_USER="calminer" ^
+  -e DATABASE_PASSWORD="s3cret" ^
+  -e DATABASE_NAME="calminer" ^
+  -e DATABASE_SCHEMA="public" ^
+  calminer:latest
 ```
 
 If you maintain a Postgres or Redis dependency locally, consider authoring a `docker compose` stack that pairs them with the app container. The Docker image expects the database to be reachable and migrations executed before serving traffic.
@@ -66,15 +74,23 @@ The project includes a referential `currency` table and migration/backfill tooli
 
 ### Run migrations and backfill (development)
 
-Ensure `DATABASE_URL` is set in your PowerShell session to point at a development Postgres instance.
+Configure the granular database settings in your PowerShell session before running migrations.
 
 ```powershell
-$env:DATABASE_URL = 'postgresql://user:pass@host/db'
+$env:DATABASE_DRIVER = 'postgresql'
+$env:DATABASE_HOST = 'localhost'
+$env:DATABASE_PORT = '5432'
+$env:DATABASE_USER = 'calminer'
+$env:DATABASE_PASSWORD = 's3cret'
+$env:DATABASE_NAME = 'calminer'
+$env:DATABASE_SCHEMA = 'public'
 python scripts/run_migrations.py
 python scripts/backfill_currency.py --dry-run
 python scripts/backfill_currency.py --create-missing
 ```
 
+> ℹ️ The application still accepts `DATABASE_URL` as a fallback if the granular variables are not set.
+
 Use `--dry-run` first to verify what will change.
 
 ## Database Objects
@@ -91,10 +107,10 @@ The database contains tables such as `capex`, `opex`, `chemical_consumption`, `f
 
 ## Where to look next
 
-- Architecture overview & chapters: [architecture](docs/architecture/README.md) (per-chapter files under `docs/architecture/`)
+- Architecture overview & chapters: [architecture](architecture/README.md) (per-chapter files under `docs/architecture/`)
-- [Testing & CI](docs/architecture/14_testing_ci.md)
+- [Testing & CI](architecture/14_testing_ci.md)
-- [Development setup](docs/architecture/15_development_setup.md)
+- [Development setup](architecture/15_development_setup.md)
-- Implementation plan & roadmap: [Solution strategy](docs/architecture/04_solution_strategy_extended.md)
+- Implementation plan & roadmap: [Solution strategy](architecture/04_solution_strategy.md)
-- Routes: [routes](routes/)
+- Routes: [routes](../routes/)
-- Services: [services](services/)
+- Services: [services](../services/)
-- Scripts: [scripts](scripts/) (migrations and backfills)
+- Scripts: [scripts](../scripts/) (migrations and backfills)

requirements.txt

@@ -10,3 +10,4 @@ numpy
 pytest
 pytest-cov
 pytest-httpx
+playwright

scripts/backfill_currency.py

@@ -6,21 +6,34 @@ Usage:
   python scripts/backfill_currency.py --create-missing
 
 This script is intentionally cautious: it defaults to dry-run mode and will refuse to run
-if DATABASE_URL is not set. It supports creating missing currency rows when `--create-missing`
+if database connection settings are missing. It supports creating missing currency rows when `--create-missing`
 is provided. Always run against a development/staging database first.
 """
 from __future__ import annotations
-import os
 import argparse
+import importlib
+import sys
+from pathlib import Path
+
 from sqlalchemy import text, create_engine
 
 
-def load_env_dburl() -> str:
+PROJECT_ROOT = Path(__file__).resolve().parent.parent
-    db = os.environ.get("DATABASE_URL")
+if str(PROJECT_ROOT) not in sys.path:
-    if not db:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+
+def load_database_url() -> str:
+    try:
+        db_module = importlib.import_module("config.database")
+    except RuntimeError as exc:
         raise RuntimeError(
-            "DATABASE_URL not set — set it to your dev/staging DB before running this script")
+            "Database configuration missing: set DATABASE_URL or provide granular "
-    return db
+            "variables (DATABASE_DRIVER, DATABASE_HOST, DATABASE_PORT, DATABASE_USER, "
+            "DATABASE_PASSWORD, DATABASE_NAME, optional DATABASE_SCHEMA)."
+        ) from exc
+
+    return getattr(db_module, "DATABASE_URL")
 
 
 def backfill(db_url: str, dry_run: bool = True, create_missing: bool = False) -> None:
@@ -41,12 +54,12 @@ def backfill(db_url: str, dry_run: bool = True, create_missing: bool = False) ->
                 # insert and return id
                 conn.execute(text("INSERT INTO currency (code, name, symbol, is_active) VALUES (:c, :n, NULL, TRUE)"), {
                              "c": code, "n": code})
-                if db_url.startswith('sqlite:'):
+                r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), {
-                    r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), {
+                                  "code": code}).fetchone()
-                                      "code": code}).fetchone()
+                if not r2:
-                else:
+                    raise RuntimeError(
-                    r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), {
+                        f"Unable to determine currency ID for '{code}' after insert"
-                                      "code": code}).fetchone()
+                    )
                 return r2[0]
             return None
 
@@ -95,7 +108,7 @@ def main() -> None:
                         help="Create missing currency rows in the currency table")
     args = parser.parse_args()
 
-    db = load_env_dburl()
+    db = load_database_url()
     backfill(db, dry_run=args.dry_run, create_missing=args.create_missing)
 
 

tests/e2e/conftest.py

@@ -1,12 +1,14 @@
 import os
 import subprocess
 import time
-from typing import Generator
+from typing import Dict, Generator
 
 import pytest
+# type: ignore[import]
 from playwright.sync_api import Browser, Page, Playwright, sync_playwright
 
 import httpx
+from sqlalchemy.engine import make_url
 
 # Use a different port for the test server to avoid conflicts
 TEST_PORT = 8001
@@ -16,6 +18,8 @@ BASE_URL = f"http://localhost:{TEST_PORT}"
 @pytest.fixture(scope="session", autouse=True)
 def live_server() -> Generator[str, None, None]:
     """Launch a live test server in a separate process."""
+    env = _prepare_database_environment(os.environ.copy())
+
     process = subprocess.Popen(
         [
             "uvicorn",
@@ -26,7 +30,7 @@ def live_server() -> Generator[str, None, None]:
         ],
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
-        env=os.environ.copy(),
+        env=env,
     )
 
     deadline = time.perf_counter() + 30
@@ -85,3 +89,35 @@ def page(browser: Browser, live_server: str) -> Generator[Page, None, None]:
     page.wait_for_load_state("networkidle")
     yield page
     page.close()
+
+
+def _prepare_database_environment(env: Dict[str, str]) -> Dict[str, str]:
+    """Ensure granular database env vars are available for the app under test."""
+
+    required = ("DATABASE_HOST", "DATABASE_USER", "DATABASE_NAME")
+    if all(env.get(key) for key in required):
+        return env
+
+    legacy_url = env.get("DATABASE_URL")
+    if not legacy_url:
+        return env
+
+    url = make_url(legacy_url)
+    env.setdefault("DATABASE_DRIVER", url.drivername)
+    if url.host:
+        env.setdefault("DATABASE_HOST", url.host)
+    if url.port:
+        env.setdefault("DATABASE_PORT", str(url.port))
+    if url.username:
+        env.setdefault("DATABASE_USER", url.username)
+    if url.password:
+        env.setdefault("DATABASE_PASSWORD", url.password)
+    if url.database:
+        env.setdefault("DATABASE_NAME", url.database)
+
+    query_options = dict(url.query) if url.query else {}
+    options = query_options.get("options")
+    if isinstance(options, str) and "search_path=" in options:
+        env.setdefault("DATABASE_SCHEMA", options.split("search_path=")[-1])
+
+    return env