diff --git a/.env.example b/.env.example index f059443..7393cde 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,14 @@ # Example environment variables for CalMiner -# PostgreSQL database connection URL -database_url=postgresql://:@localhost:5432/calminer \ No newline at end of file +# PostgreSQL connection settings +DATABASE_DRIVER=postgresql +DATABASE_HOST=localhost +DATABASE_PORT=5432 +DATABASE_USER= +DATABASE_PASSWORD= +DATABASE_NAME=calminer +# Optional: set a schema (comma-separated for multiple entries) +# DATABASE_SCHEMA=public + +# Legacy fallback (still supported, but granular settings are preferred) +# DATABASE_URL=postgresql://:@localhost:5432/calminer \ No newline at end of file diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index a872058..dfdffa6 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -18,4 +18,12 @@ jobs: docker pull ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest docker stop calminer || true docker rm calminer || true - docker run -d --name calminer -p 8000:8000 ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest + docker run -d --name calminer -p 8000:8000 \ + -e DATABASE_DRIVER=${{ secrets.DATABASE_DRIVER }} \ + -e DATABASE_HOST=${{ secrets.DATABASE_HOST }} \ + -e DATABASE_PORT=${{ secrets.DATABASE_PORT }} \ + -e DATABASE_USER=${{ secrets.DATABASE_USER }} \ + -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \ + -e DATABASE_NAME=${{ secrets.DATABASE_NAME }} \ + -e DATABASE_SCHEMA=${{ secrets.DATABASE_SCHEMA }} \ + ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml.bak similarity index 83% rename from .gitea/workflows/test.yml rename to .gitea/workflows/test.yml.bak index 5b80d1b..97c4f4e 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml.bak @@ -6,13 +6,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: "3.10" - name: Cache pip - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }} diff --git a/Dockerfile b/Dockerfile index 363d24b..db79a39 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,11 +5,11 @@ FROM python:3.10-slim AS builder WORKDIR /app COPY requirements.txt /app/requirements.txt RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential gcc libpq-dev \ - && python -m pip install --upgrade pip \ - && pip install --no-cache-dir --prefix=/install -r /app/requirements.txt \ - && apt-get purge -y --auto-remove build-essential gcc \ - && rm -rf /var/lib/apt/lists/* + && apt-get install -y --no-install-recommends build-essential gcc libpq-dev \ + && python -m pip install --upgrade pip \ + && pip install --no-cache-dir --prefix=/install -r /app/requirements.txt \ + && apt-get purge -y --auto-remove build-essential gcc \ + && rm -rf /var/lib/apt/lists/* FROM python:3.10-slim WORKDIR /app @@ -19,7 +19,14 @@ COPY --from=builder /install /usr/local # Production environment variables ENV PYTHONDONTWRITEBYTECODE=1 \ - PYTHONUNBUFFERED=1 + PYTHONUNBUFFERED=1 \ + DATABASE_DRIVER=postgresql \ + DATABASE_HOST=localhost \ + DATABASE_PORT=5432 \ + DATABASE_USER=calminer \ + DATABASE_PASSWORD=changeme \ + DATABASE_NAME=calminer \ + DATABASE_SCHEMA=public # Copy application code COPY . /app diff --git a/README.md b/README.md index 438ccff..4a3b6fc 100644 --- a/README.md +++ b/README.md @@ -45,8 +45,16 @@ docker build -t calminer:latest . # Run the container (exposes FastAPI on http://localhost:8000) docker run --rm -p 8000:8000 calminer:latest -# Provide environment variables (e.g., database URL) -docker run --rm -p 8000:8000 -e DATABASE_URL="postgresql://user:pass@host/db" calminer:latest +# Provide database configuration via granular environment variables +docker run --rm -p 8000:8000 ^ + -e DATABASE_DRIVER="postgresql" ^ + -e DATABASE_HOST="db.host" ^ + -e DATABASE_PORT="5432" ^ + -e DATABASE_USER="calminer" ^ + -e DATABASE_PASSWORD="s3cret" ^ + -e DATABASE_NAME="calminer" ^ + -e DATABASE_SCHEMA="public" ^ + calminer:latest ``` Use `docker compose` or an orchestrator of your choice to co-locate PostgreSQL/Redis alongside the app when needed. The image expects migrations to be applied before startup. @@ -59,4 +67,4 @@ CalMiner uses Gitea Actions workflows stored in `.gitea/workflows/`: - `build-and-push.yml` builds the Docker image, reuses cached layers, and pushes to the configured registry. - `deploy.yml` pulls the pushed image on the target host and restarts the container. -Pipelines assume the following secrets are provisioned in the Gitea instance: `GITEA_USERNAME`, `GITEA_PASSWORD`, `GITEA_REGISTRY`, `SSH_HOST`, `SSH_USERNAME`, and `SSH_PRIVATE_KEY`. +Pipelines assume the following secrets are provisioned in the Gitea instance: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, and `SSH_PRIVATE_KEY`. diff --git a/config/database.py b/config/database.py index 6421095..ad8a8f5 100644 --- a/config/database.py +++ b/config/database.py @@ -1,12 +1,63 @@ from sqlalchemy import create_engine +from sqlalchemy.engine import URL from sqlalchemy.orm import declarative_base, sessionmaker import os from dotenv import load_dotenv + load_dotenv() -DATABASE_URL = os.environ.get("DATABASE_URL") -if not DATABASE_URL: - raise RuntimeError("DATABASE_URL environment variable is not set") + + +def _build_database_url() -> str: + """Construct the SQLAlchemy database URL from granular environment vars. + + Falls back to `DATABASE_URL` for backward compatibility. + """ + + legacy_url = os.environ.get("DATABASE_URL") + if legacy_url: + return legacy_url + + driver = os.environ.get("DATABASE_DRIVER", "postgresql") + host = os.environ.get("DATABASE_HOST") + port = os.environ.get("DATABASE_PORT", "5432") + user = os.environ.get("DATABASE_USER") + password = os.environ.get("DATABASE_PASSWORD") + database = os.environ.get("DATABASE_NAME") + schema = os.environ.get("DATABASE_SCHEMA", "public") + + missing = [ + var_name + for var_name, value in ( + ("DATABASE_HOST", host), + ("DATABASE_USER", user), + ("DATABASE_NAME", database), + ) + if not value + ] + + if missing: + raise RuntimeError( + "Missing database configuration: set DATABASE_URL or provide " + f"granular variables ({', '.join(missing)})" + ) + + url = URL.create( + drivername=driver, + username=user, + password=password, + host=host, + port=int(port) if port else None, + database=database, + ) + + if schema: + url = url.set(query={"options": f"-csearch_path={schema}"}) + + return str(url) + + +DATABASE_URL = _build_database_url() engine = create_engine(DATABASE_URL, echo=True, future=True) SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) diff --git a/docs/architecture/01_introduction_and_goals.md b/docs/architecture/01_introduction_and_goals.md index d72ec6d..cc5d1c1 100644 --- a/docs/architecture/01_introduction_and_goals.md +++ b/docs/architecture/01_introduction_and_goals.md @@ -38,7 +38,7 @@ Frontend components are server-rendered Jinja2 templates, with Chart.js powering ### Current implementation status (summary) -- Currency normalization, simulation scaffold, and reporting service exist; see [quickstart](docs/quickstart.md) for full status and migration instructions. +- Currency normalization, simulation scaffold, and reporting service exist; see [quickstart](../quickstart.md) for full status and migration instructions. ## MVP Features (migrated) diff --git a/docs/architecture/04_solution_strategy/04_01_implementation_plan_20251020.md b/docs/architecture/04_solution_strategy/04_01_implementation_plan_20251020.md index 4ce9d96..b711cc7 100644 --- a/docs/architecture/04_solution_strategy/04_01_implementation_plan_20251020.md +++ b/docs/architecture/04_solution_strategy/04_01_implementation_plan_20251020.md @@ -6,7 +6,7 @@ This file contains the implementation plan (MVP features, steps, and estimates). 1. Connect to PostgreSQL database with schema `calminer`. 1. Create and activate a virtual environment and install dependencies via `requirements.txt`. -1. Define environment variables in `.env`, including `DATABASE_URL`. +1. Define database environment variables in `.env` (e.g., `DATABASE_DRIVER`, `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME`, optional `DATABASE_SCHEMA`). 1. Configure FastAPI entrypoint in `main.py` to include routers. ## Feature: Scenario Management @@ -107,4 +107,4 @@ This file contains the implementation plan (MVP features, steps, and estimates). 1. Write unit tests in `tests/unit/test_reporting.py`. 1. Enhance UI in `components/Dashboard.html` with charts. -See [UI and Style](docs/architecture/13_ui_and_style.md) for the UI template audit, layout guidance, and next steps. +See [UI and Style](../13_ui_and_style.md) for the UI template audit, layout guidance, and next steps. diff --git a/docs/architecture/05_building_block_view.md b/docs/architecture/05_building_block_view.md index c0aca7c..7767780 100644 --- a/docs/architecture/05_building_block_view.md +++ b/docs/architecture/05_building_block_view.md @@ -8,13 +8,13 @@ status: draft ## Architecture overview -This overview complements [architecture](docs/architecture/README.md) with a high-level map of CalMiner's module layout and request flow. +This overview complements [architecture](README.md) with a high-level map of CalMiner's module layout and request flow. Refer to the detailed architecture chapters in `docs/architecture/`: -- Module map & components: [Building Block View](docs/architecture/05_building_block_view.md) -- Request flow & runtime interactions: [Runtime View](docs/architecture/06_runtime_view.md) -- Simulation roadmap & strategy: [Solution Strategy](docs/architecture/04_solution_strategy.md) +- Module map & components: [Building Block View](05_building_block_view.md) +- Request flow & runtime interactions: [Runtime View](06_runtime_view.md) +- Simulation roadmap & strategy: [Solution Strategy](04_solution_strategy.md) ## System Components diff --git a/docs/architecture/07_deployment_view.md b/docs/architecture/07_deployment_view.md index c0749ec..fb7de7c 100644 --- a/docs/architecture/07_deployment_view.md +++ b/docs/architecture/07_deployment_view.md @@ -64,8 +64,8 @@ The Docker-based deployment path aligns with the solution strategy documented in ### Image Build - The multi-stage `Dockerfile` installs dependencies in a builder layer (including system compilers and Python packages) and copies only the required runtime artifacts to the final image. -- Build arguments are minimal; environment configuration (e.g., `DATABASE_URL`) is supplied at runtime. Secrets and configuration should be passed via environment variables or an orchestrator. -- The resulting image exposes port `8000` and starts `uvicorn main:app` (s. [README.md](../README.md)). +- Build arguments are minimal; database configuration is supplied at runtime via granular variables (`DATABASE_DRIVER`, `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME`, optional `DATABASE_SCHEMA`). Secrets and configuration should be passed via environment variables or an orchestrator. +- The resulting image exposes port `8000` and starts `uvicorn main:app` (s. [README.md](../../README.md)). ### Runtime Environment @@ -79,7 +79,7 @@ The Docker-based deployment path aligns with the solution strategy documented in - `test.yml` executes the pytest suite using cached pip dependencies. - `build-and-push.yml` logs into the container registry, rebuilds the Docker image using GitHub Actions cache-backed layers, and pushes `latest` (and additional tags as required). - `deploy.yml` connects to the target host via SSH, pulls the pushed tag, stops any existing container, and launches the new version. -- Required secrets: `GITEA_REGISTRY`, `GITEA_USERNAME`, `GITEA_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. +- Required secrets: `REGISTRY_URL`, `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. - Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [14 — Testing & CI](14_testing_ci.md) up to date. ## Integrations and Future Work (deployment-related) diff --git a/docs/architecture/14_testing_ci.md b/docs/architecture/14_testing_ci.md index af1fa7c..c125d3d 100644 --- a/docs/architecture/14_testing_ci.md +++ b/docs/architecture/14_testing_ci.md @@ -24,7 +24,7 @@ CalMiner uses a combination of unit, integration, and end-to-end tests to ensure - `test.yml` runs on every push with cached Python dependencies via `actions/cache@v3`. - `build-and-push.yml` builds the Docker image with `docker/build-push-action@v2`, reusing GitHub Actions cache-backed layers, and pushes to the Gitea registry. - `deploy.yml` connects to the target host (via `appleboy/ssh-action`) to pull the freshly pushed image and restart the container. -- Mandatory secrets: `GITEA_USERNAME`, `GITEA_PASSWORD`, `GITEA_REGISTRY`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. +- Mandatory secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. - Run tests on pull requests to shared branches; enforce coverage target ≥80% (pytest-cov). ### Running Tests diff --git a/docs/architecture/15_development_setup.md b/docs/architecture/15_development_setup.md index 223199b..93e8bcb 100644 --- a/docs/architecture/15_development_setup.md +++ b/docs/architecture/15_development_setup.md @@ -10,7 +10,7 @@ This document outlines the local development environment and steps to get the pr ## Clone and Project Setup -```powershell +````powershell # Clone the repository git clone https://git.allucanget.biz/allucanget/calminer.git cd calminer @@ -36,28 +36,34 @@ pip install -r requirements.txt ```sql CREATE USER calminer_user WITH PASSWORD 'your_password'; -``` +```` 1. Create database: -```sql +````sql CREATE DATABASE calminer; ```python ## Environment Variables 1. Copy `.env.example` to `.env` at project root. -1. Edit `.env` to set database connection string: +1. Edit `.env` to set database connection details: ```dotenv -DATABASE_URL=postgresql://:@localhost:5432/calminer -``` +DATABASE_DRIVER=postgresql +DATABASE_HOST=localhost +DATABASE_PORT=5432 +DATABASE_USER=calminer_user +DATABASE_PASSWORD=your_password +DATABASE_NAME=calminer +DATABASE_SCHEMA=public +```` -1. The application uses `python-dotenv` to load these variables. +1. The application uses `python-dotenv` to load these variables. A legacy `DATABASE_URL` value is still accepted if the granular keys are omitted. ## Running the Application -```powershell +````powershell # Start the FastAPI server uvicorn main:app --reload ```python @@ -66,6 +72,6 @@ uvicorn main:app --reload ```powershell pytest -``` +```` E2E tests use Playwright and a session-scoped `live_server` fixture that starts the app at `http://localhost:8001` for browser-driven tests. diff --git a/docs/quickstart.md b/docs/quickstart.md index 4b67bc2..ebf3da8 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -34,7 +34,15 @@ docker build -t calminer:latest . docker run --rm -p 8000:8000 calminer:latest # Supply environment variables (e.g., Postgres connection) -docker run --rm -p 8000:8000 -e DATABASE_URL="postgresql://user:pass@host/db" calminer:latest +docker run --rm -p 8000:8000 ^ + -e DATABASE_DRIVER="postgresql" ^ + -e DATABASE_HOST="db.host" ^ + -e DATABASE_PORT="5432" ^ + -e DATABASE_USER="calminer" ^ + -e DATABASE_PASSWORD="s3cret" ^ + -e DATABASE_NAME="calminer" ^ + -e DATABASE_SCHEMA="public" ^ + calminer:latest ``` If you maintain a Postgres or Redis dependency locally, consider authoring a `docker compose` stack that pairs them with the app container. The Docker image expects the database to be reachable and migrations executed before serving traffic. @@ -66,15 +74,23 @@ The project includes a referential `currency` table and migration/backfill tooli ### Run migrations and backfill (development) -Ensure `DATABASE_URL` is set in your PowerShell session to point at a development Postgres instance. +Configure the granular database settings in your PowerShell session before running migrations. ```powershell -$env:DATABASE_URL = 'postgresql://user:pass@host/db' +$env:DATABASE_DRIVER = 'postgresql' +$env:DATABASE_HOST = 'localhost' +$env:DATABASE_PORT = '5432' +$env:DATABASE_USER = 'calminer' +$env:DATABASE_PASSWORD = 's3cret' +$env:DATABASE_NAME = 'calminer' +$env:DATABASE_SCHEMA = 'public' python scripts/run_migrations.py python scripts/backfill_currency.py --dry-run python scripts/backfill_currency.py --create-missing ``` +> ℹ️ The application still accepts `DATABASE_URL` as a fallback if the granular variables are not set. + Use `--dry-run` first to verify what will change. ## Database Objects @@ -91,10 +107,10 @@ The database contains tables such as `capex`, `opex`, `chemical_consumption`, `f ## Where to look next -- Architecture overview & chapters: [architecture](docs/architecture/README.md) (per-chapter files under `docs/architecture/`) -- [Testing & CI](docs/architecture/14_testing_ci.md) -- [Development setup](docs/architecture/15_development_setup.md) -- Implementation plan & roadmap: [Solution strategy](docs/architecture/04_solution_strategy_extended.md) -- Routes: [routes](routes/) -- Services: [services](services/) -- Scripts: [scripts](scripts/) (migrations and backfills) +- Architecture overview & chapters: [architecture](architecture/README.md) (per-chapter files under `docs/architecture/`) +- [Testing & CI](architecture/14_testing_ci.md) +- [Development setup](architecture/15_development_setup.md) +- Implementation plan & roadmap: [Solution strategy](architecture/04_solution_strategy.md) +- Routes: [routes](../routes/) +- Services: [services](../services/) +- Scripts: [scripts](../scripts/) (migrations and backfills) diff --git a/scripts/backfill_currency.py b/scripts/backfill_currency.py index 4c4614b..15c330b 100644 --- a/scripts/backfill_currency.py +++ b/scripts/backfill_currency.py @@ -6,21 +6,34 @@ Usage: python scripts/backfill_currency.py --create-missing This script is intentionally cautious: it defaults to dry-run mode and will refuse to run -if DATABASE_URL is not set. It supports creating missing currency rows when `--create-missing` +if database connection settings are missing. It supports creating missing currency rows when `--create-missing` is provided. Always run against a development/staging database first. """ from __future__ import annotations -import os import argparse +import importlib +import sys +from pathlib import Path + from sqlalchemy import text, create_engine -def load_env_dburl() -> str: - db = os.environ.get("DATABASE_URL") - if not db: +PROJECT_ROOT = Path(__file__).resolve().parent.parent +if str(PROJECT_ROOT) not in sys.path: + sys.path.insert(0, str(PROJECT_ROOT)) + + +def load_database_url() -> str: + try: + db_module = importlib.import_module("config.database") + except RuntimeError as exc: raise RuntimeError( - "DATABASE_URL not set — set it to your dev/staging DB before running this script") - return db + "Database configuration missing: set DATABASE_URL or provide granular " + "variables (DATABASE_DRIVER, DATABASE_HOST, DATABASE_PORT, DATABASE_USER, " + "DATABASE_PASSWORD, DATABASE_NAME, optional DATABASE_SCHEMA)." + ) from exc + + return getattr(db_module, "DATABASE_URL") def backfill(db_url: str, dry_run: bool = True, create_missing: bool = False) -> None: @@ -41,12 +54,12 @@ def backfill(db_url: str, dry_run: bool = True, create_missing: bool = False) -> # insert and return id conn.execute(text("INSERT INTO currency (code, name, symbol, is_active) VALUES (:c, :n, NULL, TRUE)"), { "c": code, "n": code}) - if db_url.startswith('sqlite:'): - r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), { - "code": code}).fetchone() - else: - r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), { - "code": code}).fetchone() + r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), { + "code": code}).fetchone() + if not r2: + raise RuntimeError( + f"Unable to determine currency ID for '{code}' after insert" + ) return r2[0] return None @@ -95,7 +108,7 @@ def main() -> None: help="Create missing currency rows in the currency table") args = parser.parse_args() - db = load_env_dburl() + db = load_database_url() backfill(db, dry_run=args.dry_run, create_missing=args.create_missing) diff --git a/tests/e2e/conftest.py b/tests/e2e/conftest.py index f1f3238..0745210 100644 --- a/tests/e2e/conftest.py +++ b/tests/e2e/conftest.py @@ -1,12 +1,14 @@ import os import subprocess import time -from typing import Generator +from typing import Dict, Generator import pytest +# type: ignore[import] from playwright.sync_api import Browser, Page, Playwright, sync_playwright import httpx +from sqlalchemy.engine import make_url # Use a different port for the test server to avoid conflicts TEST_PORT = 8001 @@ -16,6 +18,8 @@ BASE_URL = f"http://localhost:{TEST_PORT}" @pytest.fixture(scope="session", autouse=True) def live_server() -> Generator[str, None, None]: """Launch a live test server in a separate process.""" + env = _prepare_database_environment(os.environ.copy()) + process = subprocess.Popen( [ "uvicorn", @@ -26,7 +30,7 @@ def live_server() -> Generator[str, None, None]: ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, - env=os.environ.copy(), + env=env, ) deadline = time.perf_counter() + 30 @@ -85,3 +89,35 @@ def page(browser: Browser, live_server: str) -> Generator[Page, None, None]: page.wait_for_load_state("networkidle") yield page page.close() + + +def _prepare_database_environment(env: Dict[str, str]) -> Dict[str, str]: + """Ensure granular database env vars are available for the app under test.""" + + required = ("DATABASE_HOST", "DATABASE_USER", "DATABASE_NAME") + if all(env.get(key) for key in required): + return env + + legacy_url = env.get("DATABASE_URL") + if not legacy_url: + return env + + url = make_url(legacy_url) + env.setdefault("DATABASE_DRIVER", url.drivername) + if url.host: + env.setdefault("DATABASE_HOST", url.host) + if url.port: + env.setdefault("DATABASE_PORT", str(url.port)) + if url.username: + env.setdefault("DATABASE_USER", url.username) + if url.password: + env.setdefault("DATABASE_PASSWORD", url.password) + if url.database: + env.setdefault("DATABASE_NAME", url.database) + + query_options = dict(url.query) if url.query else {} + options = query_options.get("options") + if isinstance(options, str) and "search_path=" in options: + env.setdefault("DATABASE_SCHEMA", options.split("search_path=")[-1]) + + return env