allucanget/calminer
2
0
Fork 0
Code Pull Requests Actions Packages Releases Activity

feat: Refactor database configuration to use granular environment variables; update Docker and CI/CD workflows accordingly
Some checks failed
Build and Push Docker Image / build-and-push (push) Failing after 6s
Details
Deploy to Server / deploy (push) Failing after 2s
Details

Browse Source
This commit is contained in:
zwitschi
2025-10-23 19:17:24 +02:00
parent 8c3062fd80
commit 8dedfb8f26
15 changed files with 219 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
.env.example
View File

@@ -1,4 +1,14 @@
# Example environment variables for CalMiner
# PostgreSQL database connection URL
database_url=postgresql://<user>:<password>@localhost:5432/calminer
# PostgreSQL connection settings
DATABASE_DRIVER=postgresql
DATABASE_HOST=localhost
DATABASE_PORT=5432
DATABASE_USER=<user>
DATABASE_PASSWORD=<password>
DATABASE_NAME=calminer
# Optional: set a schema (comma-separated for multiple entries)
# DATABASE_SCHEMA=public
# Legacy fallback (still supported, but granular settings are preferred)
# DATABASE_URL=postgresql://<user>:<password>@localhost:5432/calminer

10
.gitea/workflows/deploy.yml
View File

@@ -18,4 +18,12 @@ jobs:
docker pull ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest
docker stop calminer || true
docker rm calminer || true
docker run -d --name calminer -p 8000:8000 ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest
docker run -d --name calminer -p 8000:8000 \
-e DATABASE_DRIVER=${{ secrets.DATABASE_DRIVER }} \
-e DATABASE_HOST=${{ secrets.DATABASE_HOST }} \
-e DATABASE_PORT=${{ secrets.DATABASE_PORT }} \
-e DATABASE_USER=${{ secrets.DATABASE_USER }} \
-e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
-e DATABASE_NAME=${{ secrets.DATABASE_NAME }} \
-e DATABASE_SCHEMA=${{ secrets.DATABASE_SCHEMA }} \
${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest

6
.gitea/workflows/test.yml → .gitea/workflows/test.yml.bak
View File

@@ -6,13 +6,13 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Checkout code
uses: actions/checkout@v2
uses: actions/checkout@v4
- name: Set up Python
uses: actions/setup-python@v2
uses: actions/setup-python@v5
with:
python-version: "3.10"
- name: Cache pip
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('requirements.txt') }}

9
Dockerfile
View File

@@ -19,7 +19,14 @@ COPY --from=builder /install /usr/local
# Production environment variables
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
PYTHONUNBUFFERED=1 \
DATABASE_DRIVER=postgresql \
DATABASE_HOST=localhost \
DATABASE_PORT=5432 \
DATABASE_USER=calminer \
DATABASE_PASSWORD=changeme \
DATABASE_NAME=calminer \
DATABASE_SCHEMA=public
# Copy application code
COPY . /app

14
README.md
View File

@@ -45,8 +45,16 @@ docker build -t calminer:latest .
# Run the container (exposes FastAPI on http://localhost:8000)
docker run --rm -p 8000:8000 calminer:latest
# Provide environment variables (e.g., database URL)
docker run --rm -p 8000:8000 -e DATABASE_URL="postgresql://user:pass@host/db" calminer:latest
# Provide database configuration via granular environment variables
docker run --rm -p 8000:8000 ^
-e DATABASE_DRIVER="postgresql" ^
-e DATABASE_HOST="db.host" ^
-e DATABASE_PORT="5432" ^
-e DATABASE_USER="calminer" ^
-e DATABASE_PASSWORD="s3cret" ^
-e DATABASE_NAME="calminer" ^
-e DATABASE_SCHEMA="public" ^
calminer:latest
```
Use `docker compose` or an orchestrator of your choice to co-locate PostgreSQL/Redis alongside the app when needed. The image expects migrations to be applied before startup.
@@ -59,4 +67,4 @@ CalMiner uses Gitea Actions workflows stored in `.gitea/workflows/`:
- `build-and-push.yml` builds the Docker image, reuses cached layers, and pushes to the configured registry.
- `deploy.yml` pulls the pushed image on the target host and restarts the container.
Pipelines assume the following secrets are provisioned in the Gitea instance: `GITEA_USERNAME`, `GITEA_PASSWORD`, `GITEA_REGISTRY`, `SSH_HOST`, `SSH_USERNAME`, and `SSH_PRIVATE_KEY`.
Pipelines assume the following secrets are provisioned in the Gitea instance: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, and `SSH_PRIVATE_KEY`.

57
config/database.py
View File

@@ -1,12 +1,63 @@
from sqlalchemy import create_engine
from sqlalchemy.engine import URL
from sqlalchemy.orm import declarative_base, sessionmaker
import os
from dotenv import load_dotenv
load_dotenv()
DATABASE_URL = os.environ.get("DATABASE_URL")
if not DATABASE_URL:
raise RuntimeError("DATABASE_URL environment variable is not set")
def _build_database_url() -> str:
"""Construct the SQLAlchemy database URL from granular environment vars.
Falls back to `DATABASE_URL` for backward compatibility.
"""
legacy_url = os.environ.get("DATABASE_URL")
if legacy_url:
return legacy_url
driver = os.environ.get("DATABASE_DRIVER", "postgresql")
host = os.environ.get("DATABASE_HOST")
port = os.environ.get("DATABASE_PORT", "5432")
user = os.environ.get("DATABASE_USER")
password = os.environ.get("DATABASE_PASSWORD")
database = os.environ.get("DATABASE_NAME")
schema = os.environ.get("DATABASE_SCHEMA", "public")
missing = [
var_name
for var_name, value in (
("DATABASE_HOST", host),
("DATABASE_USER", user),
("DATABASE_NAME", database),
)
if not value
]
if missing:
raise RuntimeError(
"Missing database configuration: set DATABASE_URL or provide "
f"granular variables ({', '.join(missing)})"
)
url = URL.create(
drivername=driver,
username=user,
password=password,
host=host,
port=int(port) if port else None,
database=database,
)
if schema:
url = url.set(query={"options": f"-csearch_path={schema}"})
return str(url)
DATABASE_URL = _build_database_url()
engine = create_engine(DATABASE_URL, echo=True, future=True)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)

2
docs/architecture/01_introduction_and_goals.md
View File

@@ -38,7 +38,7 @@ Frontend components are server-rendered Jinja2 templates, with Chart.js powering
### Current implementation status (summary)
- Currency normalization, simulation scaffold, and reporting service exist; see [quickstart](docs/quickstart.md) for full status and migration instructions.
- Currency normalization, simulation scaffold, and reporting service exist; see [quickstart](../quickstart.md) for full status and migration instructions.
## MVP Features (migrated)

4
docs/architecture/04_solution_strategy/04_01_implementation_plan_20251020.md
View File

@@ -6,7 +6,7 @@ This file contains the implementation plan (MVP features, steps, and estimates).
1. Connect to PostgreSQL database with schema `calminer`.
1. Create and activate a virtual environment and install dependencies via `requirements.txt`.
1. Define environment variables in `.env`, including `DATABASE_URL`.
1. Define database environment variables in `.env` (e.g., `DATABASE_DRIVER`, `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME`, optional `DATABASE_SCHEMA`).
1. Configure FastAPI entrypoint in `main.py` to include routers.
## Feature: Scenario Management
@@ -107,4 +107,4 @@ This file contains the implementation plan (MVP features, steps, and estimates).
1. Write unit tests in `tests/unit/test_reporting.py`.
1. Enhance UI in `components/Dashboard.html` with charts.
See [UI and Style](docs/architecture/13_ui_and_style.md) for the UI template audit, layout guidance, and next steps.
See [UI and Style](../13_ui_and_style.md) for the UI template audit, layout guidance, and next steps.

8
docs/architecture/05_building_block_view.md
View File

@@ -8,13 +8,13 @@ status: draft
## Architecture overview
This overview complements [architecture](docs/architecture/README.md) with a high-level map of CalMiner's module layout and request flow.
This overview complements [architecture](README.md) with a high-level map of CalMiner's module layout and request flow.
Refer to the detailed architecture chapters in `docs/architecture/`:
- Module map & components: [Building Block View](docs/architecture/05_building_block_view.md)
- Request flow & runtime interactions: [Runtime View](docs/architecture/06_runtime_view.md)
- Simulation roadmap & strategy: [Solution Strategy](docs/architecture/04_solution_strategy.md)
- Module map & components: [Building Block View](05_building_block_view.md)
- Request flow & runtime interactions: [Runtime View](06_runtime_view.md)
- Simulation roadmap & strategy: [Solution Strategy](04_solution_strategy.md)
## System Components

6
docs/architecture/07_deployment_view.md
View File

@@ -64,8 +64,8 @@ The Docker-based deployment path aligns with the solution strategy documented in
### Image Build
- The multi-stage `Dockerfile` installs dependencies in a builder layer (including system compilers and Python packages) and copies only the required runtime artifacts to the final image.
- Build arguments are minimal; environment configuration (e.g., `DATABASE_URL`) is supplied at runtime. Secrets and configuration should be passed via environment variables or an orchestrator.
- The resulting image exposes port `8000` and starts `uvicorn main:app` (s. [README.md](../README.md)).
- Build arguments are minimal; database configuration is supplied at runtime via granular variables (`DATABASE_DRIVER`, `DATABASE_HOST`, `DATABASE_PORT`, `DATABASE_USER`, `DATABASE_PASSWORD`, `DATABASE_NAME`, optional `DATABASE_SCHEMA`). Secrets and configuration should be passed via environment variables or an orchestrator.
- The resulting image exposes port `8000` and starts `uvicorn main:app` (s. [README.md](../../README.md)).
### Runtime Environment
@@ -79,7 +79,7 @@ The Docker-based deployment path aligns with the solution strategy documented in
- `test.yml` executes the pytest suite using cached pip dependencies.
- `build-and-push.yml` logs into the container registry, rebuilds the Docker image using GitHub Actions cache-backed layers, and pushes `latest` (and additional tags as required).
- `deploy.yml` connects to the target host via SSH, pulls the pushed tag, stops any existing container, and launches the new version.
- Required secrets: `GITEA_REGISTRY`, `GITEA_USERNAME`, `GITEA_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
- Required secrets: `REGISTRY_URL`, `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
- Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [14 — Testing & CI](14_testing_ci.md) up to date.
## Integrations and Future Work (deployment-related)

2
docs/architecture/14_testing_ci.md
View File

@@ -24,7 +24,7 @@ CalMiner uses a combination of unit, integration, and end-to-end tests to ensure
- `test.yml` runs on every push with cached Python dependencies via `actions/cache@v3`.
- `build-and-push.yml` builds the Docker image with `docker/build-push-action@v2`, reusing GitHub Actions cache-backed layers, and pushes to the Gitea registry.
- `deploy.yml` connects to the target host (via `appleboy/ssh-action`) to pull the freshly pushed image and restart the container.
- Mandatory secrets: `GITEA_USERNAME`, `GITEA_PASSWORD`, `GITEA_REGISTRY`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
- Mandatory secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
- Run tests on pull requests to shared branches; enforce coverage target ≥80% (pytest-cov).
### Running Tests

24
docs/architecture/15_development_setup.md
View File

@@ -10,7 +10,7 @@ This document outlines the local development environment and steps to get the pr
## Clone and Project Setup
```powershell
````powershell
# Clone the repository
git clone https://git.allucanget.biz/allucanget/calminer.git
cd calminer
@@ -36,28 +36,34 @@ pip install -r requirements.txt
```sql
CREATE USER calminer_user WITH PASSWORD 'your_password';
```
````
1. Create database:
```sql
````sql
CREATE DATABASE calminer;
```python
## Environment Variables
1. Copy `.env.example` to `.env` at project root.
1. Edit `.env` to set database connection string:
1. Edit `.env` to set database connection details:
```dotenv
DATABASE_URL=postgresql://<user>:<password>@localhost:5432/calminer
```
DATABASE_DRIVER=postgresql
DATABASE_HOST=localhost
DATABASE_PORT=5432
DATABASE_USER=calminer_user
DATABASE_PASSWORD=your_password
DATABASE_NAME=calminer
DATABASE_SCHEMA=public
````
1. The application uses `python-dotenv` to load these variables.
1. The application uses `python-dotenv` to load these variables. A legacy `DATABASE_URL` value is still accepted if the granular keys are omitted.
## Running the Application
```powershell
````powershell
# Start the FastAPI server
uvicorn main:app --reload
```python
@@ -66,6 +72,6 @@ uvicorn main:app --reload
```powershell
pytest
```
````
E2E tests use Playwright and a session-scoped `live_server` fixture that starts the app at `http://localhost:8001` for browser-driven tests.

36
docs/quickstart.md
View File

@@ -34,7 +34,15 @@ docker build -t calminer:latest .
docker run --rm -p 8000:8000 calminer:latest
# Supply environment variables (e.g., Postgres connection)
docker run --rm -p 8000:8000 -e DATABASE_URL="postgresql://user:pass@host/db" calminer:latest
docker run --rm -p 8000:8000 ^
-e DATABASE_DRIVER="postgresql" ^
-e DATABASE_HOST="db.host" ^
-e DATABASE_PORT="5432" ^
-e DATABASE_USER="calminer" ^
-e DATABASE_PASSWORD="s3cret" ^
-e DATABASE_NAME="calminer" ^
-e DATABASE_SCHEMA="public" ^
calminer:latest
```
If you maintain a Postgres or Redis dependency locally, consider authoring a `docker compose` stack that pairs them with the app container. The Docker image expects the database to be reachable and migrations executed before serving traffic.
@@ -66,15 +74,23 @@ The project includes a referential `currency` table and migration/backfill tooli
### Run migrations and backfill (development)
Ensure `DATABASE_URL` is set in your PowerShell session to point at a development Postgres instance.
Configure the granular database settings in your PowerShell session before running migrations.
```powershell
$env:DATABASE_URL = 'postgresql://user:pass@host/db'
$env:DATABASE_DRIVER = 'postgresql'
$env:DATABASE_HOST = 'localhost'
$env:DATABASE_PORT = '5432'
$env:DATABASE_USER = 'calminer'
$env:DATABASE_PASSWORD = 's3cret'
$env:DATABASE_NAME = 'calminer'
$env:DATABASE_SCHEMA = 'public'
python scripts/run_migrations.py
python scripts/backfill_currency.py --dry-run
python scripts/backfill_currency.py --create-missing
```
> ℹ️ The application still accepts `DATABASE_URL` as a fallback if the granular variables are not set.
Use `--dry-run` first to verify what will change.
## Database Objects
@@ -91,10 +107,10 @@ The database contains tables such as `capex`, `opex`, `chemical_consumption`, `f
## Where to look next
- Architecture overview & chapters: [architecture](docs/architecture/README.md) (per-chapter files under `docs/architecture/`)
- [Testing & CI](docs/architecture/14_testing_ci.md)
- [Development setup](docs/architecture/15_development_setup.md)
- Implementation plan & roadmap: [Solution strategy](docs/architecture/04_solution_strategy_extended.md)
- Routes: [routes](routes/)
- Services: [services](services/)
- Scripts: [scripts](scripts/) (migrations and backfills)
- Architecture overview & chapters: [architecture](architecture/README.md) (per-chapter files under `docs/architecture/`)
- [Testing & CI](architecture/14_testing_ci.md)
- [Development setup](architecture/15_development_setup.md)
- Implementation plan & roadmap: [Solution strategy](architecture/04_solution_strategy.md)
- Routes: [routes](../routes/)
- Services: [services](../services/)
- Scripts: [scripts](../scripts/) (migrations and backfills)

37
scripts/backfill_currency.py
View File

@@ -6,21 +6,34 @@ Usage:
python scripts/backfill_currency.py --create-missing
This script is intentionally cautious: it defaults to dry-run mode and will refuse to run
if DATABASE_URL is not set. It supports creating missing currency rows when `--create-missing`
if database connection settings are missing. It supports creating missing currency rows when `--create-missing`
is provided. Always run against a development/staging database first.
"""
from __future__ import annotations
import os
import argparse
import importlib
import sys
from pathlib import Path
from sqlalchemy import text, create_engine
def load_env_dburl() -> str:
db = os.environ.get("DATABASE_URL")
if not db:
PROJECT_ROOT = Path(__file__).resolve().parent.parent
if str(PROJECT_ROOT) not in sys.path:
sys.path.insert(0, str(PROJECT_ROOT))
def load_database_url() -> str:
try:
db_module = importlib.import_module("config.database")
except RuntimeError as exc:
raise RuntimeError(
"DATABASE_URL not set — set it to your dev/staging DB before running this script")
return db
"Database configuration missing: set DATABASE_URL or provide granular "
"variables (DATABASE_DRIVER, DATABASE_HOST, DATABASE_PORT, DATABASE_USER, "
"DATABASE_PASSWORD, DATABASE_NAME, optional DATABASE_SCHEMA)."
) from exc
return getattr(db_module, "DATABASE_URL")
def backfill(db_url: str, dry_run: bool = True, create_missing: bool = False) -> None:
@@ -41,12 +54,12 @@ def backfill(db_url: str, dry_run: bool = True, create_missing: bool = False) ->
# insert and return id
conn.execute(text("INSERT INTO currency (code, name, symbol, is_active) VALUES (:c, :n, NULL, TRUE)"), {
"c": code, "n": code})
if db_url.startswith('sqlite:'):
r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), {
"code": code}).fetchone()
else:
r2 = conn.execute(text("SELECT id FROM currency WHERE code = :code"), {
"code": code}).fetchone()
if not r2:
raise RuntimeError(
f"Unable to determine currency ID for '{code}' after insert"
)
return r2[0]
return None
@@ -95,7 +108,7 @@ def main() -> None:
help="Create missing currency rows in the currency table")
args = parser.parse_args()
db = load_env_dburl()
db = load_database_url()
backfill(db, dry_run=args.dry_run, create_missing=args.create_missing)

40
tests/e2e/conftest.py
View File

@@ -1,12 +1,14 @@
import os
import subprocess
import time
from typing import Generator
from typing import Dict, Generator
import pytest
# type: ignore[import]
from playwright.sync_api import Browser, Page, Playwright, sync_playwright
import httpx
from sqlalchemy.engine import make_url
# Use a different port for the test server to avoid conflicts
TEST_PORT = 8001
@@ -16,6 +18,8 @@ BASE_URL = f"http://localhost:{TEST_PORT}"
@pytest.fixture(scope="session", autouse=True)
def live_server() -> Generator[str, None, None]:
"""Launch a live test server in a separate process."""
env = _prepare_database_environment(os.environ.copy())
process = subprocess.Popen(
[
"uvicorn",
@@ -26,7 +30,7 @@ def live_server() -> Generator[str, None, None]:
],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
env=os.environ.copy(),
env=env,
)
deadline = time.perf_counter() + 30
@@ -85,3 +89,35 @@ def page(browser: Browser, live_server: str) -> Generator[Page, None, None]:
page.wait_for_load_state("networkidle")
yield page
page.close()
def _prepare_database_environment(env: Dict[str, str]) -> Dict[str, str]:
"""Ensure granular database env vars are available for the app under test."""
required = ("DATABASE_HOST", "DATABASE_USER", "DATABASE_NAME")
if all(env.get(key) for key in required):
return env
legacy_url = env.get("DATABASE_URL")
if not legacy_url:
return env
url = make_url(legacy_url)
env.setdefault("DATABASE_DRIVER", url.drivername)
if url.host:
env.setdefault("DATABASE_HOST", url.host)
if url.port:
env.setdefault("DATABASE_PORT", str(url.port))
if url.username:
env.setdefault("DATABASE_USER", url.username)
if url.password:
env.setdefault("DATABASE_PASSWORD", url.password)
if url.database:
env.setdefault("DATABASE_NAME", url.database)
query_options = dict(url.query) if url.query else {}
options = query_options.get("options")
if isinstance(options, str) and "search_path=" in options:
env.setdefault("DATABASE_SCHEMA", options.split("search_path=")[-1])
return env