diff --git a/.gitea/workflows/build-and-push.yml b/.gitea/workflows/build-and-push.yml index a2246a0..cab1385 100644 --- a/.gitea/workflows/build-and-push.yml +++ b/.gitea/workflows/build-and-push.yml @@ -1,11 +1,16 @@ name: Build and Push Docker Image on: - push: + workflow_run: + workflows: + - Run Tests branches: - main + types: + - completed jobs: build-and-push: + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: DEFAULT_BRANCH: main @@ -14,6 +19,8 @@ jobs: REGISTRY_URL: ${{ secrets.REGISTRY_URL }} REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -26,6 +33,14 @@ jobs: event_name="${GITHUB_EVENT_NAME:-}" sha="${GITHUB_SHA:-}" + if [ -z "$ref_name" ] && [ -n "${WORKFLOW_RUN_HEAD_BRANCH:-}" ]; then + ref_name="${WORKFLOW_RUN_HEAD_BRANCH}" + fi + + if [ -z "$sha" ] && [ -n "${WORKFLOW_RUN_HEAD_SHA:-}" ]; then + sha="${WORKFLOW_RUN_HEAD_SHA}" + fi + if [ "$ref_name" = "${DEFAULT_BRANCH:-main}" ]; then echo "on_default=true" >> "$GITHUB_OUTPUT" else diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index ddc1b02..6566347 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -1,11 +1,16 @@ name: Deploy to Server on: - push: + workflow_run: + workflows: + - Build and Push Docker Image branches: - main + types: + - completed jobs: deploy: + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: DEFAULT_BRANCH: main @@ -14,6 +19,8 @@ jobs: REGISTRY_URL: ${{ secrets.REGISTRY_URL }} REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} steps: - name: SSH and deploy uses: appleboy/ssh-action@master @@ -22,7 +29,15 @@ jobs: username: ${{ secrets.SSH_USERNAME }} key: ${{ secrets.SSH_PRIVATE_KEY }} script: | - docker pull ${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}:latest + IMAGE_SHA="${{ env.WORKFLOW_RUN_HEAD_SHA }}" + IMAGE_PATH="${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}" + + if [ -z "$IMAGE_SHA" ]; then + echo "Missing workflow run head SHA; aborting deployment." >&2 + exit 1 + fi + + docker pull "$IMAGE_PATH:$IMAGE_SHA" docker stop calminer || true docker rm calminer || true docker run -d --name calminer -p 8000:8000 \ @@ -33,4 +48,4 @@ jobs: -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \ -e DATABASE_NAME=${{ secrets.DATABASE_NAME }} \ -e DATABASE_SCHEMA=${{ secrets.DATABASE_SCHEMA }} \ - ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest + "$IMAGE_PATH:$IMAGE_SHA" diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index cd2b2bf..6d7a4bc 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2,7 +2,13 @@ name: Run Tests on: [push] jobs: - test: + tests: + name: ${{ matrix.target }} tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: [unit, e2e] services: postgres: image: postgres:16-alpine @@ -10,14 +16,11 @@ jobs: POSTGRES_DB: calminer_ci POSTGRES_USER: calminer POSTGRES_PASSWORD: secret - ports: - - 5432:5432 options: >- --health-cmd "pg_isready -U calminer -d calminer_ci" --health-interval 10s --health-timeout 5s --health-retries 10 - runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 @@ -54,6 +57,7 @@ jobs: pip install -r requirements.txt pip install -r requirements-test.txt - name: Install Playwright browsers + if: ${{ matrix.target == 'e2e' }} run: | python -m playwright install --with-deps - name: Wait for database service @@ -122,4 +126,9 @@ jobs: env: DATABASE_URL: postgresql+psycopg2://calminer:secret@postgres:5432/calminer_ci DATABASE_SCHEMA: public - run: pytest + run: | + if [ "${{ matrix.target }}" = "unit" ]; then + pytest tests/unit + else + pytest tests/e2e + fi diff --git a/.gitignore b/.gitignore index 1be0350..6422f19 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ logs/ # SQLite database *.sqlite3 test*.db + +# Docker files +.runner diff --git a/docs/architecture/14_testing_ci.md b/docs/architecture/07_deployment/07_01_testing_ci.md similarity index 86% rename from docs/architecture/14_testing_ci.md rename to docs/architecture/07_deployment/07_01_testing_ci.md index 91ad3a7..e1401f9 100644 --- a/docs/architecture/14_testing_ci.md +++ b/docs/architecture/07_deployment/07_01_testing_ci.md @@ -21,9 +21,9 @@ CalMiner uses a combination of unit, integration, and end-to-end tests to ensure ### CI/CD - Use Gitea Actions for CI/CD; workflows live under `.gitea/workflows/`. -- `test.yml` runs on every push, provisions a temporary Postgres 16 service, waits for readiness, executes the setup script in dry-run and live modes, installs Playwright browsers, and finally runs the full pytest suite. -- `build-and-push.yml` builds the Docker image with `docker/build-push-action@v2`, reusing GitHub Actions cache-backed layers, and pushes to the Gitea registry. -- `deploy.yml` connects to the target host (via `appleboy/ssh-action`) to pull the freshly pushed image and restart the container. +- `test.yml` runs on every push, provisions a temporary Postgres 16 service, waits for readiness, executes the setup script in dry-run and live modes, then fans out into parallel matrix jobs for unit (`pytest tests/unit`) and end-to-end (`pytest tests/e2e`) suites. Playwright browsers install only for the E2E job. +- `build-and-push.yml` runs only after the **Run Tests** workflow finishes successfully (triggered via `workflow_run` on `main`). Once tests pass, it builds the Docker image with `docker/build-push-action@v2`, reuses cache-backed layers, and pushes to the Gitea registry. +- `deploy.yml` runs only after the build workflow reports success on `main`. It connects to the target host (via `appleboy/ssh-action`), pulls the Docker image tagged with the build commit SHA, and restarts the container with that exact image reference. - Mandatory secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. - Run tests on pull requests to shared branches; enforce coverage target ≥80% (pytest-cov). diff --git a/docs/architecture/07_deployment_view.md b/docs/architecture/07_deployment_view.md index 2c9b171..9619741 100644 --- a/docs/architecture/07_deployment_view.md +++ b/docs/architecture/07_deployment_view.md @@ -18,30 +18,34 @@ The CalMiner application is deployed using a multi-tier architecture consisting ```mermaid graph TD - A[Client Layer
(Web Browsers)] --> B[Web Application Layer
(FastAPI)] - B --> C[Database Layer
(PostgreSQL)] + A[Client Layer] --> B[Web Application Layer] + B --> C[Database Layer] ``` ## Infrastructure Components The infrastructure components for the application include: -- **Web Server**: Hosts the FastAPI application and serves API endpoints. -- **Database Server**: PostgreSQL database for persisting application data. -- **Static File Server**: Serves static assets such as CSS, JavaScript, and image files. - **Reverse Proxy (optional)**: An Nginx or Apache server can be used as a reverse proxy. - **Containerization**: Docker images are generated via the repository `Dockerfile`, using a multi-stage build to keep the final runtime minimal. - **CI/CD Pipeline**: Automated pipelines (Gitea Actions) run tests, build/push Docker images, and trigger deployments. + - **Gitea Actions Workflows**: Located under `.gitea/workflows/`, these workflows handle testing, building, pushing, and deploying the application. + - **Gitea Action Runners**: Self-hosted runners execute the CI/CD workflows. + - **Testing and Continuous Integration**: Automated tests ensure code quality before deployment, also documented in [Testing & CI](07_deployment/07_01_testing_ci.md.md). +- **Docker Infrastructure**: Docker is used to containerize the application for consistent deployment across environments. +- **Portainer**: Production deployment environment for managing Docker containers. + - **Web Server**: Hosts the FastAPI application and serves API endpoints. + - **Database Server**: PostgreSQL database for persisting application data. + - **Static File Server**: Serves static assets such as CSS, JavaScript, and image files. - **Cloud Infrastructure (optional)**: The application can be deployed on cloud platforms. ```mermaid graph TD - A[Web Server] --> B[Database Server] - A --> C[Static File Server] - A --> D[Reverse Proxy] - A --> E[Containerization] - A --> F[CI/CD Pipeline] - A --> G[Cloud Infrastructure] + W[Web Server] --> DB[Database Server] + W --> S[Static File Server] + P[Reverse Proxy] --> W + C[CI/CD Pipeline] --> W + F[Containerization] --> W ``` ## Environments @@ -74,7 +78,7 @@ The production environment is set up for serving live traffic and includes: ## Containerized Deployment Flow -The Docker-based deployment path aligns with the solution strategy documented in [04 — Solution Strategy](04_solution_strategy.md) and the CI practices captured in [14 — Testing & CI](14_testing_ci.md). +The Docker-based deployment path aligns with the solution strategy documented in [Solution Strategy](04_solution_strategy.md) and the CI practices captured in [Testing & CI](07_deployment/07_01_testing_ci.md.md). ### Image Build @@ -95,7 +99,7 @@ The Docker-based deployment path aligns with the solution strategy documented in - `build-and-push.yml` logs into the container registry, rebuilds the Docker image using GitHub Actions cache-backed layers, and pushes `latest` (and additional tags as required). - `deploy.yml` connects to the target host via SSH, pulls the pushed tag, stops any existing container, and launches the new version. - Required secrets: `REGISTRY_URL`, `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. -- Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [14 — Testing & CI](14_testing_ci.md) up to date. +- Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [Testing & CI](07_deployment/07_01_testing_ci.md.md) up to date. ## Integrations and Future Work (deployment-related) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 063cb38..2025345 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -16,11 +16,11 @@ This folder mirrors the arc42 chapter structure (adapted to Markdown). - [05 Building Block View](05_building_block_view.md) - [06 Runtime View](06_runtime_view.md) - [07 Deployment View](07_deployment_view.md) + - [Testing & CI](07_deployment/07_01_testing_ci.md.md) - [08 Concepts](08_concepts.md) - [09 Architecture Decisions](09_architecture_decisions.md) - [10 Quality Requirements](10_quality_requirements.md) - [11 Technical Risks](11_technical_risks.md) - [12 Glossary](12_glossary.md) - [13 UI and Style](13_ui_and_style.md) -- [14 Testing & CI](14_testing_ci.md) - [15 Development Setup](15_development_setup.md) diff --git a/docs/quickstart.md b/docs/quickstart.md index 428f245..4682375 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -245,7 +245,7 @@ The database contains tables such as `capex`, `opex`, `chemical_consumption`, `f ## Where to look next - Architecture overview & chapters: [architecture](architecture/README.md) (per-chapter files under `docs/architecture/`) -- [Testing & CI](architecture/14_testing_ci.md) +- [Testing & CI](architecture/07_deployment/07_01_testing_ci.md.md) - [Development setup](architecture/15_development_setup.md) - Implementation plan & roadmap: [Solution strategy](architecture/04_solution_strategy.md) - Routes: [routes](../routes/)