feat: Update CI workflows for Docker image build and deployment, enhance test configurations, and add testing documentation

.gitea/workflows/build-and-push.yml

@@ -1,11 +1,16 @@
 name: Build and Push Docker Image
 on:
-  push:
+  workflow_run:
+    workflows:
+      - Run Tests
     branches:
       - main
+    types:
+      - completed
 
 jobs:
   build-and-push:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     env:
       DEFAULT_BRANCH: main
@@ -14,6 +19,8 @@ jobs:
       REGISTRY_URL: ${{ secrets.REGISTRY_URL }}
       REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
       REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+      WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
+      WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -26,6 +33,14 @@ jobs:
           event_name="${GITHUB_EVENT_NAME:-}"
           sha="${GITHUB_SHA:-}"
 
+          if [ -z "$ref_name" ] && [ -n "${WORKFLOW_RUN_HEAD_BRANCH:-}" ]; then
+            ref_name="${WORKFLOW_RUN_HEAD_BRANCH}"
+          fi
+
+          if [ -z "$sha" ] && [ -n "${WORKFLOW_RUN_HEAD_SHA:-}" ]; then
+            sha="${WORKFLOW_RUN_HEAD_SHA}"
+          fi
+
           if [ "$ref_name" = "${DEFAULT_BRANCH:-main}" ]; then
             echo "on_default=true" >> "$GITHUB_OUTPUT"
           else

.gitea/workflows/deploy.yml

@@ -1,11 +1,16 @@
 name: Deploy to Server
 on:
-  push:
+  workflow_run:
+    workflows:
+      - Build and Push Docker Image
     branches:
       - main
+    types:
+      - completed
 
 jobs:
   deploy:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
     runs-on: ubuntu-latest
     env:
       DEFAULT_BRANCH: main
@@ -14,6 +19,8 @@ jobs:
       REGISTRY_URL: ${{ secrets.REGISTRY_URL }}
       REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
       REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
+      WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
+      WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
     steps:
       - name: SSH and deploy
         uses: appleboy/ssh-action@master
@@ -22,7 +29,15 @@ jobs:
           username: ${{ secrets.SSH_USERNAME }}
           key: ${{ secrets.SSH_PRIVATE_KEY }}
           script: |
-            docker pull ${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}:latest
+            IMAGE_SHA="${{ env.WORKFLOW_RUN_HEAD_SHA }}"
+            IMAGE_PATH="${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}"
+
+            if [ -z "$IMAGE_SHA" ]; then
+              echo "Missing workflow run head SHA; aborting deployment." >&2
+              exit 1
+            fi
+
+            docker pull "$IMAGE_PATH:$IMAGE_SHA"
             docker stop calminer || true
             docker rm calminer || true
             docker run -d --name calminer -p 8000:8000 \
@@ -33,4 +48,4 @@ jobs:
               -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
               -e DATABASE_NAME=${{ secrets.DATABASE_NAME }} \
               -e DATABASE_SCHEMA=${{ secrets.DATABASE_SCHEMA }} \
-              ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest
+              "$IMAGE_PATH:$IMAGE_SHA"

.gitea/workflows/test.yml

@@ -2,7 +2,13 @@ name: Run Tests
 on: [push]
 
 jobs:
-  test:
+  tests:
+    name: ${{ matrix.target }} tests
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [unit, e2e]
     services:
       postgres:
         image: postgres:16-alpine
@@ -10,14 +16,11 @@ jobs:
           POSTGRES_DB: calminer_ci
           POSTGRES_USER: calminer
           POSTGRES_PASSWORD: secret
-        ports:
-          - 5432:5432
         options: >-
           --health-cmd "pg_isready -U calminer -d calminer_ci"
           --health-interval 10s
           --health-timeout 5s
           --health-retries 10
-    runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -54,6 +57,7 @@ jobs:
           pip install -r requirements.txt
           pip install -r requirements-test.txt
       - name: Install Playwright browsers
+        if: ${{ matrix.target == 'e2e' }}
         run: |
           python -m playwright install --with-deps
       - name: Wait for database service
@@ -122,4 +126,9 @@ jobs:
         env:
           DATABASE_URL: postgresql+psycopg2://calminer:secret@postgres:5432/calminer_ci
           DATABASE_SCHEMA: public
-        run: pytest
+        run: |
+          if [ "${{ matrix.target }}" = "unit" ]; then
+            pytest tests/unit
+          else
+            pytest tests/e2e
+          fi

.gitignore

@@ -45,3 +45,6 @@ logs/
 # SQLite database
 *.sqlite3
 test*.db
+
+# Docker files
+.runner

docs/architecture/07_deployment/07_01_testing_ci.md

@@ -21,9 +21,9 @@ CalMiner uses a combination of unit, integration, and end-to-end tests to ensure
 ### CI/CD
 
 - Use Gitea Actions for CI/CD; workflows live under `.gitea/workflows/`.
-- `test.yml` runs on every push, provisions a temporary Postgres 16 service, waits for readiness, executes the setup script in dry-run and live modes, installs Playwright browsers, and finally runs the full pytest suite.
-- `build-and-push.yml` builds the Docker image with `docker/build-push-action@v2`, reusing GitHub Actions cache-backed layers, and pushes to the Gitea registry.
-- `deploy.yml` connects to the target host (via `appleboy/ssh-action`) to pull the freshly pushed image and restart the container.
+- `test.yml` runs on every push, provisions a temporary Postgres 16 service, waits for readiness, executes the setup script in dry-run and live modes, then fans out into parallel matrix jobs for unit (`pytest tests/unit`) and end-to-end (`pytest tests/e2e`) suites. Playwright browsers install only for the E2E job.
+- `build-and-push.yml` runs only after the **Run Tests** workflow finishes successfully (triggered via `workflow_run` on `main`). Once tests pass, it builds the Docker image with `docker/build-push-action@v2`, reuses cache-backed layers, and pushes to the Gitea registry.
+- `deploy.yml` runs only after the build workflow reports success on `main`. It connects to the target host (via `appleboy/ssh-action`), pulls the Docker image tagged with the build commit SHA, and restarts the container with that exact image reference.
 - Mandatory secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
 - Run tests on pull requests to shared branches; enforce coverage target ≥80% (pytest-cov).
 

docs/architecture/07_deployment_view.md

@@ -18,30 +18,34 @@ The CalMiner application is deployed using a multi-tier architecture consisting
 
 ```mermaid
 graph TD
-    A[Client Layer<br/>(Web Browsers)] --> B[Web Application Layer<br/>(FastAPI)]
-    B --> C[Database Layer<br/>(PostgreSQL)]
+    A[Client Layer] --> B[Web Application Layer]
+    B --> C[Database Layer]
 ```
 
 ## Infrastructure Components
 
 The infrastructure components for the application include:
 
-- **Web Server**: Hosts the FastAPI application and serves API endpoints.
-- **Database Server**: PostgreSQL database for persisting application data.
-- **Static File Server**: Serves static assets such as CSS, JavaScript, and image files.
 - **Reverse Proxy (optional)**: An Nginx or Apache server can be used as a reverse proxy.
 - **Containerization**: Docker images are generated via the repository `Dockerfile`, using a multi-stage build to keep the final runtime minimal.
 - **CI/CD Pipeline**: Automated pipelines (Gitea Actions) run tests, build/push Docker images, and trigger deployments.
+  - **Gitea Actions Workflows**: Located under `.gitea/workflows/`, these workflows handle testing, building, pushing, and deploying the application.
+  - **Gitea Action Runners**: Self-hosted runners execute the CI/CD workflows.
+  - **Testing and Continuous Integration**: Automated tests ensure code quality before deployment, also documented in [Testing & CI](07_deployment/07_01_testing_ci.md.md).
+- **Docker Infrastructure**: Docker is used to containerize the application for consistent deployment across environments.
+- **Portainer**: Production deployment environment for managing Docker containers.
+  - **Web Server**: Hosts the FastAPI application and serves API endpoints.
+  - **Database Server**: PostgreSQL database for persisting application data.
+  - **Static File Server**: Serves static assets such as CSS, JavaScript, and image files.
 - **Cloud Infrastructure (optional)**: The application can be deployed on cloud platforms.
 
 ```mermaid
 graph TD
-    A[Web Server] --> B[Database Server]
-    A --> C[Static File Server]
-    A --> D[Reverse Proxy]
-    A --> E[Containerization]
-    A --> F[CI/CD Pipeline]
-    A --> G[Cloud Infrastructure]
+    W[Web Server] --> DB[Database Server]
+    W --> S[Static File Server]
+    P[Reverse Proxy] --> W
+    C[CI/CD Pipeline] --> W
+    F[Containerization] --> W
 ```
 
 ## Environments
@@ -74,7 +78,7 @@ The production environment is set up for serving live traffic and includes:
 
 ## Containerized Deployment Flow
 
-The Docker-based deployment path aligns with the solution strategy documented in [04 — Solution Strategy](04_solution_strategy.md) and the CI practices captured in [14 — Testing & CI](14_testing_ci.md).
+The Docker-based deployment path aligns with the solution strategy documented in [Solution Strategy](04_solution_strategy.md) and the CI practices captured in [Testing & CI](07_deployment/07_01_testing_ci.md.md).
 
 ### Image Build
 
@@ -95,7 +99,7 @@ The Docker-based deployment path aligns with the solution strategy documented in
 - `build-and-push.yml` logs into the container registry, rebuilds the Docker image using GitHub Actions cache-backed layers, and pushes `latest` (and additional tags as required).
 - `deploy.yml` connects to the target host via SSH, pulls the pushed tag, stops any existing container, and launches the new version.
 - Required secrets: `REGISTRY_URL`, `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`.
-- Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [14 — Testing & CI](14_testing_ci.md) up to date.
+- Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [Testing & CI](07_deployment/07_01_testing_ci.md.md) up to date.
 
 ## Integrations and Future Work (deployment-related)
 

docs/architecture/README.md

@@ -16,11 +16,11 @@ This folder mirrors the arc42 chapter structure (adapted to Markdown).
 - [05 Building Block View](05_building_block_view.md)
 - [06 Runtime View](06_runtime_view.md)
 - [07 Deployment View](07_deployment_view.md)
+  - [Testing & CI](07_deployment/07_01_testing_ci.md.md)
 - [08 Concepts](08_concepts.md)
 - [09 Architecture Decisions](09_architecture_decisions.md)
 - [10 Quality Requirements](10_quality_requirements.md)
 - [11 Technical Risks](11_technical_risks.md)
 - [12 Glossary](12_glossary.md)
 - [13 UI and Style](13_ui_and_style.md)
-- [14 Testing & CI](14_testing_ci.md)
 - [15 Development Setup](15_development_setup.md)

docs/quickstart.md

@@ -245,7 +245,7 @@ The database contains tables such as `capex`, `opex`, `chemical_consumption`, `f
 ## Where to look next
 
 - Architecture overview & chapters: [architecture](architecture/README.md) (per-chapter files under `docs/architecture/`)
-- [Testing & CI](architecture/14_testing_ci.md)
+- [Testing & CI](architecture/07_deployment/07_01_testing_ci.md.md)
 - [Development setup](architecture/15_development_setup.md)
 - Implementation plan & roadmap: [Solution strategy](architecture/04_solution_strategy.md)
 - Routes: [routes](../routes/)