From 0550928a2fdb2212ef5e923132e7ea41de62a53b Mon Sep 17 00:00:00 2001 From: zwitschi Date: Sat, 25 Oct 2025 21:28:49 +0200 Subject: [PATCH 1/2] feat: Update CI workflows for Docker image build and deployment, enhance test configurations, and add testing documentation --- .gitea/workflows/build-and-push.yml | 17 ++++++++++- .gitea/workflows/deploy.yml | 21 +++++++++++-- .gitea/workflows/test.yml | 19 ++++++++---- .gitignore | 3 ++ .../07_01_testing_ci.md} | 6 ++-- docs/architecture/07_deployment_view.md | 30 +++++++++++-------- docs/architecture/README.md | 2 +- docs/quickstart.md | 2 +- 8 files changed, 73 insertions(+), 27 deletions(-) rename docs/architecture/{14_testing_ci.md => 07_deployment/07_01_testing_ci.md} (86%) diff --git a/.gitea/workflows/build-and-push.yml b/.gitea/workflows/build-and-push.yml index a2246a0..cab1385 100644 --- a/.gitea/workflows/build-and-push.yml +++ b/.gitea/workflows/build-and-push.yml @@ -1,11 +1,16 @@ name: Build and Push Docker Image on: - push: + workflow_run: + workflows: + - Run Tests branches: - main + types: + - completed jobs: build-and-push: + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: DEFAULT_BRANCH: main @@ -14,6 +19,8 @@ jobs: REGISTRY_URL: ${{ secrets.REGISTRY_URL }} REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} steps: - name: Checkout code uses: actions/checkout@v4 @@ -26,6 +33,14 @@ jobs: event_name="${GITHUB_EVENT_NAME:-}" sha="${GITHUB_SHA:-}" + if [ -z "$ref_name" ] && [ -n "${WORKFLOW_RUN_HEAD_BRANCH:-}" ]; then + ref_name="${WORKFLOW_RUN_HEAD_BRANCH}" + fi + + if [ -z "$sha" ] && [ -n "${WORKFLOW_RUN_HEAD_SHA:-}" ]; then + sha="${WORKFLOW_RUN_HEAD_SHA}" + fi + if [ "$ref_name" = "${DEFAULT_BRANCH:-main}" ]; then echo "on_default=true" >> "$GITHUB_OUTPUT" else diff --git a/.gitea/workflows/deploy.yml b/.gitea/workflows/deploy.yml index ddc1b02..6566347 100644 --- a/.gitea/workflows/deploy.yml +++ b/.gitea/workflows/deploy.yml @@ -1,11 +1,16 @@ name: Deploy to Server on: - push: + workflow_run: + workflows: + - Build and Push Docker Image branches: - main + types: + - completed jobs: deploy: + if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ubuntu-latest env: DEFAULT_BRANCH: main @@ -14,6 +19,8 @@ jobs: REGISTRY_URL: ${{ secrets.REGISTRY_URL }} REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }} REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }} + WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} + WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }} steps: - name: SSH and deploy uses: appleboy/ssh-action@master @@ -22,7 +29,15 @@ jobs: username: ${{ secrets.SSH_USERNAME }} key: ${{ secrets.SSH_PRIVATE_KEY }} script: | - docker pull ${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}:latest + IMAGE_SHA="${{ env.WORKFLOW_RUN_HEAD_SHA }}" + IMAGE_PATH="${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}" + + if [ -z "$IMAGE_SHA" ]; then + echo "Missing workflow run head SHA; aborting deployment." >&2 + exit 1 + fi + + docker pull "$IMAGE_PATH:$IMAGE_SHA" docker stop calminer || true docker rm calminer || true docker run -d --name calminer -p 8000:8000 \ @@ -33,4 +48,4 @@ jobs: -e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \ -e DATABASE_NAME=${{ secrets.DATABASE_NAME }} \ -e DATABASE_SCHEMA=${{ secrets.DATABASE_SCHEMA }} \ - ${{ secrets.REGISTRY_URL }}/${{ secrets.REGISTRY_USERNAME }}/calminer:latest + "$IMAGE_PATH:$IMAGE_SHA" diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index cd2b2bf..6d7a4bc 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -2,7 +2,13 @@ name: Run Tests on: [push] jobs: - test: + tests: + name: ${{ matrix.target }} tests + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + target: [unit, e2e] services: postgres: image: postgres:16-alpine @@ -10,14 +16,11 @@ jobs: POSTGRES_DB: calminer_ci POSTGRES_USER: calminer POSTGRES_PASSWORD: secret - ports: - - 5432:5432 options: >- --health-cmd "pg_isready -U calminer -d calminer_ci" --health-interval 10s --health-timeout 5s --health-retries 10 - runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 @@ -54,6 +57,7 @@ jobs: pip install -r requirements.txt pip install -r requirements-test.txt - name: Install Playwright browsers + if: ${{ matrix.target == 'e2e' }} run: | python -m playwright install --with-deps - name: Wait for database service @@ -122,4 +126,9 @@ jobs: env: DATABASE_URL: postgresql+psycopg2://calminer:secret@postgres:5432/calminer_ci DATABASE_SCHEMA: public - run: pytest + run: | + if [ "${{ matrix.target }}" = "unit" ]; then + pytest tests/unit + else + pytest tests/e2e + fi diff --git a/.gitignore b/.gitignore index 1be0350..6422f19 100644 --- a/.gitignore +++ b/.gitignore @@ -45,3 +45,6 @@ logs/ # SQLite database *.sqlite3 test*.db + +# Docker files +.runner diff --git a/docs/architecture/14_testing_ci.md b/docs/architecture/07_deployment/07_01_testing_ci.md similarity index 86% rename from docs/architecture/14_testing_ci.md rename to docs/architecture/07_deployment/07_01_testing_ci.md index 91ad3a7..e1401f9 100644 --- a/docs/architecture/14_testing_ci.md +++ b/docs/architecture/07_deployment/07_01_testing_ci.md @@ -21,9 +21,9 @@ CalMiner uses a combination of unit, integration, and end-to-end tests to ensure ### CI/CD - Use Gitea Actions for CI/CD; workflows live under `.gitea/workflows/`. -- `test.yml` runs on every push, provisions a temporary Postgres 16 service, waits for readiness, executes the setup script in dry-run and live modes, installs Playwright browsers, and finally runs the full pytest suite. -- `build-and-push.yml` builds the Docker image with `docker/build-push-action@v2`, reusing GitHub Actions cache-backed layers, and pushes to the Gitea registry. -- `deploy.yml` connects to the target host (via `appleboy/ssh-action`) to pull the freshly pushed image and restart the container. +- `test.yml` runs on every push, provisions a temporary Postgres 16 service, waits for readiness, executes the setup script in dry-run and live modes, then fans out into parallel matrix jobs for unit (`pytest tests/unit`) and end-to-end (`pytest tests/e2e`) suites. Playwright browsers install only for the E2E job. +- `build-and-push.yml` runs only after the **Run Tests** workflow finishes successfully (triggered via `workflow_run` on `main`). Once tests pass, it builds the Docker image with `docker/build-push-action@v2`, reuses cache-backed layers, and pushes to the Gitea registry. +- `deploy.yml` runs only after the build workflow reports success on `main`. It connects to the target host (via `appleboy/ssh-action`), pulls the Docker image tagged with the build commit SHA, and restarts the container with that exact image reference. - Mandatory secrets: `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `REGISTRY_URL`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. - Run tests on pull requests to shared branches; enforce coverage target ≥80% (pytest-cov). diff --git a/docs/architecture/07_deployment_view.md b/docs/architecture/07_deployment_view.md index 2c9b171..9619741 100644 --- a/docs/architecture/07_deployment_view.md +++ b/docs/architecture/07_deployment_view.md @@ -18,30 +18,34 @@ The CalMiner application is deployed using a multi-tier architecture consisting ```mermaid graph TD - A[Client Layer
(Web Browsers)] --> B[Web Application Layer
(FastAPI)] - B --> C[Database Layer
(PostgreSQL)] + A[Client Layer] --> B[Web Application Layer] + B --> C[Database Layer] ``` ## Infrastructure Components The infrastructure components for the application include: -- **Web Server**: Hosts the FastAPI application and serves API endpoints. -- **Database Server**: PostgreSQL database for persisting application data. -- **Static File Server**: Serves static assets such as CSS, JavaScript, and image files. - **Reverse Proxy (optional)**: An Nginx or Apache server can be used as a reverse proxy. - **Containerization**: Docker images are generated via the repository `Dockerfile`, using a multi-stage build to keep the final runtime minimal. - **CI/CD Pipeline**: Automated pipelines (Gitea Actions) run tests, build/push Docker images, and trigger deployments. + - **Gitea Actions Workflows**: Located under `.gitea/workflows/`, these workflows handle testing, building, pushing, and deploying the application. + - **Gitea Action Runners**: Self-hosted runners execute the CI/CD workflows. + - **Testing and Continuous Integration**: Automated tests ensure code quality before deployment, also documented in [Testing & CI](07_deployment/07_01_testing_ci.md.md). +- **Docker Infrastructure**: Docker is used to containerize the application for consistent deployment across environments. +- **Portainer**: Production deployment environment for managing Docker containers. + - **Web Server**: Hosts the FastAPI application and serves API endpoints. + - **Database Server**: PostgreSQL database for persisting application data. + - **Static File Server**: Serves static assets such as CSS, JavaScript, and image files. - **Cloud Infrastructure (optional)**: The application can be deployed on cloud platforms. ```mermaid graph TD - A[Web Server] --> B[Database Server] - A --> C[Static File Server] - A --> D[Reverse Proxy] - A --> E[Containerization] - A --> F[CI/CD Pipeline] - A --> G[Cloud Infrastructure] + W[Web Server] --> DB[Database Server] + W --> S[Static File Server] + P[Reverse Proxy] --> W + C[CI/CD Pipeline] --> W + F[Containerization] --> W ``` ## Environments @@ -74,7 +78,7 @@ The production environment is set up for serving live traffic and includes: ## Containerized Deployment Flow -The Docker-based deployment path aligns with the solution strategy documented in [04 — Solution Strategy](04_solution_strategy.md) and the CI practices captured in [14 — Testing & CI](14_testing_ci.md). +The Docker-based deployment path aligns with the solution strategy documented in [Solution Strategy](04_solution_strategy.md) and the CI practices captured in [Testing & CI](07_deployment/07_01_testing_ci.md.md). ### Image Build @@ -95,7 +99,7 @@ The Docker-based deployment path aligns with the solution strategy documented in - `build-and-push.yml` logs into the container registry, rebuilds the Docker image using GitHub Actions cache-backed layers, and pushes `latest` (and additional tags as required). - `deploy.yml` connects to the target host via SSH, pulls the pushed tag, stops any existing container, and launches the new version. - Required secrets: `REGISTRY_URL`, `REGISTRY_USERNAME`, `REGISTRY_PASSWORD`, `SSH_HOST`, `SSH_USERNAME`, `SSH_PRIVATE_KEY`. -- Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [14 — Testing & CI](14_testing_ci.md) up to date. +- Extend these workflows when introducing staging/blue-green deployments; keep cross-links with [Testing & CI](07_deployment/07_01_testing_ci.md.md) up to date. ## Integrations and Future Work (deployment-related) diff --git a/docs/architecture/README.md b/docs/architecture/README.md index 063cb38..2025345 100644 --- a/docs/architecture/README.md +++ b/docs/architecture/README.md @@ -16,11 +16,11 @@ This folder mirrors the arc42 chapter structure (adapted to Markdown). - [05 Building Block View](05_building_block_view.md) - [06 Runtime View](06_runtime_view.md) - [07 Deployment View](07_deployment_view.md) + - [Testing & CI](07_deployment/07_01_testing_ci.md.md) - [08 Concepts](08_concepts.md) - [09 Architecture Decisions](09_architecture_decisions.md) - [10 Quality Requirements](10_quality_requirements.md) - [11 Technical Risks](11_technical_risks.md) - [12 Glossary](12_glossary.md) - [13 UI and Style](13_ui_and_style.md) -- [14 Testing & CI](14_testing_ci.md) - [15 Development Setup](15_development_setup.md) diff --git a/docs/quickstart.md b/docs/quickstart.md index 428f245..4682375 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -245,7 +245,7 @@ The database contains tables such as `capex`, `opex`, `chemical_consumption`, `f ## Where to look next - Architecture overview & chapters: [architecture](architecture/README.md) (per-chapter files under `docs/architecture/`) -- [Testing & CI](architecture/14_testing_ci.md) +- [Testing & CI](architecture/07_deployment/07_01_testing_ci.md.md) - [Development setup](architecture/15_development_setup.md) - Implementation plan & roadmap: [Solution strategy](architecture/04_solution_strategy.md) - Routes: [routes](../routes/) From 70db34d088b47a0c478b954da7083f05894bd798 Mon Sep 17 00:00:00 2001 From: zwitschi Date: Sat, 25 Oct 2025 22:00:28 +0200 Subject: [PATCH 2/2] feat: Implement composite action for Python environment setup and refactor test workflow to utilize it --- .gitea/actions/setup-python-env/action.yml | 111 +++++++++++++++++ .gitea/workflows/test.yml | 115 +++--------------- .../07_deployment/07_01_testing_ci.md | 106 +++++++++++++++- 3 files changed, 229 insertions(+), 103 deletions(-) create mode 100644 .gitea/actions/setup-python-env/action.yml diff --git a/.gitea/actions/setup-python-env/action.yml b/.gitea/actions/setup-python-env/action.yml new file mode 100644 index 0000000..b85b21e --- /dev/null +++ b/.gitea/actions/setup-python-env/action.yml @@ -0,0 +1,111 @@ +name: Setup Python Environment +description: Configure Python, proxies, dependencies, and optional database setup for CI jobs. +author: CalMiner Team +inputs: + python-version: + description: Python version to install. + required: false + default: "3.10" + install-playwright: + description: Install Playwright browsers when true. + required: false + default: "false" + install-requirements: + description: Space-delimited list of requirement files to install. + required: false + default: "requirements.txt requirements-test.txt" + run-db-setup: + description: Run database wait and setup scripts when true. + required: false + default: "true" + db-dry-run: + description: Execute setup script dry run before live run when true. + required: false + default: "true" +runs: + using: composite + steps: + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - name: Configure apt proxy + shell: bash + run: | + set -euo pipefail + PROXY_HOST="http://apt-cacher:3142" + if ! curl -fsS --connect-timeout 3 "${PROXY_HOST}" >/dev/null; then + PROXY_HOST="http://192.168.88.14:3142" + fi + echo "Using APT proxy ${PROXY_HOST}" + { + echo "http_proxy=${PROXY_HOST}" + echo "https_proxy=${PROXY_HOST}" + echo "HTTP_PROXY=${PROXY_HOST}" + echo "HTTPS_PROXY=${PROXY_HOST}" + } >> "$GITHUB_ENV" + sudo tee /etc/apt/apt.conf.d/01proxy >/dev/null <&2 + exit 1 + fi + done + fi + - name: Install Playwright browsers + if: ${{ inputs.install-playwright == 'true' }} + shell: bash + run: | + set -euo pipefail + python -m playwright install --with-deps + - name: Wait for database service + if: ${{ inputs.run-db-setup == 'true' }} + shell: bash + run: | + set -euo pipefail + python - <<'PY' + import os + import time + + import psycopg2 + + dsn = ( + f"dbname={os.environ['DATABASE_SUPERUSER_DB']} " + f"user={os.environ['DATABASE_SUPERUSER']} " + f"password={os.environ['DATABASE_SUPERUSER_PASSWORD']} " + f"host={os.environ['DATABASE_HOST']} " + f"port={os.environ['DATABASE_PORT']}" + ) + + for attempt in range(30): + try: + with psycopg2.connect(dsn): + break + except psycopg2.OperationalError: + time.sleep(2) + else: + raise SystemExit("Postgres service did not become available") + PY + - name: Run database setup (dry run) + if: ${{ inputs.run-db-setup == 'true' && inputs.db-dry-run == 'true' }} + shell: bash + run: | + set -euo pipefail + python scripts/setup_database.py --ensure-database --ensure-role --ensure-schema --initialize-schema --run-migrations --seed-data --dry-run -v + - name: Run database setup + if: ${{ inputs.run-db-setup == 'true' }} + shell: bash + run: | + set -euo pipefail + python scripts/setup_database.py --ensure-database --ensure-role --ensure-schema --initialize-schema --run-migrations --seed-data -v diff --git a/.gitea/workflows/test.yml b/.gitea/workflows/test.yml index 6d7a4bc..53def95 100644 --- a/.gitea/workflows/test.yml +++ b/.gitea/workflows/test.yml @@ -5,6 +5,18 @@ jobs: tests: name: ${{ matrix.target }} tests runs-on: ubuntu-latest + env: + DATABASE_DRIVER: postgresql + DATABASE_HOST: postgres + DATABASE_PORT: "5432" + DATABASE_NAME: calminer_ci + DATABASE_USER: calminer + DATABASE_PASSWORD: secret + DATABASE_SCHEMA: public + DATABASE_SUPERUSER: calminer + DATABASE_SUPERUSER_PASSWORD: secret + DATABASE_SUPERUSER_DB: calminer_ci + DATABASE_URL: postgresql+psycopg2://calminer:secret@postgres:5432/calminer_ci strategy: fail-fast: false matrix: @@ -24,108 +36,11 @@ jobs: steps: - name: Checkout code uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 + - name: Prepare Python environment + uses: ./.gitea/actions/setup-python-env with: - python-version: "3.10" - - name: Configure apt proxy - run: | - set -euo pipefail - PROXY_HOST="http://apt-cacher:3142" - if ! curl -fsS --connect-timeout 3 "${PROXY_HOST}" >/dev/null; then - PROXY_HOST="http://192.168.88.14:3142" - fi - echo "Using APT proxy ${PROXY_HOST}" - echo "http_proxy=${PROXY_HOST}" >> "$GITHUB_ENV" - echo "https_proxy=${PROXY_HOST}" >> "$GITHUB_ENV" - echo "HTTP_PROXY=${PROXY_HOST}" >> "$GITHUB_ENV" - echo "HTTPS_PROXY=${PROXY_HOST}" >> "$GITHUB_ENV" - sudo tee /etc/apt/apt.conf.d/01proxy >/dev/null <.py @@ -45,7 +45,7 @@ tests/ test_.py fixtures/ conftest.py -```python +``` ### Fixtures and Test Data @@ -116,3 +116,103 @@ pytest tests/e2e/ --headed - Stop, remove, and relaunch the `calminer` container exposing port 8000. When adding new workflows, mirror this structure to ensure secrets, caching, and deployment steps remain aligned with the production environment. + +## CI Owner Coordination Notes + +### Key Findings + +- Self-hosted runner: ASUS System Product Name chassis with AMD Ryzen 7 7700X (8 physical cores / 16 threads) and 63.2 GB usable RAM; `act_runner` configuration not overridden, so only one workflow job runs concurrently today. +- Unit test matrix job: completes 117 pytest cases in roughly 4.1 seconds after Postgres spins up; Docker services consume ~150 MB for `postgres:16-alpine`, with minimal sustained CPU load once tests begin. +- End-to-end matrix job: `pytest tests/e2e` averages 21‑22 seconds of execution, but a cold run downloads ~179 MB of apt packages plus ~470 MB of Playwright browser bundles (Chromium, Firefox, WebKit, FFmpeg), exceeding 650 MB network transfer and adding several gigabytes of disk writes if caches are absent. +- Both jobs reuse existing Python package caches when available; absent a shared cache service, repeated Playwright installs remain the dominant cost driver for cold executions. + +### Open Questions + +- Can we raise the runner concurrency above the default single job, or provision an additional runner, so the test matrix can execute without serializing queued workflows? +- Is there a central cache or artifact service available for Python wheels and Playwright browser bundles to avoid ~650 MB downloads on cold starts? +- Are we permitted to bake Playwright browsers into the base runner image, or should we pursue a shared cache/proxy solution instead? + +### Outreach Draft + +```text +Subject: CalMiner CI parallelization support + +Hi , + +We recently updated the CalMiner test workflow to fan out unit and Playwright E2E suites in parallel. While validating the change, we gathered the following: + +- Runner host: ASUS System Product Name with AMD Ryzen 7 7700X (8 cores / 16 threads), ~63 GB RAM, default `act_runner` concurrency (1 job at a time). +- Unit job finishes in ~4.1 s once Postgres is ready; light CPU and network usage. +- E2E job finishes in ~22 s, but a cold run pulls ~179 MB of apt packages plus ~470 MB of Playwright browser payloads (>650 MB download, several GB disk writes) because we do not have a shared cache yet. + +To move forward, could you help with the following? + +1. Confirm whether we can raise the runner concurrency limit or provision an additional runner so parallel jobs do not queue behind one another. +2. Let us know if a central cache (Artifactory, Nexus, etc.) is available for Python wheels and Playwright browser bundles, or if we should consider baking the browsers into the runner image instead. +3. Share any guidance on preferred caching or proxy solutions for large binary installs on self-hosted runners. + +Once we have clarity, we can finalize the parallel rollout and update the documentation accordingly. + +Thanks, + +``` + +## Workflow Optimization Opportunities + +### `test.yml` + +- Run the apt-proxy setup once via a composite action or preconfigured runner image if additional matrix jobs are added. +- Collapse dependency installation into a single `pip install -r requirements-test.txt` call (includes base requirements) once caching is restored. +- Investigate caching or pre-baking Playwright browser binaries to eliminate >650 MB cold downloads per run. + +### `build-and-push.yml` + +- Skip QEMU setup or explicitly constrain Buildx to linux/amd64 to reduce startup time. +- Enable `cache-from` / `cache-to` settings (registry or `type=gha`) to reuse Docker build layers between runs. + +### `deploy.yml` + +- Extract deployment script into a reusable shell script or compose file to minimize inline secrets and ease multi-environment scaling. +- Add a post-deploy health check (e.g., `curl` readiness probe) before declaring success. + +### Priority Overview + +1. Restore shared caching for Python wheels and Playwright browsers once infrastructure exposes the cache service (highest impact on runtime and bandwidth; requires coordination with CI owners). +2. Enable Docker layer caching in `build-and-push.yml` to shorten build cycles (medium effort, immediate benefit to release workflows). +3. Add post-deploy health verification to `deploy.yml` (low effort, improves confidence in automation). +4. Streamline redundant setup steps in `test.yml` (medium effort once cache strategy is in place; consider composite actions or base image updates). + +### Setup Consolidation Opportunities + +- `Run Tests` matrix jobs each execute the apt proxy configuration, pip installs, database wait, and setup scripts. A composite action or shell script wrapper could centralize these routines and parameterize target-specific behavior (unit vs e2e) to avoid copy/paste maintenance as additional jobs (lint, type check) are introduced. +- Both the test and build workflows perform a `checkout` step; while unavoidable per workflow, shared git submodules or sparse checkout rules could be encapsulated in a composite action to keep options consistent. +- The database setup script currently runs twice (dry-run and live) for every matrix leg. Evaluate whether the dry-run remains necessary once migrations stabilize; if retained, consider adding an environment variable toggle to skip redundant seed operations for read-only suites (e.g., lint). + +### Proposed Shared Setup Action + +- Location: `.gitea/actions/setup-python-env/action.yml` (composite action). +- Inputs: + - `python-version` (default `3.10`): forwarded to `actions/setup-python`. + - `install-playwright` (default `false`): when `true`, run `python -m playwright install --with-deps`. + - `install-requirements` (default `requirements.txt requirements-test.txt`): space-delimited list pip installs iterate over. + - `run-db-setup` (default `true`): toggles database wait + setup scripts. + - `db-dry-run` (default `true`): controls whether the dry-run invocation executes. +- Steps encapsulated: + 1. Set up Python via `actions/setup-python@v5` using provided version. + 2. Configure apt proxy via shared shell snippet (with graceful fallback when proxy offline). + 3. Iterate over requirement files and execute `pip install -r `. + 4. If `install-playwright == true`, install browsers. + 5. If `run-db-setup == true`, run the wait-for-Postgres python snippet and call `scripts/setup_database.py`, honoring `db-dry-run` toggle. +- Usage sketch (in `test.yml`): + +```yaml + - name: Prepare Python environment + uses: ./.gitea/actions/setup-python-env + with: + install-playwright: ${{ matrix.target == 'e2e' }} + db-dry-run: true +``` + +- Benefits: centralizes proxy logic and dependency installs, reduces duplication across matrix jobs, and keeps future lint/type-check jobs lightweight by disabling database setup. +- Implementation status: action available at `.gitea/actions/setup-python-env` and consumed by `test.yml`; extend to additional workflows as they adopt the shared routine. +- Obsolete steps removed: individual apt proxy, dependency install, Playwright, and database setup commands pruned from `test.yml` once the composite action was integrated.