fix: container/compose/CI overhaul
This commit is contained in:
@@ -1,167 +0,0 @@
|
|||||||
name: Setup Python Environment
|
|
||||||
description: Configure Python, proxies, dependencies, and optional database setup for CI jobs.
|
|
||||||
author: CalMiner Team
|
|
||||||
inputs:
|
|
||||||
python-version:
|
|
||||||
description: Python version to install.
|
|
||||||
required: false
|
|
||||||
default: '3.10'
|
|
||||||
use-system-python:
|
|
||||||
description: Skip setup-python and rely on the system Python already available in the environment.
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
install-playwright:
|
|
||||||
description: Install Playwright browsers when true.
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
install-requirements:
|
|
||||||
description: Space-delimited list of requirement files to install.
|
|
||||||
required: false
|
|
||||||
default: 'requirements.txt requirements-test.txt'
|
|
||||||
run-db-setup:
|
|
||||||
description: Run database wait and setup scripts when true.
|
|
||||||
required: false
|
|
||||||
default: 'true'
|
|
||||||
db-dry-run:
|
|
||||||
description: Execute setup script dry run before live run when true.
|
|
||||||
required: false
|
|
||||||
default: 'true'
|
|
||||||
create-venv:
|
|
||||||
description: Create an isolated virtual environment when using the system Python.
|
|
||||||
required: false
|
|
||||||
default: 'false'
|
|
||||||
runs:
|
|
||||||
using: composite
|
|
||||||
steps:
|
|
||||||
- name: Set up Python
|
|
||||||
if: ${{ inputs.use-system-python != 'true' }}
|
|
||||||
uses: actions/setup-python@v5
|
|
||||||
with:
|
|
||||||
python-version: ${{ inputs.python-version }}
|
|
||||||
|
|
||||||
- name: Verify system Python
|
|
||||||
if: ${{ inputs.use-system-python == 'true' }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
if ! command -v python >/dev/null 2>&1; then
|
|
||||||
echo "Python executable not found on PATH" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
python --version
|
|
||||||
python -m pip --version >/dev/null 2>&1 || python -m ensurepip --upgrade
|
|
||||||
python -m pip --version
|
|
||||||
- name: Create virtual environment
|
|
||||||
if: ${{ inputs.use-system-python == 'true' && inputs.create-venv == 'true' }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
if [ -z "${RUNNER_TEMP:-}" ]; then
|
|
||||||
echo "RUNNER_TEMP is not set; cannot create virtual environment" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
VENV_PATH="$(mktemp -d "${RUNNER_TEMP%/}/ci-venv-XXXXXX")"
|
|
||||||
python -m venv "${VENV_PATH}"
|
|
||||||
PATH_ENTRY=""
|
|
||||||
if [ -f "${VENV_PATH}/bin/activate" ]; then
|
|
||||||
PATH_ENTRY="${VENV_PATH}/bin"
|
|
||||||
elif [ -f "${VENV_PATH}/Scripts/activate" ]; then
|
|
||||||
PATH_ENTRY="${VENV_PATH}/Scripts"
|
|
||||||
else
|
|
||||||
echo "Unable to locate virtual environment scripts" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
export PATH="${PATH_ENTRY}:${PATH}"
|
|
||||||
echo "${PATH_ENTRY}" >> "${GITHUB_PATH}"
|
|
||||||
echo "VIRTUAL_ENV=${VENV_PATH}" >> "${GITHUB_ENV}"
|
|
||||||
# Re-evaluate the python binary for subsequent steps
|
|
||||||
python --version
|
|
||||||
python -m pip --version
|
|
||||||
- name: Configure apt proxy
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
PROXY_HOST="http://apt-cacher:3142"
|
|
||||||
if ! curl -fsS --connect-timeout 3 "${PROXY_HOST}" >/dev/null; then
|
|
||||||
PROXY_HOST="http://192.168.88.14:3142"
|
|
||||||
fi
|
|
||||||
echo "Using APT proxy ${PROXY_HOST}"
|
|
||||||
{
|
|
||||||
echo "http_proxy=${PROXY_HOST}"
|
|
||||||
echo "https_proxy=${PROXY_HOST}"
|
|
||||||
echo "HTTP_PROXY=${PROXY_HOST}"
|
|
||||||
echo "HTTPS_PROXY=${PROXY_HOST}"
|
|
||||||
} >> "$GITHUB_ENV"
|
|
||||||
if command -v sudo >/dev/null 2>&1; then
|
|
||||||
printf 'Acquire::http::Proxy "%s";\nAcquire::https::Proxy "%s";\n' "${PROXY_HOST}" "${PROXY_HOST}" | sudo tee /etc/apt/apt.conf.d/01proxy >/dev/null
|
|
||||||
elif [ "$(id -u)" -eq 0 ]; then
|
|
||||||
printf 'Acquire::http::Proxy "%s";\nAcquire::https::Proxy "%s";\n' "${PROXY_HOST}" "${PROXY_HOST}" > /etc/apt/apt.conf.d/01proxy
|
|
||||||
else
|
|
||||||
echo "Skipping /etc/apt/apt.conf.d/01proxy update; sudo/root not available" >&2
|
|
||||||
fi
|
|
||||||
- name: Install dependencies
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
requirements="${{ inputs.install-requirements }}"
|
|
||||||
if [ -n "${requirements}" ]; then
|
|
||||||
for requirement in ${requirements}; do
|
|
||||||
if [ -f "${requirement}" ]; then
|
|
||||||
python -m pip install -r "${requirement}"
|
|
||||||
else
|
|
||||||
echo "Requirement file ${requirement} not found" >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
done
|
|
||||||
fi
|
|
||||||
- name: Install Playwright browsers
|
|
||||||
if: ${{ inputs.install-playwright == 'true' }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
python -m playwright install --with-deps
|
|
||||||
- name: Wait for database service
|
|
||||||
if: ${{ inputs.run-db-setup == 'true' }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
python - <<'PY'
|
|
||||||
import os
|
|
||||||
import time
|
|
||||||
|
|
||||||
import psycopg2
|
|
||||||
|
|
||||||
dsn = (
|
|
||||||
f"dbname={os.environ['DATABASE_SUPERUSER_DB']} "
|
|
||||||
f"user={os.environ['DATABASE_SUPERUSER']} "
|
|
||||||
f"password={os.environ['DATABASE_SUPERUSER_PASSWORD']} "
|
|
||||||
f"host={os.environ['DATABASE_HOST']} "
|
|
||||||
f"port={os.environ['DATABASE_PORT']}"
|
|
||||||
)
|
|
||||||
|
|
||||||
max_attempts = 30
|
|
||||||
for attempt in range(max_attempts):
|
|
||||||
try:
|
|
||||||
with psycopg2.connect(dsn):
|
|
||||||
break
|
|
||||||
except psycopg2.OperationalError as exc:
|
|
||||||
print(
|
|
||||||
f"Attempt {attempt + 1}/{max_attempts} failed to connect to Postgres: {exc}",
|
|
||||||
flush=True,
|
|
||||||
)
|
|
||||||
time.sleep(2)
|
|
||||||
else:
|
|
||||||
raise SystemExit("Postgres service did not become available")
|
|
||||||
PY
|
|
||||||
- name: Run database setup (dry run)
|
|
||||||
if: ${{ inputs.run-db-setup == 'true' && inputs.db-dry-run == 'true' }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
python scripts/setup_database.py --ensure-database --ensure-role --ensure-schema --initialize-schema --run-migrations --seed-data --dry-run -v
|
|
||||||
- name: Run database setup
|
|
||||||
if: ${{ inputs.run-db-setup == 'true' }}
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
python scripts/setup_database.py --ensure-database --ensure-role --ensure-schema --initialize-schema --run-migrations --seed-data -v
|
|
||||||
@@ -1,102 +0,0 @@
|
|||||||
name: Build and Push Docker Image
|
|
||||||
on:
|
|
||||||
workflow_run:
|
|
||||||
workflows:
|
|
||||||
- Run E2E Tests
|
|
||||||
types:
|
|
||||||
- completed
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
build-and-push:
|
|
||||||
if: ${{ github.event_name != 'workflow_run' || (
|
|
||||||
github.event.workflow_run.conclusion == 'success' && (
|
|
||||||
github.event.workflow_run.head_branch == 'main' ||
|
|
||||||
github.event.workflow_run.head_branch == 'refs/heads/main' ||
|
|
||||||
(
|
|
||||||
!github.event.workflow_run.head_branch && (
|
|
||||||
github.event.workflow_run.repository.default_branch == 'main' ||
|
|
||||||
github.event.workflow_run.repository.default_branch == 'refs/heads/main' ||
|
|
||||||
github.event.repository.default_branch == 'main' ||
|
|
||||||
github.event.repository.default_branch == 'refs/heads/main'
|
|
||||||
)
|
|
||||||
)
|
|
||||||
) }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
env:
|
|
||||||
DEFAULT_BRANCH: main
|
|
||||||
REGISTRY_ORG: allucanget
|
|
||||||
REGISTRY_IMAGE_NAME: calminer
|
|
||||||
REGISTRY_URL: ${{ secrets.REGISTRY_URL }}
|
|
||||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
|
||||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
|
||||||
WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
|
||||||
WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
||||||
WORKFLOW_RUN_REPO_DEFAULT_BRANCH: ${{ github.event.workflow_run.repository.default_branch }}
|
|
||||||
REPOSITORY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Collect workflow metadata
|
|
||||||
id: meta
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
ref_name="${GITHUB_REF_NAME:-${GITHUB_REF##*/}}"
|
|
||||||
event_name="${GITHUB_EVENT_NAME:-}"
|
|
||||||
sha="${GITHUB_SHA:-}"
|
|
||||||
|
|
||||||
if [ -z "$ref_name" ] && [ -n "${WORKFLOW_RUN_HEAD_BRANCH:-}" ]; then
|
|
||||||
ref_name="${WORKFLOW_RUN_HEAD_BRANCH}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$ref_name" ] && [ -n "${WORKFLOW_RUN_REPO_DEFAULT_BRANCH:-}" ]; then
|
|
||||||
ref_name="${WORKFLOW_RUN_REPO_DEFAULT_BRANCH}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$ref_name" ] && [ -n "${REPOSITORY_DEFAULT_BRANCH:-}" ]; then
|
|
||||||
ref_name="${REPOSITORY_DEFAULT_BRANCH}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$sha" ] && [ -n "${WORKFLOW_RUN_HEAD_SHA:-}" ]; then
|
|
||||||
sha="${WORKFLOW_RUN_HEAD_SHA}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [[ "$ref_name" == refs/heads/* ]]; then
|
|
||||||
ref_name="${ref_name#refs/heads/}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ "$ref_name" = "${DEFAULT_BRANCH:-main}" ]; then
|
|
||||||
echo "on_default=true" >> "$GITHUB_OUTPUT"
|
|
||||||
else
|
|
||||||
echo "on_default=false" >> "$GITHUB_OUTPUT"
|
|
||||||
fi
|
|
||||||
|
|
||||||
echo "ref_name=$ref_name" >> "$GITHUB_OUTPUT"
|
|
||||||
echo "event_name=$event_name" >> "$GITHUB_OUTPUT"
|
|
||||||
echo "sha=$sha" >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
- name: Set up QEMU and Buildx
|
|
||||||
uses: docker/setup-buildx-action@v3
|
|
||||||
with:
|
|
||||||
install: false
|
|
||||||
|
|
||||||
- name: Log in to Gitea registry
|
|
||||||
if: ${{ steps.meta.outputs.on_default == 'true' }}
|
|
||||||
uses: docker/login-action@v3
|
|
||||||
continue-on-error: true
|
|
||||||
with:
|
|
||||||
registry: ${{ env.REGISTRY_URL }}
|
|
||||||
username: ${{ env.REGISTRY_USERNAME }}
|
|
||||||
password: ${{ env.REGISTRY_PASSWORD }}
|
|
||||||
|
|
||||||
- name: Build and push Docker image
|
|
||||||
uses: docker/build-push-action@v4
|
|
||||||
with:
|
|
||||||
context: .
|
|
||||||
file: Dockerfile
|
|
||||||
push: ${{ steps.meta.outputs.on_default == 'true' && steps.meta.outputs.event_name != 'pull_request' && (env.REGISTRY_URL != '' && env.REGISTRY_USERNAME != '' && env.REGISTRY_PASSWORD != '') }}
|
|
||||||
tags: |
|
|
||||||
${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}:latest
|
|
||||||
${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}:${{ steps.meta.outputs.sha }}
|
|
||||||
cache-from: type=gha
|
|
||||||
cache-to: type=gha,mode=max
|
|
||||||
@@ -1,99 +0,0 @@
|
|||||||
name: Deploy to Server
|
|
||||||
on:
|
|
||||||
workflow_run:
|
|
||||||
workflows:
|
|
||||||
- Build and Push Docker Image
|
|
||||||
types:
|
|
||||||
- completed
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
deploy:
|
|
||||||
if: ${{ github.event_name != 'workflow_run' || (
|
|
||||||
github.event.workflow_run.conclusion == 'success' && (
|
|
||||||
github.event.workflow_run.head_branch == 'main' ||
|
|
||||||
github.event.workflow_run.head_branch == 'refs/heads/main' ||
|
|
||||||
(
|
|
||||||
!github.event.workflow_run.head_branch && (
|
|
||||||
github.event.workflow_run.repository.default_branch == 'main' ||
|
|
||||||
github.event.workflow_run.repository.default_branch == 'refs/heads/main' ||
|
|
||||||
github.event.repository.default_branch == 'main' ||
|
|
||||||
github.event.repository.default_branch == 'refs/heads/main'
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
) }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
env:
|
|
||||||
DEFAULT_BRANCH: main
|
|
||||||
REGISTRY_ORG: allucanget
|
|
||||||
REGISTRY_IMAGE_NAME: calminer
|
|
||||||
REGISTRY_URL: ${{ secrets.REGISTRY_URL }}
|
|
||||||
REGISTRY_USERNAME: ${{ secrets.REGISTRY_USERNAME }}
|
|
||||||
REGISTRY_PASSWORD: ${{ secrets.REGISTRY_PASSWORD }}
|
|
||||||
WORKFLOW_RUN_HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }}
|
|
||||||
WORKFLOW_RUN_HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
|
||||||
WORKFLOW_RUN_REPO_DEFAULT_BRANCH: ${{ github.event.workflow_run.repository.default_branch }}
|
|
||||||
REPOSITORY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
|
|
||||||
steps:
|
|
||||||
- name: SSH and deploy
|
|
||||||
uses: appleboy/ssh-action@master
|
|
||||||
with:
|
|
||||||
host: ${{ secrets.SSH_HOST }}
|
|
||||||
username: ${{ secrets.SSH_USERNAME }}
|
|
||||||
key: ${{ secrets.SSH_PRIVATE_KEY }}
|
|
||||||
script: |
|
|
||||||
IMAGE_SHA="${{ env.WORKFLOW_RUN_HEAD_SHA }}"
|
|
||||||
FALLBACK_BRANCH="${{ env.WORKFLOW_RUN_HEAD_BRANCH }}"
|
|
||||||
IMAGE_TAG="${IMAGE_SHA}"
|
|
||||||
IMAGE_PATH="${{ env.REGISTRY_URL }}/${{ env.REGISTRY_ORG }}/${{ env.REGISTRY_IMAGE_NAME }}"
|
|
||||||
|
|
||||||
if [ -z "$FALLBACK_BRANCH" ]; then
|
|
||||||
FALLBACK_BRANCH="${{ env.WORKFLOW_RUN_REPO_DEFAULT_BRANCH }}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$FALLBACK_BRANCH" ]; then
|
|
||||||
FALLBACK_BRANCH="${{ env.REPOSITORY_DEFAULT_BRANCH }}"
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$IMAGE_TAG" ] && [ -n "$FALLBACK_BRANCH" ]; then
|
|
||||||
case "$FALLBACK_BRANCH" in
|
|
||||||
refs/heads/*)
|
|
||||||
FALLBACK_BRANCH="${FALLBACK_BRANCH#refs/heads/}"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
|
|
||||||
if [ "$FALLBACK_BRANCH" = "${DEFAULT_BRANCH:-main}" ]; then
|
|
||||||
IMAGE_TAG="latest"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -z "$IMAGE_TAG" ]; then
|
|
||||||
echo "Missing workflow run head SHA and no default-branch fallback available; aborting deployment." >&2
|
|
||||||
exit 1
|
|
||||||
fi
|
|
||||||
|
|
||||||
docker pull "$IMAGE_PATH:$IMAGE_TAG"
|
|
||||||
docker stop calminer || true
|
|
||||||
docker rm calminer || true
|
|
||||||
docker run -d --name calminer -p 8000:8000 \
|
|
||||||
-e DATABASE_DRIVER=${{ secrets.DATABASE_DRIVER }} \
|
|
||||||
-e DATABASE_HOST=${{ secrets.DATABASE_HOST }} \
|
|
||||||
-e DATABASE_PORT=${{ secrets.DATABASE_PORT }} \
|
|
||||||
-e DATABASE_USER=${{ secrets.DATABASE_USER }} \
|
|
||||||
-e DATABASE_PASSWORD=${{ secrets.DATABASE_PASSWORD }} \
|
|
||||||
-e DATABASE_NAME=${{ secrets.DATABASE_NAME }} \
|
|
||||||
-e DATABASE_SCHEMA=${{ secrets.DATABASE_SCHEMA }} \
|
|
||||||
"$IMAGE_PATH:$IMAGE_TAG"
|
|
||||||
|
|
||||||
for attempt in {1..10}; do
|
|
||||||
if curl -fsS http://localhost:8000/health >/dev/null; then
|
|
||||||
echo "Deployment health check passed"
|
|
||||||
exit 0
|
|
||||||
fi
|
|
||||||
echo "Health check attempt ${attempt} failed; retrying in 3s"
|
|
||||||
sleep 3
|
|
||||||
done
|
|
||||||
|
|
||||||
echo "Deployment health check failed after retries" >&2
|
|
||||||
docker logs calminer >&2 || true
|
|
||||||
exit 1
|
|
||||||
@@ -1,114 +0,0 @@
|
|||||||
name: Run E2E Tests
|
|
||||||
|
|
||||||
on:
|
|
||||||
push:
|
|
||||||
branches-ignore:
|
|
||||||
- main
|
|
||||||
- refs/heads/main
|
|
||||||
workflow_run:
|
|
||||||
workflows:
|
|
||||||
- Run Tests
|
|
||||||
types:
|
|
||||||
- completed
|
|
||||||
workflow_dispatch:
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
e2e:
|
|
||||||
name: E2E Tests
|
|
||||||
if: ${{ github.event_name == 'workflow_dispatch' ||
|
|
||||||
github.event_name == 'push' ||
|
|
||||||
(
|
|
||||||
github.event_name == 'workflow_run' &&
|
|
||||||
github.event.workflow_run.conclusion == 'success' &&
|
|
||||||
(
|
|
||||||
github.event.workflow_run.head_branch == 'main' ||
|
|
||||||
github.event.workflow_run.head_branch == 'refs/heads/main' ||
|
|
||||||
(
|
|
||||||
!github.event.workflow_run.head_branch &&
|
|
||||||
(
|
|
||||||
github.event.workflow_run.repository.default_branch == 'main' ||
|
|
||||||
github.event.workflow_run.repository.default_branch == 'refs/heads/main' ||
|
|
||||||
github.event.repository.default_branch == 'main' ||
|
|
||||||
github.event.repository.default_branch == 'refs/heads/main'
|
|
||||||
)
|
|
||||||
)
|
|
||||||
)
|
|
||||||
) }}
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
container: mcr.microsoft.com/playwright/python:v1.55.0-jammy
|
|
||||||
env:
|
|
||||||
DATABASE_DRIVER: postgresql
|
|
||||||
DATABASE_HOST: postgres
|
|
||||||
DATABASE_PORT: '5432'
|
|
||||||
DATABASE_NAME: calminer_ci
|
|
||||||
DATABASE_USER: calminer
|
|
||||||
DATABASE_PASSWORD: secret
|
|
||||||
DATABASE_SCHEMA: public
|
|
||||||
DATABASE_SUPERUSER: calminer
|
|
||||||
DATABASE_SUPERUSER_PASSWORD: secret
|
|
||||||
DATABASE_SUPERUSER_DB: calminer_ci
|
|
||||||
DATABASE_URL: postgresql+psycopg2://calminer:secret@postgres:5432/calminer_ci
|
|
||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:16
|
|
||||||
env:
|
|
||||||
POSTGRES_DB: calminer_ci
|
|
||||||
POSTGRES_USER: calminer
|
|
||||||
POSTGRES_PASSWORD: secret
|
|
||||||
options: >-
|
|
||||||
--health-cmd "pg_isready -U calminer -d calminer_ci"
|
|
||||||
--health-interval 10s
|
|
||||||
--health-timeout 5s
|
|
||||||
--health-retries 10
|
|
||||||
steps:
|
|
||||||
- name: Install Node.js runtime
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
export DEBIAN_FRONTEND=noninteractive
|
|
||||||
curl -fsSL https://deb.nodesource.com/setup_20.x | bash -
|
|
||||||
apt-get install -y nodejs
|
|
||||||
|
|
||||||
- name: Checkout code (workflow_run)
|
|
||||||
if: ${{ github.event_name == 'workflow_run' }}
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
with:
|
|
||||||
ref: ${{ github.event.workflow_run.head_sha }}
|
|
||||||
|
|
||||||
- name: Checkout code (manual)
|
|
||||||
if: ${{ github.event_name != 'workflow_run' }}
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Export PYTHONPATH
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
echo "PYTHONPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Prepare Python environment
|
|
||||||
uses: ./.gitea/actions/setup-python-env
|
|
||||||
with:
|
|
||||||
use-system-python: 'true'
|
|
||||||
install-playwright: 'true'
|
|
||||||
run-db-setup: 'true'
|
|
||||||
|
|
||||||
- name: Run e2e tests
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
mkdir -p artifacts/pytest
|
|
||||||
pytest tests/e2e --junitxml=artifacts/pytest/e2e-results.xml
|
|
||||||
|
|
||||||
- name: Upload pytest results
|
|
||||||
if: always()
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: e2e-pytest-results
|
|
||||||
path: artifacts/pytest/
|
|
||||||
|
|
||||||
- name: Upload Playwright artifacts
|
|
||||||
if: failure()
|
|
||||||
uses: actions/upload-artifact@v3
|
|
||||||
with:
|
|
||||||
name: playwright-artifacts
|
|
||||||
path: playwright-report
|
|
||||||
@@ -1,72 +0,0 @@
|
|||||||
name: Run Tests
|
|
||||||
on: [push]
|
|
||||||
|
|
||||||
jobs:
|
|
||||||
lint:
|
|
||||||
name: Lint
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Export PYTHONPATH
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
echo "PYTHONPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Prepare Python environment
|
|
||||||
uses: ./.gitea/actions/setup-python-env
|
|
||||||
with:
|
|
||||||
use-system-python: 'true'
|
|
||||||
run-db-setup: 'false'
|
|
||||||
create-venv: 'true'
|
|
||||||
|
|
||||||
- name: Run lint checks
|
|
||||||
run: ruff check .
|
|
||||||
|
|
||||||
unit:
|
|
||||||
name: Unit Tests
|
|
||||||
runs-on: ubuntu-latest
|
|
||||||
env:
|
|
||||||
DATABASE_DRIVER: postgresql
|
|
||||||
DATABASE_HOST: postgres
|
|
||||||
DATABASE_PORT: '5432'
|
|
||||||
DATABASE_NAME: calminer_ci
|
|
||||||
DATABASE_USER: calminer
|
|
||||||
DATABASE_PASSWORD: secret
|
|
||||||
DATABASE_SCHEMA: public
|
|
||||||
DATABASE_SUPERUSER: calminer
|
|
||||||
DATABASE_SUPERUSER_PASSWORD: secret
|
|
||||||
DATABASE_SUPERUSER_DB: calminer_ci
|
|
||||||
DATABASE_URL: postgresql+psycopg2://calminer:secret@postgres:5432/calminer_ci
|
|
||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:16
|
|
||||||
env:
|
|
||||||
POSTGRES_DB: calminer_ci
|
|
||||||
POSTGRES_USER: calminer
|
|
||||||
POSTGRES_PASSWORD: secret
|
|
||||||
options: >-
|
|
||||||
--health-cmd "pg_isready -U calminer -d calminer_ci"
|
|
||||||
--health-interval 10s
|
|
||||||
--health-timeout 5s
|
|
||||||
--health-retries 10
|
|
||||||
steps:
|
|
||||||
- name: Checkout code
|
|
||||||
uses: actions/checkout@v4
|
|
||||||
|
|
||||||
- name: Export PYTHONPATH
|
|
||||||
shell: bash
|
|
||||||
run: |
|
|
||||||
set -euo pipefail
|
|
||||||
echo "PYTHONPATH=${{ github.workspace }}" >> "$GITHUB_ENV"
|
|
||||||
|
|
||||||
- name: Prepare Python environment
|
|
||||||
uses: ./.gitea/actions/setup-python-env
|
|
||||||
with:
|
|
||||||
use-system-python: 'true'
|
|
||||||
create-venv: 'true'
|
|
||||||
|
|
||||||
- name: Run unit tests
|
|
||||||
run: pytest tests/unit
|
|
||||||
@@ -1,50 +0,0 @@
|
|||||||
services:
|
|
||||||
api:
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
command: uvicorn main:app --host 0.0.0.0 --port 8000 --reload
|
|
||||||
ports:
|
|
||||||
- "8000:8000"
|
|
||||||
environment:
|
|
||||||
- DATABASE_HOST=db
|
|
||||||
- DATABASE_PORT=5432
|
|
||||||
- DATABASE_USER=calminer
|
|
||||||
- DATABASE_PASSWORD=calminer
|
|
||||||
- DATABASE_NAME=calminer_dev
|
|
||||||
volumes:
|
|
||||||
- .:/app
|
|
||||||
depends_on:
|
|
||||||
db:
|
|
||||||
condition: service_healthy
|
|
||||||
networks:
|
|
||||||
- calminer_backend
|
|
||||||
|
|
||||||
db:
|
|
||||||
image: postgres:16
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
- POSTGRES_DB=calminer_dev
|
|
||||||
- POSTGRES_USER=calminer
|
|
||||||
- POSTGRES_PASSWORD=calminer
|
|
||||||
- LANG=en_US.UTF-8
|
|
||||||
- LC_ALL=en_US.UTF-8
|
|
||||||
- POSTGRES_INITDB_ARGS=--encoding=UTF8 --locale=en_US.UTF-8
|
|
||||||
ports:
|
|
||||||
- "5432:5432"
|
|
||||||
volumes:
|
|
||||||
- pg_data_dev:/var/lib/postgresql/data
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "pg_isready -U calminer -d calminer_dev"]
|
|
||||||
interval: 10s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 5
|
|
||||||
networks:
|
|
||||||
- calminer_backend
|
|
||||||
|
|
||||||
networks:
|
|
||||||
calminer_backend:
|
|
||||||
driver: bridge
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
pg_data_dev:
|
|
||||||
@@ -1,23 +0,0 @@
|
|||||||
version: "3.9"
|
|
||||||
|
|
||||||
services:
|
|
||||||
postgres:
|
|
||||||
image: postgres:16-alpine
|
|
||||||
container_name: calminer_postgres_local
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
POSTGRES_DB: calminer_local
|
|
||||||
POSTGRES_USER: calminer
|
|
||||||
POSTGRES_PASSWORD: secret
|
|
||||||
ports:
|
|
||||||
- "5433:5432"
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "pg_isready -U calminer -d calminer_local"]
|
|
||||||
interval: 10s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 10
|
|
||||||
volumes:
|
|
||||||
- postgres_data:/var/lib/postgresql/data
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
postgres_data:
|
|
||||||
@@ -1,130 +0,0 @@
|
|||||||
services:
|
|
||||||
api:
|
|
||||||
image: ${CALMINER_IMAGE:-calminer-api:latest}
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
restart: unless-stopped
|
|
||||||
env_file:
|
|
||||||
- config/setup_production.env
|
|
||||||
environment:
|
|
||||||
UVICORN_WORKERS: ${UVICORN_WORKERS:-2}
|
|
||||||
UVICORN_LOG_LEVEL: ${UVICORN_LOG_LEVEL:-info}
|
|
||||||
command:
|
|
||||||
[
|
|
||||||
"sh",
|
|
||||||
"-c",
|
|
||||||
"uvicorn main:app --host 0.0.0.0 --port 8000 --workers ${UVICORN_WORKERS:-2} --log-level ${UVICORN_LOG_LEVEL:-info}",
|
|
||||||
]
|
|
||||||
ports:
|
|
||||||
- "${CALMINER_API_PORT:-8000}:8000"
|
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpus: ${API_LIMIT_CPUS:-1.0}
|
|
||||||
memory: ${API_LIMIT_MEMORY:-1g}
|
|
||||||
reservations:
|
|
||||||
memory: ${API_RESERVATION_MEMORY:-512m}
|
|
||||||
healthcheck:
|
|
||||||
test:
|
|
||||||
- "CMD-SHELL"
|
|
||||||
- 'python -c "import urllib.request; urllib.request.urlopen(''http://127.0.0.1:8000/health'').read()"'
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 5
|
|
||||||
start_period: 30s
|
|
||||||
networks:
|
|
||||||
- calminer_backend
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "10m"
|
|
||||||
max-file: "3"
|
|
||||||
labels:
|
|
||||||
- "traefik.enable=true"
|
|
||||||
- "traefik.http.routers.calminer.rule=Host(`${CALMINER_DOMAIN}`)"
|
|
||||||
- "traefik.http.routers.calminer.entrypoints=websecure"
|
|
||||||
- "traefik.http.routers.calminer.tls.certresolver=letsencrypt"
|
|
||||||
- "traefik.http.services.calminer.loadbalancer.server.port=8000"
|
|
||||||
|
|
||||||
traefik:
|
|
||||||
image: traefik:v3.1
|
|
||||||
restart: unless-stopped
|
|
||||||
command:
|
|
||||||
- "--providers.docker=true"
|
|
||||||
- "--providers.docker.exposedbydefault=false"
|
|
||||||
- "--entrypoints.web.address=:80"
|
|
||||||
- "--entrypoints.websecure.address=:443"
|
|
||||||
- "--certificatesresolvers.letsencrypt.acme.tlschallenge=true"
|
|
||||||
- "--certificatesresolvers.letsencrypt.acme.email=${TRAEFIK_ACME_EMAIL:?TRAEFIK_ACME_EMAIL not set}"
|
|
||||||
- "--certificatesresolvers.letsencrypt.acme.storage=/letsencrypt/acme.json"
|
|
||||||
ports:
|
|
||||||
- "80:80"
|
|
||||||
- "443:443"
|
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpus: ${TRAEFIK_LIMIT_CPUS:-0.5}
|
|
||||||
memory: ${TRAEFIK_LIMIT_MEMORY:-512m}
|
|
||||||
volumes:
|
|
||||||
- /var/run/docker.sock:/var/run/docker.sock:ro
|
|
||||||
- traefik_letsencrypt:/letsencrypt
|
|
||||||
networks:
|
|
||||||
- calminer_backend
|
|
||||||
profiles:
|
|
||||||
- reverse-proxy
|
|
||||||
healthcheck:
|
|
||||||
test:
|
|
||||||
- "CMD"
|
|
||||||
- "traefik"
|
|
||||||
- "healthcheck"
|
|
||||||
- "--entrypoints=web"
|
|
||||||
- "--entrypoints=websecure"
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 5
|
|
||||||
|
|
||||||
postgres:
|
|
||||||
image: postgres:16
|
|
||||||
profiles:
|
|
||||||
- local-db
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
POSTGRES_DB: ${POSTGRES_DB:-calminer}
|
|
||||||
POSTGRES_USER: ${POSTGRES_USER:-calminer}
|
|
||||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-changeme}
|
|
||||||
LANG: en_US.UTF-8
|
|
||||||
LC_ALL: en_US.UTF-8
|
|
||||||
POSTGRES_INITDB_ARGS: --encoding=UTF8 --locale=en_US.UTF-8
|
|
||||||
ports:
|
|
||||||
- "${CALMINER_DB_PORT:-5432}:5432"
|
|
||||||
deploy:
|
|
||||||
resources:
|
|
||||||
limits:
|
|
||||||
cpus: ${POSTGRES_LIMIT_CPUS:-1.0}
|
|
||||||
memory: ${POSTGRES_LIMIT_MEMORY:-2g}
|
|
||||||
reservations:
|
|
||||||
memory: ${POSTGRES_RESERVATION_MEMORY:-1g}
|
|
||||||
volumes:
|
|
||||||
- pg_data_prod:/var/lib/postgresql/data
|
|
||||||
- ./backups:/backups
|
|
||||||
healthcheck:
|
|
||||||
test:
|
|
||||||
[
|
|
||||||
"CMD-SHELL",
|
|
||||||
"pg_isready -U ${POSTGRES_USER:-calminer} -d ${POSTGRES_DB:-calminer}",
|
|
||||||
]
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 5
|
|
||||||
networks:
|
|
||||||
- calminer_backend
|
|
||||||
|
|
||||||
networks:
|
|
||||||
calminer_backend:
|
|
||||||
name: ${CALMINER_NETWORK:-calminer_backend}
|
|
||||||
driver: bridge
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
pg_data_prod:
|
|
||||||
traefik_letsencrypt:
|
|
||||||
@@ -1,82 +0,0 @@
|
|||||||
services:
|
|
||||||
tests:
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
command: >
|
|
||||||
sh -c "set -eu; pip install -r requirements-test.txt; python scripts/setup_database.py --ensure-database --ensure-role --ensure-schema --initialize-schema --run-migrations --seed-data --dry-run -v; python scripts/setup_database.py --ensure-database --ensure-role --ensure-schema --initialize-schema --run-migrations --seed-data -v; pytest $${PYTEST_TARGET:-tests/unit}"
|
|
||||||
environment:
|
|
||||||
DATABASE_DRIVER: postgresql
|
|
||||||
DATABASE_HOST: postgres
|
|
||||||
DATABASE_PORT: 5432
|
|
||||||
DATABASE_NAME: calminer_test
|
|
||||||
DATABASE_USER: calminer_test
|
|
||||||
DATABASE_PASSWORD: calminer_test_password
|
|
||||||
DATABASE_SCHEMA: public
|
|
||||||
DATABASE_SUPERUSER: postgres
|
|
||||||
DATABASE_SUPERUSER_PASSWORD: postgres
|
|
||||||
DATABASE_SUPERUSER_DB: postgres
|
|
||||||
DATABASE_URL: postgresql+psycopg2://calminer_test:calminer_test_password@postgres:5432/calminer_test
|
|
||||||
PYTEST_TARGET: tests/unit
|
|
||||||
PYTHONPATH: /app
|
|
||||||
depends_on:
|
|
||||||
postgres:
|
|
||||||
condition: service_healthy
|
|
||||||
volumes:
|
|
||||||
- .:/app
|
|
||||||
- pip_cache_test:/root/.cache/pip
|
|
||||||
networks:
|
|
||||||
- calminer_test
|
|
||||||
|
|
||||||
api:
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
command: uvicorn main:app --host 0.0.0.0 --port 8000 --reload
|
|
||||||
environment:
|
|
||||||
DATABASE_DRIVER: postgresql
|
|
||||||
DATABASE_HOST: postgres
|
|
||||||
DATABASE_PORT: 5432
|
|
||||||
DATABASE_NAME: calminer_test
|
|
||||||
DATABASE_USER: calminer_test
|
|
||||||
DATABASE_PASSWORD: calminer_test_password
|
|
||||||
DATABASE_SCHEMA: public
|
|
||||||
DATABASE_URL: postgresql+psycopg2://calminer_test:calminer_test_password@postgres:5432/calminer_test
|
|
||||||
PYTHONPATH: /app
|
|
||||||
depends_on:
|
|
||||||
postgres:
|
|
||||||
condition: service_healthy
|
|
||||||
ports:
|
|
||||||
- "8001:8000"
|
|
||||||
networks:
|
|
||||||
- calminer_test
|
|
||||||
|
|
||||||
postgres:
|
|
||||||
image: postgres:16
|
|
||||||
restart: unless-stopped
|
|
||||||
environment:
|
|
||||||
POSTGRES_DB: calminer_test
|
|
||||||
POSTGRES_USER: postgres
|
|
||||||
POSTGRES_PASSWORD: postgres
|
|
||||||
LANG: en_US.UTF-8
|
|
||||||
LC_ALL: en_US.UTF-8
|
|
||||||
POSTGRES_INITDB_ARGS: --encoding=UTF8 --locale=en_US.UTF-8
|
|
||||||
healthcheck:
|
|
||||||
test: ["CMD-SHELL", "pg_isready -U postgres -d calminer_test"]
|
|
||||||
interval: 10s
|
|
||||||
timeout: 5s
|
|
||||||
retries: 5
|
|
||||||
ports:
|
|
||||||
- "5433:5432"
|
|
||||||
volumes:
|
|
||||||
- pg_data_test:/var/lib/postgresql/data
|
|
||||||
networks:
|
|
||||||
- calminer_test
|
|
||||||
|
|
||||||
networks:
|
|
||||||
calminer_test:
|
|
||||||
driver: bridge
|
|
||||||
|
|
||||||
volumes:
|
|
||||||
pg_data_test:
|
|
||||||
pip_cache_test:
|
|
||||||
@@ -1,39 +0,0 @@
|
|||||||
services:
|
|
||||||
api:
|
|
||||||
image: ${CALMINER_IMAGE:-calminer-api:latest}
|
|
||||||
build:
|
|
||||||
context: .
|
|
||||||
dockerfile: Dockerfile
|
|
||||||
restart: unless-stopped
|
|
||||||
env_file:
|
|
||||||
- config/setup_production.env
|
|
||||||
environment:
|
|
||||||
UVICORN_WORKERS: ${UVICORN_WORKERS:-2}
|
|
||||||
UVICORN_LOG_LEVEL: ${UVICORN_LOG_LEVEL:-info}
|
|
||||||
command:
|
|
||||||
[
|
|
||||||
"sh",
|
|
||||||
"-c",
|
|
||||||
"uvicorn main:app --host 0.0.0.0 --port 8000 --workers ${UVICORN_WORKERS:-2} --log-level ${UVICORN_LOG_LEVEL:-info}",
|
|
||||||
]
|
|
||||||
ports:
|
|
||||||
- "${CALMINER_API_PORT:-8000}:8000"
|
|
||||||
healthcheck:
|
|
||||||
test:
|
|
||||||
- "CMD-SHELL"
|
|
||||||
- 'python -c "import urllib.request; urllib.request.urlopen(''http://127.0.0.1:8000/docs'').read()"'
|
|
||||||
interval: 30s
|
|
||||||
timeout: 10s
|
|
||||||
retries: 5
|
|
||||||
start_period: 30s
|
|
||||||
networks:
|
|
||||||
- calminer_backend
|
|
||||||
logging:
|
|
||||||
driver: json-file
|
|
||||||
options:
|
|
||||||
max-size: "10m"
|
|
||||||
max-file: "3"
|
|
||||||
|
|
||||||
networks:
|
|
||||||
calminer_backend:
|
|
||||||
driver: bridge
|
|
||||||
Reference in New Issue
Block a user