Add initial project structure with Docker, CI, and FastAPI setup

- Create Dockerfile and docker-compose.yml for containerization
- Add CI configuration for linting and testing
- Implement FastAPI application with health check routes
- Set up database schema using DuckDB
- Include environment configuration and secrets management
- Add README with project objectives and key features
- Implement logging setup and configuration
- Create initial tests for health route
- Add HTML templates for web interface

.dockerignore

@@ -0,0 +1,11 @@
+.git
+.github
+.venv
+__pycache__
+.pytest_cache
+.mypy_cache
+.ruff_cache
+data
+logs
+*.pyc
+.env

.env.example

@@ -0,0 +1,9 @@
+APP_ENV=dev
+APP_HOST=0.0.0.0
+APP_PORT=8000
+LOG_LEVEL=INFO
+LOG_JSON=true
+DUCKDB_PATH=./data/arbitrade.duckdb
+FERNET_KEY=
+KRAKEN_API_KEY=
+KRAKEN_API_SECRET=

.gitea/workflows/ci.yml

@@ -0,0 +1,53 @@
+name: CI
+
+on:
+  push:
+    branches: ["main", "master"]
+    tags: ["v*"]
+  pull_request:
+
+jobs:
+  lint-test-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install project + dev deps
+        run: |
+          python -m pip install --upgrade pip
+          pip install -e .[dev]
+
+      - name: Ruff
+        run: ruff check .
+
+      - name: Black
+        run: black --check .
+
+      - name: MyPy
+        run: mypy src
+
+      - name: Tests
+        run: pytest -q
+
+      - name: Login to Gitea registry
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v3
+        with:
+          registry: git.allucanget.biz
+          username: ${{ secrets.GITEA_REGISTRY_USERNAME }}
+          password: ${{ secrets.GITEA_REGISTRY_TOKEN }}
+
+      - name: Build and push image
+        if: github.event_name != 'pull_request'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          push: true
+          tags: git.allucanget.biz/${{ secrets.GITEA_REGISTRY_NAMESPACE }}/arbitrade:${{ github.sha }}

.gitignore

@@ -0,0 +1,41 @@
+# Instructions
+.github/instructions/
+
+# Python
+__pycache__/
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+*.egg-info/
+.eggs/
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+# Virtual environments
+.venv/
+venv/
+
+# IDE / OS
+.vscode/
+.idea/
+.DS_Store
+Thumbs.db
+
+# Env / secrets
+.env
+.env.*
+!.env.example
+secrets/
+
+# Local database / runtime data
+data/*.duckdb
+data/*.duckdb.wal
+data/*.duckdb.tmp
+logs/
+
+# Node assets if used for frontend tooling
+node_modules/

.pre-commit-config.yaml

@@ -0,0 +1,17 @@
+repos:
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.6.8
+    hooks:
+      - id: ruff
+      - id: ruff-format
+  - repo: https://github.com/psf/black
+    rev: 24.8.0
+    hooks:
+      - id: black
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: v1.11.2
+    hooks:
+      - id: mypy
+        additional_dependencies:
+          - pydantic>=2.9.0
+          - pydantic-settings>=2.5.0

Dockerfile

@@ -0,0 +1,18 @@
+FROM python:3.12-slim AS base
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir --upgrade pip
+
+COPY pyproject.toml README.md /app/
+COPY src /app/src
+COPY web /app/web
+
+RUN pip install --no-cache-dir .
+
+EXPOSE 8000
+
+CMD ["python", "-m", "arbitrade.main"]

README.md

@@ -0,0 +1,23 @@
+# Arbitrage Trading Bot
+
+## Objective
+
+- Develop an arbitrage trading bot that can identify and exploit price discrepancies across different currency pairs to generate profits.
+- Ensure the bot operates efficiently, securely, and can adapt to changing market conditions.
+- Implement risk management strategies to minimize potential losses.
+- Continuously monitor and optimize the bot's performance.
+- Provide a user-friendly interface for monitoring and controlling the bot's operations.
+- Integrate with kraken exchange for executing trades and accessing market data.
+- Implement a robust logging and alerting system to track the bot's activities and notify the user of important events or issues.
+
+## Key Features
+
+1. **Market Data Collection**: The bot will collect real-time market data, including price, volume, and order book information.
+2. **Price Discrepancy Detection**: The bot will analyze the collected data to identify price discrepancies between currency pairs across different cryptocurrencies and fiat currencies.
+3. **Trade Execution**: The bot will execute trades automatically when a profitable arbitrage opportunity is detected, ensuring that it can capitalize on the price differences before they disappear.
+4. **Risk Management**: The bot will implement risk management strategies, such as setting stop-loss orders and limiting the amount of capital allocated to each trade, to minimize potential losses.
+5. **Performance Monitoring**: The bot will continuously monitor its performance, tracking metrics such as profit and loss, win rate, and average trade duration, to identify areas for improvement.
+6. **User Interface**: The bot will provide a user-friendly interface that allows users to monitor the bot's activities, view performance metrics, and control its operations.
+7. **Integration with Kraken Exchange**: The bot will integrate with the Kraken exchange to access market data and execute trades, ensuring that it can operate effectively in the cryptocurrency market.
+8. **Logging and Alerting System**: The bot will implement a robust logging system to track all activities and transactions, and an alerting system to notify the user of important events, such as successful trades, errors, or significant market changes.
+9. **Security Measures**: The bot will implement security measures to protect user data and prevent unauthorized access, including encryption of sensitive information and secure authentication methods.

docker-compose.yml

@@ -0,0 +1,13 @@
+services:
+  arbitrade:
+    image: git.allucanget.biz/OWNER/arbitrade:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+    env_file:
+      - .env
+    ports:
+      - "8000:8000"
+    volumes:
+      - ./data:/app/data
+    restart: unless-stopped

pyproject.toml

@@ -0,0 +1,71 @@
+[build-system]
+requires = ["hatchling>=1.25.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "arbitrade"
+version = "0.1.0"
+description = "Low-latency Kraken arbitrage bot"
+readme = "README.md"
+requires-python = ">=3.12"
+dependencies = [
+  "cryptography>=43.0.0",
+  "duckdb>=1.1.0",
+  "fastapi>=0.115.0",
+  "httptools>=0.6.1",
+  "httpx>=0.28.0",
+  "jinja2>=3.1.0",
+  "keyring>=25.0.0",
+  "orjson>=3.10.0",
+  "pydantic>=2.9.0",
+  "pydantic-settings>=2.5.0",
+  "structlog>=24.4.0",
+  "sortedcontainers>=2.4.0",
+  "uvicorn[standard]>=0.31.0",
+  "uvloop>=0.20.0; platform_system != 'Windows'",
+  "websockets>=13.0",
+]
+
+[project.optional-dependencies]
+dev = [
+  "black>=24.8.0",
+  "mypy>=1.11.0",
+  "pre-commit>=3.8.0",
+  "pytest>=8.3.0",
+  "pytest-asyncio>=0.24.0",
+  "ruff>=0.6.0",
+]
+
+[project.scripts]
+arbitrade = "arbitrade.main:main"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/arbitrade"]
+
+[tool.black]
+line-length = 100
+target-version = ["py312"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.ruff.lint]
+select = ["E", "F", "I", "UP", "B", "N", "ASYNC"]
+ignore = ["E203"]
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+warn_return_any = true
+warn_unused_configs = true
+pretty = true
+mypy_path = "src"
+
+[[tool.mypy.overrides]]
+module = ["duckdb", "keyring", "uvloop"]
+ignore_missing_imports = true
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]
+asyncio_mode = "auto"

src/arbitrade/__init__.py

@@ -0,0 +1,3 @@
+__all__ = ["__version__"]
+
+__version__ = "0.1.0"

src/arbitrade/api/app.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from fastapi import FastAPI
+
+from arbitrade.api.routes import router
+from arbitrade.config.settings import Settings
+from arbitrade.logging_setup import configure_logging
+from arbitrade.storage.db import DuckDBStore
+
+
+def create_app(settings: Settings) -> FastAPI:
+    configure_logging(settings.log_level, settings.log_json)
+
+    db = DuckDBStore(settings)
+    db.migrate()
+
+    app = FastAPI(title="arbitrade", version="0.1.0")
+    app.include_router(router)
+    return app

src/arbitrade/api/routes.py

@@ -0,0 +1,28 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+from fastapi import APIRouter, Request
+from fastapi.responses import HTMLResponse, JSONResponse
+from fastapi.templating import Jinja2Templates
+
+router = APIRouter()
+templates = Jinja2Templates(directory="web/templates")
+
+
+@router.get("/", response_class=HTMLResponse)
+async def home(request: Request) -> HTMLResponse:
+    return templates.TemplateResponse(
+        request=request,
+        name="health.html",
+        context={
+            "status": "ok",
+            "time": datetime.now(UTC).isoformat(),
+            "title": "Arbitrade Health",
+        },
+    )
+
+
+@router.get("/health", response_class=JSONResponse)
+async def health() -> JSONResponse:
+    return JSONResponse({"status": "ok", "service": "arbitrade"})

src/arbitrade/config/secrets.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import base64
+import os
+from dataclasses import dataclass
+
+import keyring
+from cryptography.fernet import Fernet
+
+
+@dataclass(slots=True)
+class SecretStore:
+    service_name: str = "arbitrade"
+
+    def _load_or_create_key(self, key_env: str | None = None) -> bytes:
+        if key_env:
+            return key_env.encode("utf-8")
+
+        existing = keyring.get_password(self.service_name, "fernet_key")
+        if existing:
+            return existing.encode("utf-8")
+
+        generated = Fernet.generate_key()
+        keyring.set_password(self.service_name, "fernet_key", generated.decode("utf-8"))
+        return generated
+
+    def encrypt(self, plaintext: str, key_env: str | None = None) -> str:
+        key = self._load_or_create_key(key_env)
+        token = Fernet(key).encrypt(plaintext.encode("utf-8"))
+        return token.decode("utf-8")
+
+    def decrypt(self, ciphertext: str, key_env: str | None = None) -> str:
+        key = self._load_or_create_key(key_env)
+        value = Fernet(key).decrypt(ciphertext.encode("utf-8"))
+        return value.decode("utf-8")
+
+    @staticmethod
+    def generate_env_key() -> str:
+        return base64.urlsafe_b64encode(os.urandom(32)).decode("utf-8")

src/arbitrade/config/settings.py

@@ -0,0 +1,27 @@
+from __future__ import annotations
+
+from functools import lru_cache
+from pathlib import Path
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
+
+    app_env: str = Field(default="dev", alias="APP_ENV")
+    app_host: str = Field(default="0.0.0.0", alias="APP_HOST")
+    app_port: int = Field(default=8000, alias="APP_PORT")
+
+    log_level: str = Field(default="INFO", alias="LOG_LEVEL")
+    log_json: bool = Field(default=True, alias="LOG_JSON")
+
+    duckdb_path: Path = Field(default=Path("./data/arbitrade.duckdb"), alias="DUCKDB_PATH")
+
+    fernet_key: str | None = Field(default=None, alias="FERNET_KEY")
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    return Settings()

src/arbitrade/logging_setup.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+import logging
+import sys
+from typing import Any
+
+import structlog
+
+
+def configure_logging(log_level: str = "INFO", json_logs: bool = True) -> None:
+    level = getattr(logging, log_level.upper(), logging.INFO)
+
+    timestamper = structlog.processors.TimeStamper(fmt="iso", utc=True)
+
+    shared_processors: list[Any] = [
+        structlog.contextvars.merge_contextvars,
+        structlog.stdlib.add_log_level,
+        structlog.stdlib.add_logger_name,
+        timestamper,
+    ]
+
+    if json_logs:
+        renderer: Any = structlog.processors.JSONRenderer()
+    else:
+        renderer = structlog.dev.ConsoleRenderer()
+
+    structlog.configure(
+        processors=[
+            *shared_processors,
+            structlog.processors.dict_tracebacks,
+            structlog.processors.EventRenamer("message"),
+            renderer,
+        ],
+        wrapper_class=structlog.make_filtering_bound_logger(level),
+        logger_factory=structlog.stdlib.LoggerFactory(),
+        cache_logger_on_first_use=True,
+    )
+
+    logging.basicConfig(format="%(message)s", stream=sys.stdout, level=level, force=True)

src/arbitrade/main.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import platform
+from importlib import import_module
+
+import uvicorn
+
+from arbitrade.api.app import create_app
+from arbitrade.config.settings import get_settings
+
+
+def _install_uvloop_if_available() -> None:
+    if platform.system() == "Windows":
+        return
+
+    try:
+        uvloop = import_module("uvloop")
+        uvloop.install()
+    except Exception:
+        # App can still run with default asyncio loop.
+        return
+
+
+def main() -> None:
+    _install_uvloop_if_available()
+
+    settings = get_settings()
+    app = create_app(settings)
+
+    uvicorn.run(
+        app,
+        host=settings.app_host,
+        port=settings.app_port,
+        log_level=settings.log_level.lower(),
+        loop="uvloop" if platform.system() != "Windows" else "asyncio",
+        http="httptools",
+    )
+
+
+if __name__ == "__main__":
+    main()

src/arbitrade/storage/db.py

@@ -0,0 +1,59 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+
+import duckdb
+
+from arbitrade.config.settings import Settings
+
+SCHEMA_SQL = """
+CREATE TABLE IF NOT EXISTS schema_migrations (
+    version INTEGER PRIMARY KEY,
+    applied_at TIMESTAMP DEFAULT current_timestamp
+);
+
+CREATE TABLE IF NOT EXISTS opportunities (
+    id UUID DEFAULT uuid(),
+    detected_at TIMESTAMP NOT NULL,
+    cycle VARCHAR NOT NULL,
+    gross_pct DOUBLE,
+    net_pct DOUBLE,
+    est_profit DOUBLE,
+    executed BOOLEAN DEFAULT FALSE
+);
+
+CREATE TABLE IF NOT EXISTS trades (
+    id UUID DEFAULT uuid(),
+    started_at TIMESTAMP NOT NULL,
+    finished_at TIMESTAMP,
+    status VARCHAR NOT NULL,
+    realized_pnl DOUBLE,
+    capital_used DOUBLE
+);
+
+CREATE TABLE IF NOT EXISTS portfolio_snapshots (
+    snapshot_at TIMESTAMP NOT NULL,
+    balances JSON,
+    total_value_usd DOUBLE
+);
+"""
+
+
+class DuckDBStore:
+    def __init__(self, settings: Settings) -> None:
+        self._db_path = Path(settings.duckdb_path)
+        self._db_path.parent.mkdir(parents=True, exist_ok=True)
+
+    @contextmanager
+    def connect(self) -> Iterator[duckdb.DuckDBPyConnection]:
+        conn = duckdb.connect(str(self._db_path))
+        try:
+            yield conn
+        finally:
+            conn.close()
+
+    def migrate(self) -> None:
+        with self.connect() as conn:
+            conn.execute(SCHEMA_SQL)

tests/test_health.py

@@ -0,0 +1,14 @@
+import httpx
+
+from arbitrade.api.app import create_app
+from arbitrade.config.settings import Settings
+
+
+async def test_health_route_returns_ok() -> None:
+    app = create_app(Settings())
+    transport = httpx.ASGITransport(app=app)
+    async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
+        response = await client.get("/health")
+
+    assert response.status_code == 200
+    assert response.json()["status"] == "ok"

web/templates/base.html

@@ -0,0 +1,46 @@
+<!doctype html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <title>{{ title or "Arbitrade" }}</title>
+    <script src="https://unpkg.com/htmx.org@1.9.12"></script>
+    <style>
+      :root {
+        --bg: #f4f7f5;
+        --ink: #122118;
+        --accent: #1f7a4c;
+        --card: #ffffff;
+      }
+      body {
+        margin: 0;
+        font-family: "Segoe UI", sans-serif;
+        color: var(--ink);
+        background: radial-gradient(circle at top, #e9f7ef, var(--bg));
+      }
+      .wrap {
+        max-width: 720px;
+        margin: 4rem auto;
+        padding: 0 1rem;
+      }
+      .card {
+        background: var(--card);
+        border: 1px solid #d8e7dd;
+        border-radius: 12px;
+        padding: 1.25rem;
+        box-shadow: 0 6px 24px rgba(18, 33, 24, 0.08);
+      }
+      .badge {
+        display: inline-block;
+        background: #d9f3e5;
+        color: var(--accent);
+        border-radius: 999px;
+        padding: 0.2rem 0.6rem;
+        font-size: 0.85rem;
+      }
+    </style>
+  </head>
+  <body>
+    <main class="wrap">{% block content %}{% endblock %}</main>
+  </body>
+</html>

web/templates/health.html

@@ -0,0 +1,14 @@
+{% extends "base.html" %}
+
+{% block content %}
+  <section class="card">
+    <h1>Arbitrade Bootstrap Complete</h1>
+    <p><span class="badge">Status: {{ status }}</span></p>
+    <p>UTC: {{ time }}</p>
+    <p>
+      Health JSON:
+      <a href="/health" hx-get="/health" hx-target="#health-json" hx-swap="innerHTML">refresh</a>
+    </p>
+    <pre id="health-json">{"status":"ok","service":"arbitrade"}</pre>
+  </section>
+{% endblock %}