commit f3f369ad6b70d280d3f3fab954c85eaf880aaf23 Author: zwitschi Date: Mon Jun 1 09:15:38 2026 +0200 Add initial project structure with Docker, CI, and FastAPI setup - Create Dockerfile and docker-compose.yml for containerization - Add CI configuration for linting and testing - Implement FastAPI application with health check routes - Set up database schema using DuckDB - Include environment configuration and secrets management - Add README with project objectives and key features - Implement logging setup and configuration - Create initial tests for health route - Add HTML templates for web interface diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6896649 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +.git +.github +.venv +__pycache__ +.pytest_cache +.mypy_cache +.ruff_cache +data +logs +*.pyc +.env diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..cf04bea --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +APP_ENV=dev +APP_HOST=0.0.0.0 +APP_PORT=8000 +LOG_LEVEL=INFO +LOG_JSON=true +DUCKDB_PATH=./data/arbitrade.duckdb +FERNET_KEY= +KRAKEN_API_KEY= +KRAKEN_API_SECRET= diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml new file mode 100644 index 0000000..c6aa42b --- /dev/null +++ b/.gitea/workflows/ci.yml @@ -0,0 +1,53 @@ +name: CI + +on: + push: + branches: ["main", "master"] + tags: ["v*"] + pull_request: + +jobs: + lint-test-build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install project + dev deps + run: | + python -m pip install --upgrade pip + pip install -e .[dev] + + - name: Ruff + run: ruff check . + + - name: Black + run: black --check . + + - name: MyPy + run: mypy src + + - name: Tests + run: pytest -q + + - name: Login to Gitea registry + if: github.event_name != 'pull_request' + uses: docker/login-action@v3 + with: + registry: git.allucanget.biz + username: ${{ secrets.GITEA_REGISTRY_USERNAME }} + password: ${{ secrets.GITEA_REGISTRY_TOKEN }} + + - name: Build and push image + if: github.event_name != 'pull_request' + uses: docker/build-push-action@v6 + with: + context: . + push: true + tags: git.allucanget.biz/${{ secrets.GITEA_REGISTRY_NAMESPACE }}/arbitrade:${{ github.sha }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..837586e --- /dev/null +++ b/.gitignore @@ -0,0 +1,41 @@ +# Instructions +.github/instructions/ + +# Python +__pycache__/ +*.py[cod] +*.pyo +*.pyd +*.so +*.egg-info/ +.eggs/ +.pytest_cache/ +.mypy_cache/ +.ruff_cache/ +.coverage +htmlcov/ + +# Virtual environments +.venv/ +venv/ + +# IDE / OS +.vscode/ +.idea/ +.DS_Store +Thumbs.db + +# Env / secrets +.env +.env.* +!.env.example +secrets/ + +# Local database / runtime data +data/*.duckdb +data/*.duckdb.wal +data/*.duckdb.tmp +logs/ + +# Node assets if used for frontend tooling +node_modules/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..bf41ac4 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,17 @@ +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.6.8 + hooks: + - id: ruff + - id: ruff-format + - repo: https://github.com/psf/black + rev: 24.8.0 + hooks: + - id: black + - repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.11.2 + hooks: + - id: mypy + additional_dependencies: + - pydantic>=2.9.0 + - pydantic-settings>=2.5.0 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..7543ce9 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM python:3.12-slim AS base + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 + +WORKDIR /app + +RUN pip install --no-cache-dir --upgrade pip + +COPY pyproject.toml README.md /app/ +COPY src /app/src +COPY web /app/web + +RUN pip install --no-cache-dir . + +EXPOSE 8000 + +CMD ["python", "-m", "arbitrade.main"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..81c2cdd --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# Arbitrage Trading Bot + +## Objective + +- Develop an arbitrage trading bot that can identify and exploit price discrepancies across different currency pairs to generate profits. +- Ensure the bot operates efficiently, securely, and can adapt to changing market conditions. +- Implement risk management strategies to minimize potential losses. +- Continuously monitor and optimize the bot's performance. +- Provide a user-friendly interface for monitoring and controlling the bot's operations. +- Integrate with kraken exchange for executing trades and accessing market data. +- Implement a robust logging and alerting system to track the bot's activities and notify the user of important events or issues. + +## Key Features + +1. **Market Data Collection**: The bot will collect real-time market data, including price, volume, and order book information. +2. **Price Discrepancy Detection**: The bot will analyze the collected data to identify price discrepancies between currency pairs across different cryptocurrencies and fiat currencies. +3. **Trade Execution**: The bot will execute trades automatically when a profitable arbitrage opportunity is detected, ensuring that it can capitalize on the price differences before they disappear. +4. **Risk Management**: The bot will implement risk management strategies, such as setting stop-loss orders and limiting the amount of capital allocated to each trade, to minimize potential losses. +5. **Performance Monitoring**: The bot will continuously monitor its performance, tracking metrics such as profit and loss, win rate, and average trade duration, to identify areas for improvement. +6. **User Interface**: The bot will provide a user-friendly interface that allows users to monitor the bot's activities, view performance metrics, and control its operations. +7. **Integration with Kraken Exchange**: The bot will integrate with the Kraken exchange to access market data and execute trades, ensuring that it can operate effectively in the cryptocurrency market. +8. **Logging and Alerting System**: The bot will implement a robust logging system to track all activities and transactions, and an alerting system to notify the user of important events, such as successful trades, errors, or significant market changes. +9. **Security Measures**: The bot will implement security measures to protect user data and prevent unauthorized access, including encryption of sensitive information and secure authentication methods. diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f7749b2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,13 @@ +services: + arbitrade: + image: git.allucanget.biz/OWNER/arbitrade:latest + build: + context: . + dockerfile: Dockerfile + env_file: + - .env + ports: + - "8000:8000" + volumes: + - ./data:/app/data + restart: unless-stopped diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..b9eb060 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,71 @@ +[build-system] +requires = ["hatchling>=1.25.0"] +build-backend = "hatchling.build" + +[project] +name = "arbitrade" +version = "0.1.0" +description = "Low-latency Kraken arbitrage bot" +readme = "README.md" +requires-python = ">=3.12" +dependencies = [ + "cryptography>=43.0.0", + "duckdb>=1.1.0", + "fastapi>=0.115.0", + "httptools>=0.6.1", + "httpx>=0.28.0", + "jinja2>=3.1.0", + "keyring>=25.0.0", + "orjson>=3.10.0", + "pydantic>=2.9.0", + "pydantic-settings>=2.5.0", + "structlog>=24.4.0", + "sortedcontainers>=2.4.0", + "uvicorn[standard]>=0.31.0", + "uvloop>=0.20.0; platform_system != 'Windows'", + "websockets>=13.0", +] + +[project.optional-dependencies] +dev = [ + "black>=24.8.0", + "mypy>=1.11.0", + "pre-commit>=3.8.0", + "pytest>=8.3.0", + "pytest-asyncio>=0.24.0", + "ruff>=0.6.0", +] + +[project.scripts] +arbitrade = "arbitrade.main:main" + +[tool.hatch.build.targets.wheel] +packages = ["src/arbitrade"] + +[tool.black] +line-length = 100 +target-version = ["py312"] + +[tool.ruff] +line-length = 100 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "I", "UP", "B", "N", "ASYNC"] +ignore = ["E203"] + +[tool.mypy] +python_version = "3.12" +strict = true +warn_return_any = true +warn_unused_configs = true +pretty = true +mypy_path = "src" + +[[tool.mypy.overrides]] +module = ["duckdb", "keyring", "uvloop"] +ignore_missing_imports = true + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" diff --git a/src/arbitrade/__init__.py b/src/arbitrade/__init__.py new file mode 100644 index 0000000..a05eb9a --- /dev/null +++ b/src/arbitrade/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/src/arbitrade/api/__init__.py b/src/arbitrade/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/arbitrade/api/app.py b/src/arbitrade/api/app.py new file mode 100644 index 0000000..cdcef84 --- /dev/null +++ b/src/arbitrade/api/app.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from fastapi import FastAPI + +from arbitrade.api.routes import router +from arbitrade.config.settings import Settings +from arbitrade.logging_setup import configure_logging +from arbitrade.storage.db import DuckDBStore + + +def create_app(settings: Settings) -> FastAPI: + configure_logging(settings.log_level, settings.log_json) + + db = DuckDBStore(settings) + db.migrate() + + app = FastAPI(title="arbitrade", version="0.1.0") + app.include_router(router) + return app diff --git a/src/arbitrade/api/routes.py b/src/arbitrade/api/routes.py new file mode 100644 index 0000000..da884f6 --- /dev/null +++ b/src/arbitrade/api/routes.py @@ -0,0 +1,28 @@ +from __future__ import annotations + +from datetime import UTC, datetime + +from fastapi import APIRouter, Request +from fastapi.responses import HTMLResponse, JSONResponse +from fastapi.templating import Jinja2Templates + +router = APIRouter() +templates = Jinja2Templates(directory="web/templates") + + +@router.get("/", response_class=HTMLResponse) +async def home(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="health.html", + context={ + "status": "ok", + "time": datetime.now(UTC).isoformat(), + "title": "Arbitrade Health", + }, + ) + + +@router.get("/health", response_class=JSONResponse) +async def health() -> JSONResponse: + return JSONResponse({"status": "ok", "service": "arbitrade"}) diff --git a/src/arbitrade/config/__init__.py b/src/arbitrade/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/arbitrade/config/secrets.py b/src/arbitrade/config/secrets.py new file mode 100644 index 0000000..a04d347 --- /dev/null +++ b/src/arbitrade/config/secrets.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import base64 +import os +from dataclasses import dataclass + +import keyring +from cryptography.fernet import Fernet + + +@dataclass(slots=True) +class SecretStore: + service_name: str = "arbitrade" + + def _load_or_create_key(self, key_env: str | None = None) -> bytes: + if key_env: + return key_env.encode("utf-8") + + existing = keyring.get_password(self.service_name, "fernet_key") + if existing: + return existing.encode("utf-8") + + generated = Fernet.generate_key() + keyring.set_password(self.service_name, "fernet_key", generated.decode("utf-8")) + return generated + + def encrypt(self, plaintext: str, key_env: str | None = None) -> str: + key = self._load_or_create_key(key_env) + token = Fernet(key).encrypt(plaintext.encode("utf-8")) + return token.decode("utf-8") + + def decrypt(self, ciphertext: str, key_env: str | None = None) -> str: + key = self._load_or_create_key(key_env) + value = Fernet(key).decrypt(ciphertext.encode("utf-8")) + return value.decode("utf-8") + + @staticmethod + def generate_env_key() -> str: + return base64.urlsafe_b64encode(os.urandom(32)).decode("utf-8") diff --git a/src/arbitrade/config/settings.py b/src/arbitrade/config/settings.py new file mode 100644 index 0000000..407be65 --- /dev/null +++ b/src/arbitrade/config/settings.py @@ -0,0 +1,27 @@ +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from pydantic import Field +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore") + + app_env: str = Field(default="dev", alias="APP_ENV") + app_host: str = Field(default="0.0.0.0", alias="APP_HOST") + app_port: int = Field(default=8000, alias="APP_PORT") + + log_level: str = Field(default="INFO", alias="LOG_LEVEL") + log_json: bool = Field(default=True, alias="LOG_JSON") + + duckdb_path: Path = Field(default=Path("./data/arbitrade.duckdb"), alias="DUCKDB_PATH") + + fernet_key: str | None = Field(default=None, alias="FERNET_KEY") + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + return Settings() diff --git a/src/arbitrade/logging_setup.py b/src/arbitrade/logging_setup.py new file mode 100644 index 0000000..000f9da --- /dev/null +++ b/src/arbitrade/logging_setup.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import logging +import sys +from typing import Any + +import structlog + + +def configure_logging(log_level: str = "INFO", json_logs: bool = True) -> None: + level = getattr(logging, log_level.upper(), logging.INFO) + + timestamper = structlog.processors.TimeStamper(fmt="iso", utc=True) + + shared_processors: list[Any] = [ + structlog.contextvars.merge_contextvars, + structlog.stdlib.add_log_level, + structlog.stdlib.add_logger_name, + timestamper, + ] + + if json_logs: + renderer: Any = structlog.processors.JSONRenderer() + else: + renderer = structlog.dev.ConsoleRenderer() + + structlog.configure( + processors=[ + *shared_processors, + structlog.processors.dict_tracebacks, + structlog.processors.EventRenamer("message"), + renderer, + ], + wrapper_class=structlog.make_filtering_bound_logger(level), + logger_factory=structlog.stdlib.LoggerFactory(), + cache_logger_on_first_use=True, + ) + + logging.basicConfig(format="%(message)s", stream=sys.stdout, level=level, force=True) diff --git a/src/arbitrade/main.py b/src/arbitrade/main.py new file mode 100644 index 0000000..4b5e119 --- /dev/null +++ b/src/arbitrade/main.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import platform +from importlib import import_module + +import uvicorn + +from arbitrade.api.app import create_app +from arbitrade.config.settings import get_settings + + +def _install_uvloop_if_available() -> None: + if platform.system() == "Windows": + return + + try: + uvloop = import_module("uvloop") + uvloop.install() + except Exception: + # App can still run with default asyncio loop. + return + + +def main() -> None: + _install_uvloop_if_available() + + settings = get_settings() + app = create_app(settings) + + uvicorn.run( + app, + host=settings.app_host, + port=settings.app_port, + log_level=settings.log_level.lower(), + loop="uvloop" if platform.system() != "Windows" else "asyncio", + http="httptools", + ) + + +if __name__ == "__main__": + main() diff --git a/src/arbitrade/storage/__init__.py b/src/arbitrade/storage/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/arbitrade/storage/db.py b/src/arbitrade/storage/db.py new file mode 100644 index 0000000..84dc5c3 --- /dev/null +++ b/src/arbitrade/storage/db.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import duckdb + +from arbitrade.config.settings import Settings + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TIMESTAMP DEFAULT current_timestamp +); + +CREATE TABLE IF NOT EXISTS opportunities ( + id UUID DEFAULT uuid(), + detected_at TIMESTAMP NOT NULL, + cycle VARCHAR NOT NULL, + gross_pct DOUBLE, + net_pct DOUBLE, + est_profit DOUBLE, + executed BOOLEAN DEFAULT FALSE +); + +CREATE TABLE IF NOT EXISTS trades ( + id UUID DEFAULT uuid(), + started_at TIMESTAMP NOT NULL, + finished_at TIMESTAMP, + status VARCHAR NOT NULL, + realized_pnl DOUBLE, + capital_used DOUBLE +); + +CREATE TABLE IF NOT EXISTS portfolio_snapshots ( + snapshot_at TIMESTAMP NOT NULL, + balances JSON, + total_value_usd DOUBLE +); +""" + + +class DuckDBStore: + def __init__(self, settings: Settings) -> None: + self._db_path = Path(settings.duckdb_path) + self._db_path.parent.mkdir(parents=True, exist_ok=True) + + @contextmanager + def connect(self) -> Iterator[duckdb.DuckDBPyConnection]: + conn = duckdb.connect(str(self._db_path)) + try: + yield conn + finally: + conn.close() + + def migrate(self) -> None: + with self.connect() as conn: + conn.execute(SCHEMA_SQL) diff --git a/tests/test_health.py b/tests/test_health.py new file mode 100644 index 0000000..5180fcf --- /dev/null +++ b/tests/test_health.py @@ -0,0 +1,14 @@ +import httpx + +from arbitrade.api.app import create_app +from arbitrade.config.settings import Settings + + +async def test_health_route_returns_ok() -> None: + app = create_app(Settings()) + transport = httpx.ASGITransport(app=app) + async with httpx.AsyncClient(transport=transport, base_url="http://test") as client: + response = await client.get("/health") + + assert response.status_code == 200 + assert response.json()["status"] == "ok" diff --git a/web/templates/base.html b/web/templates/base.html new file mode 100644 index 0000000..2a7f813 --- /dev/null +++ b/web/templates/base.html @@ -0,0 +1,46 @@ + + + + + + {{ title or "Arbitrade" }} + + + + +
{% block content %}{% endblock %}
+ + diff --git a/web/templates/health.html b/web/templates/health.html new file mode 100644 index 0000000..aa86fd0 --- /dev/null +++ b/web/templates/health.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block content %} +
+

Arbitrade Bootstrap Complete

+

Status: {{ status }}

+

UTC: {{ time }}

+

+ Health JSON: + refresh +

+ 
{"status":"ok","service":"arbitrade"}
+
+{% endblock %}