diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 954e043..95c313f 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -43,6 +43,21 @@ jobs: - name: Tests run: pytest -q + - name: Package template smoke check + run: | + pip install build + python -m build --wheel + pip uninstall -y arbitrade + pip install --force-reinstall dist/*.whl + python - <<'PY' + from arbitrade.api import routes + + template = routes.templates.env.get_template("dashboard.html") + rendered = template.render() + if "" not in rendered: + raise SystemExit("dashboard template render smoke check failed") + PY + - name: Latency guardrails run: | python scripts/check_latency_regression.py \ diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md new file mode 100644 index 0000000..480ed0c --- /dev/null +++ b/DEPLOYMENT.md @@ -0,0 +1,152 @@ +# Deployment Guide (Coolify) + +This guide provides two supported deployment paths for Arbitrade on Coolify: + +- Build directly from Git repository in Coolify. +- Deploy prebuilt container image: `git.allucanget.biz/allucanget/arbitrade:latest`. + +Reference docs: + +- Coolify Applications: https://coolify.io/docs/applications +- Coolify Build Packs: https://coolify.io/docs/applications/build-packs +- Coolify Dockerfile Build Pack: https://coolify.io/docs/applications/build-packs/dockerfile +- Coolify Nixpacks Build Pack: https://coolify.io/docs/applications/build-packs/nixpacks +- Coolify CI/CD (Git providers): https://coolify.io/docs/applications/ci-cd +- Coolify Gitea integration: https://coolify.io/docs/applications/ci-cd/gitea/integration +- Coolify environment variables: https://coolify.io/docs/knowledge-base/environment-variables +- Coolify persistent storage: https://coolify.io/docs/knowledge-base/persistent-storage +- Coolify health checks: https://coolify.io/docs/knowledge-base/health-checks +- Coolify Docker registry credentials: https://coolify.io/docs/knowledge-base/docker/registry + +## Common Runtime Configuration + +Use these values in both deployment modes. + +### Port and health + +- Container port: `9090` +- Health check path: `/health` +- Protocol: HTTP + +### Persistent storage + +- Add a persistent volume +- Mount path: `/app/data` +- Set DB path to: `DUCKDB_PATH=/app/data/arbitrade.duckdb` + +### Required environment variables + +- `APP_ENV=prod` +- `APP_HOST=0.0.0.0` +- `APP_PORT=9090` +- `DUCKDB_PATH=/app/data/arbitrade.duckdb` +- `LOG_LEVEL=INFO` +- `LOG_JSON=true` +- `KRAKEN_API_KEY=` +- `KRAKEN_API_SECRET=` +- `KRAKEN_API_KEY_PERMISSIONS=query,trade` +- `FERNET_KEY=` + +Notes: + +- Store secrets in Coolify secret variables, not in Git. +- Keep Kraken key scope minimal (query + trade, no withdrawal). + +## Option A: Build in Coolify from Git Repository + +Recommended when you want Coolify to build from source and optionally auto-deploy on commits. + +1. Open your Coolify project and select Create New Resource. +2. Choose deployment source: + +- Public repo: use `Public repository` and provide HTTPS URL. +- Private Gitea repo: use deploy key flow from the Gitea guide. + +3. Set repository URL for this project: + +- `https://git.allucanget.biz/allucanget/arbitrade.git` (public) +- or SSH URL if private deploy key is used. + +4. Choose build pack: + +- Prefer `Dockerfile` for this repo, because Docker build behavior is explicitly defined. +- Use `Nixpacks` only if you intentionally want auto-detected build logic. + +5. Configure branch and base directory: + +- Branch: your deploy branch (for example `main`) +- Base directory: `/` + +6. Configure network: + +- Exposed port: `9090` +- Domain: set your Coolify domain/custom domain + +7. Configure environment variables and secrets from the Common Runtime Configuration section. +8. Add persistent storage mount `/app/data`. +9. Configure health check: + +- Path: `/health` +- Ensure container includes `curl` or `wget` if using UI-defined checks. + +10. Click Deploy and verify: + +- Deployment logs complete successfully. +- `GET /health` returns success. + +Optional (Git webhook auto-deploy with Gitea): + +1. In Coolify resource, open `Webhooks` and copy Manual Git Webhook URL. +2. Set webhook secret in Coolify. +3. In Gitea repo settings, add webhook URL + same secret and enable Push events. +4. Push a commit and confirm Coolify triggers deploy. + +## Option B: Deploy Prebuilt Image from Container Registry + +Recommended when CI publishes the image and Coolify only runs it. + +Image: + +- `git.allucanget.biz/allucanget/arbitrade:latest` + +1. Ensure CI publishes the image before first deployment. +2. In Coolify, select Create New Resource. +3. Choose Application deployment based on Docker Image. +4. Set image reference: + +- Registry: `git.allucanget.biz` +- Image: `git.allucanget.biz/allucanget/arbitrade:latest` + +5. Configure registry credentials in Coolify if your registry requires auth. +6. Leave build/install/start commands empty unless you need overrides. +7. Set network and health: + +- Exposed port: `9090` +- Health check path: `/health` + +8. Add environment variables and secrets from the Common Runtime Configuration section. +9. Add persistent storage mount `/app/data`. +10. Deploy and verify: + +- Logs show container start success. +- `GET /health` returns success. + +Update flow for new releases: + +- Push code and let CI publish a new `latest` image. +- Trigger redeploy in Coolify for this resource. + +## Quick Troubleshooting + +- `No available server` from proxy: +- Check health check path/port and app bind (`APP_HOST=0.0.0.0`, `APP_PORT=9090`). +- Verify health check is passing in Coolify. +- `TemplateNotFound: dashboard.html` at runtime: +- Ensure the deployed image/wheel includes package templates under `arbitrade/web/templates/*`. +- If you build from source, do not remove packaged template files under `src/arbitrade/web/templates`. +- DB resets after deploy: +- Confirm persistent mount exists at `/app/data` and `DUCKDB_PATH` points there. +- Registry pull fails: +- Re-check Docker registry credentials in Coolify. +- App starts but unavailable externally: +- Confirm exposed port is `9090` and domain is attached to this resource. diff --git a/README.md b/README.md index c16b974..9f710b2 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,7 @@ Current stack: Project plan lives in [PLAN.md](PLAN.md). Task checklist lives in [.github/instructions/TODO.md](.github/instructions/TODO.md). +Coolify deployment runbooks live in [DEPLOYMENT.md](DEPLOYMENT.md). ## Current Status diff --git a/build/lib/arbitrade/__init__.py b/build/lib/arbitrade/__init__.py new file mode 100644 index 0000000..a05eb9a --- /dev/null +++ b/build/lib/arbitrade/__init__.py @@ -0,0 +1,3 @@ +__all__ = ["__version__"] + +__version__ = "0.1.0" diff --git a/build/lib/arbitrade/alerting/__init__.py b/build/lib/arbitrade/alerting/__init__.py new file mode 100644 index 0000000..dcaa841 --- /dev/null +++ b/build/lib/arbitrade/alerting/__init__.py @@ -0,0 +1,25 @@ +"""Alerting primitives and channel clients.""" + +from arbitrade.alerting.notifier import ( + AlertEvent, + AlertNotifier, + AlertSeverity, + DiscordWebhookChannel, + EmailSmtpChannel, + SupportsAlertStatus, + TelegramChannel, + build_channels_from_settings, + dispatch_alert_nowait, +) + +__all__ = [ + "AlertEvent", + "AlertNotifier", + "AlertSeverity", + "DiscordWebhookChannel", + "EmailSmtpChannel", + "SupportsAlertStatus", + "TelegramChannel", + "build_channels_from_settings", + "dispatch_alert_nowait", +] diff --git a/build/lib/arbitrade/alerting/notifier.py b/build/lib/arbitrade/alerting/notifier.py new file mode 100644 index 0000000..0ad3ef4 --- /dev/null +++ b/build/lib/arbitrade/alerting/notifier.py @@ -0,0 +1,400 @@ +from __future__ import annotations + +import asyncio +import smtplib +from dataclasses import dataclass +from datetime import UTC, datetime +from email.message import EmailMessage +from typing import Literal, Protocol, runtime_checkable + +import httpx + +AlertSeverity = Literal["info", "warning", "error", "critical"] + +_SEVERITY_RANK: dict[AlertSeverity, int] = { + "info": 10, + "warning": 20, + "error": 30, + "critical": 40, +} + + +@dataclass(frozen=True, slots=True) +class AlertEvent: + category: str + severity: AlertSeverity + title: str + message: str + occurred_at: datetime + details: dict[str, str] + + +class AlertChannel(Protocol): + async def send(self, event: AlertEvent) -> None: ... + + +class SupportsAlerts(Protocol): + async def notify( + self, + *, + category: str, + severity: AlertSeverity, + title: str, + message: str, + details: dict[str, str] | None = None, + ) -> bool: ... + + +@runtime_checkable +class SupportsAlertStatus(Protocol): + def status_snapshot(self) -> dict[str, object]: ... + + +class AlertNotifier: + def __init__( + self, + channels: list[AlertChannel], + *, + enabled: bool = True, + min_severity: AlertSeverity = "info", + dedup_seconds: float = 0.0, + category_flags: dict[str, bool] | None = None, + ) -> None: + if dedup_seconds < 0.0: + raise ValueError("dedup_seconds must be >= 0.0") + self._channels = channels + self._enabled = enabled + self._min_severity: AlertSeverity = min_severity + self._dedup_seconds = dedup_seconds + self._category_flags = {key.lower(): value for key, value in (category_flags or {}).items()} + self._last_sent_at: dict[str, datetime] = {} + self._last_result: str = "never" + self._last_attempted_at: datetime | None = None + self._last_success_at: datetime | None = None + self._last_error: str | None = None + self._last_event_title: str | None = None + self._last_event_category: str | None = None + self._last_event_severity: AlertSeverity | None = None + self._last_channel_results: list[str] = [] + + @property + def has_channels(self) -> bool: + return bool(self._channels) + + async def notify( + self, + *, + category: str, + severity: AlertSeverity, + title: str, + message: str, + details: dict[str, str] | None = None, + ) -> bool: + if not self._enabled or not self._channels: + self._last_result = "skipped_disabled" if not self._enabled else "skipped_no_channels" + return False + + normalized_category = category.strip().lower() + if self._category_flags and not self._category_flags.get(normalized_category, True): + self._last_result = "skipped_category" + return False + + if _SEVERITY_RANK[severity] < _SEVERITY_RANK[self._min_severity]: + self._last_result = "skipped_severity" + return False + + dedup_key = f"{normalized_category}|{severity}|{title}|{message}" + now = datetime.now(UTC) + if self._dedup_seconds > 0.0: + previous = self._last_sent_at.get(dedup_key) + if previous is not None: + elapsed = (now - previous).total_seconds() + if elapsed < self._dedup_seconds: + self._last_result = "skipped_dedup" + return False + + event = AlertEvent( + category=normalized_category, + severity=severity, + title=title, + message=message, + occurred_at=now, + details=details or {}, + ) + + results = await asyncio.gather( + *(channel.send(event) for channel in self._channels), + return_exceptions=True, + ) + self._last_attempted_at = now + self._last_event_title = title + self._last_event_category = normalized_category + self._last_event_severity = severity + self._last_channel_results = [] + for channel, result in zip(self._channels, results, strict=False): + channel_name = type(channel).__name__ + if isinstance(result, Exception): + self._last_channel_results.append(f"{channel_name}: error") + else: + self._last_channel_results.append(f"{channel_name}: ok") + + if all(isinstance(result, Exception) for result in results): + self._last_result = "failed" + self._last_error = "all channels failed" + return False + + self._last_result = ( + "partial_success" + if any(isinstance(result, Exception) for result in results) + else "success" + ) + self._last_error = None + self._last_success_at = now + + self._last_sent_at[dedup_key] = now + return True + + def status_snapshot(self) -> dict[str, object]: + return { + "enabled": self._enabled, + "has_channels": self.has_channels, + "configured_channels": [type(channel).__name__ for channel in self._channels], + "min_severity": self._min_severity, + "dedup_seconds": self._dedup_seconds, + "last_result": self._last_result, + "last_attempted_at": ( + self._last_attempted_at.isoformat() if self._last_attempted_at is not None else None + ), + "last_success_at": ( + self._last_success_at.isoformat() if self._last_success_at is not None else None + ), + "last_error": self._last_error, + "last_event": ( + None + if self._last_event_title is None + else { + "title": self._last_event_title, + "category": self._last_event_category, + "severity": self._last_event_severity, + } + ), + "last_channel_results": self._last_channel_results, + } + + +def dispatch_alert_nowait( + notifier: SupportsAlerts | None, + *, + category: str, + severity: AlertSeverity, + title: str, + message: str, + details: dict[str, str] | None = None, +) -> None: + if notifier is None: + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + + loop.create_task( + notifier.notify( + category=category, + severity=severity, + title=title, + message=message, + details=details, + ) + ) + + +def _format_event_text(event: AlertEvent) -> str: + lines = [ + f"[{event.severity.upper()}] {event.title}", + f"Category: {event.category}", + f"Time: {event.occurred_at.isoformat()}", + event.message, + ] + if event.details: + lines.append("Details:") + for key, value in sorted(event.details.items()): + lines.append(f"- {key}: {value}") + return "\n".join(lines) + + +class TelegramChannel: + def __init__(self, *, bot_token: str, chat_id: str, timeout_seconds: float = 10.0) -> None: + self._bot_token = bot_token + self._chat_id = chat_id + self._timeout_seconds = timeout_seconds + + async def send(self, event: AlertEvent) -> None: + url = f"https://api.telegram.org/bot{self._bot_token}/sendMessage" + payload = { + "chat_id": self._chat_id, + "text": _format_event_text(event), + "disable_web_page_preview": True, + } + timeout = httpx.Timeout(self._timeout_seconds) + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.post(url, json=payload) + response.raise_for_status() + + +class DiscordWebhookChannel: + def __init__(self, *, webhook_url: str, timeout_seconds: float = 10.0) -> None: + self._webhook_url = webhook_url + self._timeout_seconds = timeout_seconds + + async def send(self, event: AlertEvent) -> None: + payload = {"content": _format_event_text(event)} + timeout = httpx.Timeout(self._timeout_seconds) + async with httpx.AsyncClient(timeout=timeout) as client: + response = await client.post(self._webhook_url, json=payload) + response.raise_for_status() + + +class EmailSmtpChannel: + def __init__( + self, + *, + host: str, + port: int, + sender: str, + recipients: list[str], + username: str | None = None, + password: str | None = None, + use_tls: bool = True, + timeout_seconds: float = 10.0, + ) -> None: + if not recipients: + raise ValueError("recipients must not be empty") + + self._host = host + self._port = port + self._sender = sender + self._recipients = recipients + self._username = username + self._password = password + self._use_tls = use_tls + self._timeout_seconds = timeout_seconds + + async def send(self, event: AlertEvent) -> None: + message = EmailMessage() + message["From"] = self._sender + message["To"] = ", ".join(self._recipients) + message["Subject"] = f"[{event.severity.upper()}] {event.title}" + message.set_content(_format_event_text(event)) + + await asyncio.to_thread(self._send_sync, message) + + def _send_sync(self, message: EmailMessage) -> None: + with smtplib.SMTP(self._host, self._port, timeout=self._timeout_seconds) as client: + if self._use_tls: + client.starttls() + if self._username and self._password: + client.login(self._username, self._password) + client.send_message(message) + + +class _AlertSettings(Protocol): + alerts_enabled: bool + alert_min_severity: str + alert_dedup_seconds: float + alert_on_trade_events: bool + alert_on_error_events: bool + alert_on_threshold_events: bool + alert_on_system_events: bool + + telegram_alerts_enabled: bool + telegram_bot_token: str | None + telegram_chat_id: str | None + + discord_alerts_enabled: bool + discord_webhook_url: str | None + + email_alerts_enabled: bool + email_smtp_host: str | None + email_smtp_port: int + email_smtp_username: str | None + email_smtp_password: str | None + email_alert_from: str | None + email_alert_to: str | None + email_smtp_use_tls: bool + + +def _as_alert_severity(value: str) -> AlertSeverity: + normalized = value.strip().lower() + if normalized == "info": + return "info" + if normalized == "warning": + return "warning" + if normalized == "error": + return "error" + if normalized == "critical": + return "critical" + else: + raise ValueError("alert_min_severity must be one of: info, warning, error, critical") + + +def build_channels_from_settings(settings: _AlertSettings) -> list[AlertChannel]: + channels: list[AlertChannel] = [] + + if settings.telegram_alerts_enabled: + if not settings.telegram_bot_token or not settings.telegram_chat_id: + raise ValueError("telegram alerts require bot token and chat id") + channels.append( + TelegramChannel( + bot_token=settings.telegram_bot_token, + chat_id=settings.telegram_chat_id, + ) + ) + + if settings.discord_alerts_enabled: + if not settings.discord_webhook_url: + raise ValueError("discord alerts require webhook url") + channels.append(DiscordWebhookChannel(webhook_url=settings.discord_webhook_url)) + + if settings.email_alerts_enabled: + if not settings.email_smtp_host: + raise ValueError("email alerts require SMTP host") + if not settings.email_alert_from: + raise ValueError("email alerts require sender address") + if not settings.email_alert_to: + raise ValueError("email alerts require recipient list") + + recipients = [ + address.strip() for address in settings.email_alert_to.split(",") if address.strip() + ] + channels.append( + EmailSmtpChannel( + host=settings.email_smtp_host, + port=settings.email_smtp_port, + sender=settings.email_alert_from, + recipients=recipients, + username=settings.email_smtp_username, + password=settings.email_smtp_password, + use_tls=settings.email_smtp_use_tls, + ) + ) + + return channels + + +def build_notifier_from_settings(settings: _AlertSettings) -> AlertNotifier: + severity = _as_alert_severity(settings.alert_min_severity) + channels = build_channels_from_settings(settings) + category_flags = { + "trade": settings.alert_on_trade_events, + "error": settings.alert_on_error_events, + "threshold": settings.alert_on_threshold_events, + "system": settings.alert_on_system_events, + } + return AlertNotifier( + channels, + enabled=settings.alerts_enabled, + min_severity=severity, + dedup_seconds=settings.alert_dedup_seconds, + category_flags=category_flags, + ) diff --git a/build/lib/arbitrade/api/__init__.py b/build/lib/arbitrade/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/arbitrade/api/app.py b/build/lib/arbitrade/api/app.py new file mode 100644 index 0000000..d7dd11e --- /dev/null +++ b/build/lib/arbitrade/api/app.py @@ -0,0 +1,44 @@ +from __future__ import annotations + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from arbitrade.alerting.notifier import build_notifier_from_settings +from arbitrade.api.control_state import DashboardControlState +from arbitrade.api.routes import public_router, router +from arbitrade.config.settings import Settings +from arbitrade.logging_setup import configure_logging +from arbitrade.metrics import MetricsCalculator +from arbitrade.runtime.lifecycle import graceful_shutdown, restore_runtime_state +from arbitrade.storage.db import DuckDBStore +from arbitrade.storage.repositories import AuditRepository, RuntimeStateRepository + + +def create_app(settings: Settings) -> FastAPI: + configure_logging(settings.log_level, settings.log_json) + + db = DuckDBStore(settings) + db.migrate() + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + await restore_runtime_state(app) + yield + await graceful_shutdown(app) + + app = FastAPI(title="arbitrade", version="0.1.0", lifespan=lifespan) + app.state.settings = settings + app.state.store = db + app.state.metrics = MetricsCalculator(db) + app.state.audit_repository = AuditRepository(db) + app.state.runtime_state_repository = RuntimeStateRepository(db) + app.state.alert_notifier = build_notifier_from_settings(settings) + app.state.backtest_recent_reports = [] + app.state.dashboard_controls = DashboardControlState( + is_running=not settings.kill_switch_active, + ) + app.include_router(public_router) + app.include_router(router) + return app diff --git a/build/lib/arbitrade/api/auth.py b/build/lib/arbitrade/api/auth.py new file mode 100644 index 0000000..450036a --- /dev/null +++ b/build/lib/arbitrade/api/auth.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +from secrets import compare_digest +from typing import Annotated + +from fastapi import Depends, HTTPException, Request, status +from fastapi.security import HTTPBasic, HTTPBasicCredentials + +dashboard_basic_auth = HTTPBasic(auto_error=False) + + +def require_dashboard_auth( + request: Request, + credentials: Annotated[HTTPBasicCredentials | None, Depends(dashboard_basic_auth)], +) -> None: + settings = request.app.state.settings + username = settings.dashboard_auth_username + password = settings.dashboard_auth_password + + if username is None and password is None: + return + + if username is None or password is None: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Dashboard auth misconfigured", + ) + + if ( + credentials is None + or not compare_digest(credentials.username, username) + or not compare_digest(credentials.password, password) + ): + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Not authenticated", + headers={"WWW-Authenticate": 'Basic realm="Arbitrade Dashboard"'}, + ) diff --git a/build/lib/arbitrade/api/control_state.py b/build/lib/arbitrade/api/control_state.py new file mode 100644 index 0000000..b715dcc --- /dev/null +++ b/build/lib/arbitrade/api/control_state.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import UTC, datetime + +from arbitrade.risk.kill_switch import KillSwitch + + +@dataclass(slots=True) +class DashboardControlState: + is_running: bool = True + kill_switch: KillSwitch = field(default_factory=KillSwitch) + tradable_pairs: list[str] = field(default_factory=list) + strategy_mode: str = "incremental" + strategy_profit_threshold: float = 0.0005 + strategy_max_depth_levels: int = 10 + updated_at: datetime = field(default_factory=lambda: datetime.now(UTC)) + + def mark_updated(self) -> None: + self.updated_at = datetime.now(UTC) diff --git a/build/lib/arbitrade/api/routes.py b/build/lib/arbitrade/api/routes.py new file mode 100644 index 0000000..074ac79 --- /dev/null +++ b/build/lib/arbitrade/api/routes.py @@ -0,0 +1,944 @@ +from __future__ import annotations + +import json +from asyncio import Lock +from collections.abc import AsyncIterator +from datetime import UTC, datetime +from importlib import resources +from pathlib import Path +from typing import cast +from urllib.parse import parse_qs + +import duckdb +from fastapi import APIRouter, Depends, Request +from fastapi.responses import HTMLResponse, JSONResponse, StreamingResponse +from fastapi.templating import Jinja2Templates + +from arbitrade.alerting.notifier import SupportsAlerts, SupportsAlertStatus +from arbitrade.api.auth import require_dashboard_auth +from arbitrade.api.control_state import DashboardControlState +from arbitrade.backtesting.replay import BacktestConfig, BacktestReplayEngine, load_replay_events +from arbitrade.detection.graph import CurrencyGraph, TriangularCycle +from arbitrade.storage.repositories import AuditRecord, AuditRepository + +router = APIRouter(dependencies=[Depends(require_dashboard_auth)]) +public_router = APIRouter() + + +def _resolve_templates_directory() -> str: + # Support source layout, Docker runtime (/app), and installed package data. + source_layout_path = Path( + __file__).resolve().parents[3] / "web" / "templates" + if source_layout_path.is_dir(): + return str(source_layout_path) + + docker_runtime_path = Path.cwd() / "web" / "templates" + if docker_runtime_path.is_dir(): + return str(docker_runtime_path) + + try: + package_path = resources.files( + "arbitrade").joinpath("web", "templates") + if package_path.is_dir(): + return str(package_path) + except (ModuleNotFoundError, AttributeError): + pass + + return str(source_layout_path) + + +templates = Jinja2Templates(directory=_resolve_templates_directory()) +_BACKTEST_ROOT = Path(__file__).resolve().parents[3] +_BACKTEST_RUN_LOCK = Lock() + + +def _format_metric(value: float | None, *, precision: int = 2, suffix: str = "") -> str: + if value is None: + return "—" + return f"{value:.{precision}f}{suffix}" + + +def _dashboard_metrics(request: Request) -> dict[str, str]: + metrics = request.app.state.metrics.compute() + return { + "realized_pnl": _format_metric(metrics.realized_pnl_usd, precision=2, suffix=" USD"), + "win_rate": _format_metric( + metrics.win_rate * 100.0 if metrics.win_rate is not None else None, + precision=1, + suffix="%", + ), + "avg_trade_duration": _format_metric( + metrics.avg_trade_duration_seconds, precision=1, suffix=" s" + ), + "opportunities_per_minute": _format_metric( + metrics.opportunities_per_minute, precision=1, suffix=" /min" + ), + "fill_rate": _format_metric( + metrics.fill_rate * 100.0 if metrics.fill_rate is not None else None, + precision=1, + suffix="%", + ), + "latency_p50": _format_metric(metrics.latency_p50_seconds, precision=3, suffix=" s"), + "latency_p95": _format_metric(metrics.latency_p95_seconds, precision=3, suffix=" s"), + "latency_p99": _format_metric(metrics.latency_p99_seconds, precision=3, suffix=" s"), + "generated_at": datetime.now(UTC).isoformat(), + } + + +def _table_columns(conn: duckdb.DuckDBPyConnection, table_name: str) -> set[str]: + rows = conn.execute(f"PRAGMA table_info('{table_name}')").fetchall() + return {str(row[1]) for row in rows} + + +def _dashboard_overview(request: Request) -> dict[str, object]: + store = request.app.state.store + with store.connect() as conn: + trade_columns = _table_columns(conn, "trades") + trade_ref_expr = "trade_ref" if "trade_ref" in trade_columns else "CAST(id AS VARCHAR)" + cycle_expr = "cycle" if "cycle" in trade_columns else "NULL" + if "finished_at" in trade_columns: + open_trade_filter = "finished_at IS NULL" + else: + open_trade_filter = "LOWER(status) NOT IN ('filled', 'closed', 'cancelled', 'canceled')" + + portfolio_row = conn.execute(""" + SELECT balances, total_value_usd + FROM portfolio_snapshots + ORDER BY snapshot_at DESC + LIMIT 1 + """).fetchone() + open_trades = conn.execute(f""" + SELECT {trade_ref_expr}, status, started_at, {cycle_expr} + FROM trades + WHERE {open_trade_filter} + ORDER BY started_at DESC + LIMIT 5 + """).fetchall() + rpnl = conn.execute(""" + SELECT COALESCE(SUM(COALESCE(realized_pnl, 0)), 0) + FROM trades + """).fetchone() + latest_opportunities = conn.execute(""" + SELECT cycle, net_pct, est_profit, detected_at + FROM opportunities + ORDER BY detected_at DESC + LIMIT 5 + """).fetchall() + + balances_value = "—" + total_value = "—" + if portfolio_row is not None: + balances_raw, total_value_raw = portfolio_row + balances_value = str(balances_raw) if balances_raw is not None else "—" + if total_value_raw is not None: + total_value = f"{float(total_value_raw):.2f} USD" + + open_trade_rows = [ + { + "trade_ref": str(row[0]), + "status": str(row[1]), + "started_at": row[2].isoformat() if isinstance(row[2], datetime) else "—", + "cycle": str(row[3]) if row[3] is not None else "—", + } + for row in open_trades + ] + opportunity_rows = [ + { + "cycle": str(row[0]), + "net_pct": f"{float(row[1]):.2f}%" if row[1] is not None else "—", + "est_profit": f"{float(row[2]):.2f} USD" if row[2] is not None else "—", + "detected_at": row[3].isoformat() if isinstance(row[3], datetime) else "—", + } + for row in latest_opportunities + ] + + return { + "status": "live", + "generated_at": datetime.now(UTC).isoformat(), + "balances": balances_value, + "total_value": total_value, + "open_trade_count": len(open_trade_rows), + "open_trades": open_trade_rows, + "realized_pnl_total": f"{float(rpnl[0]):.2f} USD" if rpnl else "—", + "opportunities": opportunity_rows, + } + + +def _dashboard_charts(request: Request) -> dict[str, object]: + store = request.app.state.store + with store.connect() as conn: + opportunity_rows = conn.execute(""" + SELECT detected_at, cycle, net_pct, est_profit + FROM opportunities + ORDER BY detected_at DESC + LIMIT 10 + """).fetchall() + + cr = list(reversed(opportunity_rows)) + labels = [] + for index, row in enumerate(cr): + if isinstance(row[0], datetime): + labels.append(row[0].isoformat()) + else: + labels.append(f"opportunity-{index + 1}") + np = [float(row[2]) if row[2] is not None else 0.0 for row in cr] + ep = [float(row[3]) if row[3] is not None else 0.0 for row in cr] + cycles = [str(row[1]) for row in cr] + + return { + "labels": labels, + "net_pct_values": np, + "est_profit_values": ep, + "cycles": cycles, + "has_chart_data": bool(cr), + "generated_at": datetime.now(UTC).isoformat(), + } + + +def _dashboard_controls_state(request: Request) -> DashboardControlState: + return cast(DashboardControlState, request.app.state.dashboard_controls) + + +def _audit_repository(request: Request) -> AuditRepository | None: + repository = getattr(request.app.state, "audit_repository", None) + return cast(AuditRepository | None, repository) + + +def _record_audit( + request: Request, + *, + actor: str, + event_type: str, + decision: str, + payload: dict[str, object] | None = None, +) -> None: + repository = _audit_repository(request) + if repository is None: + return + correlation_id = request.headers.get("x-request-id") + if payload is not None: + ret_pl = {str(key): payload[key] for key in payload} + else: + ret_pl = None + repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor=actor, + event_type=event_type, + decision=decision, + payload=ret_pl, + correlation_id=correlation_id, + ) + ) + + +def _dashboard_audit(request: Request, *, limit: int = 15) -> dict[str, object]: + repository = _audit_repository(request) + if repository is None: + return { + "entries": [], + "generated_at": datetime.now(UTC).isoformat(), + } + + records = repository.list_recent(limit=limit) + entries: list[dict[str, str]] = [] + for record in records: + payload_text = "—" + if record.payload: + payload_text = json.dumps(record.payload) + entries.append( + { + "occurred_at": record.occurred_at.isoformat(), + "actor": record.actor, + "event_type": record.event_type, + "decision": record.decision, + "payload": payload_text, + "correlation_id": record.correlation_id or "—", + } + ) + + return { + "entries": entries, + "generated_at": datetime.now(UTC).isoformat(), + } + + +def _alert_notifier(request: Request) -> SupportsAlerts | None: + notifier = getattr(request.app.state, "alert_notifier", None) + return cast(SupportsAlerts | None, notifier) + + +def _alert_status_snapshot(request: Request) -> dict[str, object]: + notifier = getattr(request.app.state, "alert_notifier", None) + if isinstance(notifier, SupportsAlertStatus): + return notifier.status_snapshot() + return { + "enabled": False, + "has_channels": False, + "configured_channels": [], + "min_severity": "—", + "dedup_seconds": 0.0, + "last_result": "unavailable", + "last_attempted_at": None, + "last_success_at": None, + "last_error": None, + "last_event": None, + "last_channel_results": [], + } + + +def _dashboard_controls(request: Request) -> dict[str, object]: + ctl = _dashboard_controls_state(request) + rs = request.app.state.settings + alert_status = _alert_status_snapshot(request) + last_event = alert_status.get("last_event") + last_event_title = "—" + if isinstance(last_event, dict): + title_value = last_event.get("title") + if isinstance(title_value, str): + last_event_title = title_value + + cc = alert_status.get("configured_channels") + cd = "—" + if isinstance(cc, list) and cc: + cd = ", ".join(str(channel) for channel in cc) + + ddsr = alert_status.get("dedup_seconds", 0.0) + dds = float(ddsr) if isinstance(ddsr, int | float) else 0.0 + tpd = ", ".join(ctl.tradable_pairs) if ctl.tradable_pairs else "All" + max_trade_capital_usd = ( + f"{float(rs.max_trade_capital_usd):.2f} USD" + if rs.max_trade_capital_usd is not None + else "—" + ) + max_trade_capital_usd_value = ( + f"{float(rs.max_trade_capital_usd):.2f}" if rs.max_trade_capital_usd is not None else "" + ) + max_concurrent_trades = ( + str(rs.max_concurrent_trades) if rs.max_concurrent_trades is not None else "—" + ) + max_concurrent_trades_value = ( + str(rs.max_concurrent_trades) if rs.max_concurrent_trades is not None else "" + ) + alerts_last_channel_results = [ + str(item) for item in cast(list[object], alert_status.get("last_channel_results", [])) + ] + strategy_stat_arb_enabled = bool( + getattr(rs, "strategy_enable_stat_arb_experiment", False)) + + return { + "execution_status": "running" if ctl.is_running else "stopped", + "kill_switch_status": "active" if ctl.kill_switch.is_active else "inactive", + "kill_switch_reason": ctl.kill_switch.reason or "—", + "paper_trading_mode": "enabled" if rs.paper_trading_mode else "disabled", + "trade_capital_usd": f"{float(rs.trade_capital_usd):.2f} USD", + "trade_capital_usd_value": f"{float(rs.trade_capital_usd):.2f}", + "max_trade_capital_usd": max_trade_capital_usd, + "max_trade_capital_usd_value": max_trade_capital_usd_value, + "max_concurrent_trades": max_concurrent_trades, + "max_concurrent_trades_value": max_concurrent_trades_value, + "alerts_enabled": "enabled" if bool(alert_status.get("enabled", False)) else "disabled", + "alerts_channels": cd, + "alerts_min_severity": str(alert_status.get("min_severity", "—")), + "alerts_dedup_seconds": f"{dds:.0f}", + "alerts_last_result": str(alert_status.get("last_result", "unavailable")), + "alerts_last_attempted_at": str(alert_status.get("last_attempted_at") or "—"), + "alerts_last_success_at": str(alert_status.get("last_success_at") or "—"), + "alerts_last_event_title": last_event_title, + "alerts_last_error": str(alert_status.get("last_error") or "—"), + "alerts_last_channel_results": alerts_last_channel_results, + "tradable_pairs_display": tpd, + "tradable_pairs_value": ", ".join(ctl.tradable_pairs), + "strategy_mode": ctl.strategy_mode, + "strategy_stat_arb_enabled": strategy_stat_arb_enabled, + "strategy_profit_threshold": f"{ctl.strategy_profit_threshold:.6f}", + "strategy_max_depth_levels": str(ctl.strategy_max_depth_levels), + "updated_at": ctl.updated_at.isoformat(), + "start_endpoint": "/dashboard/control/start", + "stop_endpoint": "/dashboard/control/stop", + "kill_switch_endpoint": "/dashboard/control/kill-switch", + "config_endpoint": "/dashboard/control/config", + "chart_endpoint": "/dashboard/fragment/charts", + } + + +def _parse_form_body(body: bytes) -> dict[str, str]: + parsed = parse_qs(body.decode("utf-8"), keep_blank_values=True) + return {key: values[-1] for key, values in parsed.items() if values} + + +def _form_bool(value: str | None) -> bool: + if value is None: + return False + return value.lower() in {"1", "true", "yes", "on"} + + +def _parse_comma_separated_list(value: str | None) -> list[str]: + if value is None: + return [] + + items: list[str] = [] + for raw_item in value.split(","): + item = raw_item.strip().upper() + if item and item not in items: + items.append(item) + return items + + +def _normalize_fee_profile(profile: str) -> str: + return profile.strip().lower().replace("-", "_") + + +def _fee_rate_for_profile(profile: str, custom_fee_rate: float | None) -> float: + normalized = _normalize_fee_profile(profile) + profile_map = { + "standard": 0.0026, + "maker_heavy": 0.0016, + "taker_heavy": 0.0035, + } + if normalized == "custom": + if custom_fee_rate is None: + raise ValueError("custom fee profile requires custom_fee_rate") + if custom_fee_rate < 0.0: + raise ValueError("custom_fee_rate must be >= 0") + return custom_fee_rate + if normalized not in profile_map: + valid = ", ".join(sorted(list(profile_map.keys()) + ["custom"])) + raise ValueError(f"fee_profile must be one of: {valid}") + return profile_map[normalized] + + +def _parse_balances(raw: str) -> dict[str, float]: + balances: dict[str, float] = {} + for entry in raw.split(","): + stripped = entry.strip() + if not stripped: + continue + if "=" not in stripped: + raise ValueError("starting_balances must be in ASSET=value format") + asset, value = stripped.split("=", 1) + balances[asset.strip().upper()] = float(value) + if not balances: + raise ValueError("starting_balances must include at least one balance") + return balances + + +def _resolve_workspace_path(raw: str) -> Path: + candidate = Path(raw.strip()) + if not candidate.is_absolute(): + candidate = (_BACKTEST_ROOT / candidate).resolve() + else: + candidate = candidate.resolve() + return candidate + + +def _display_path(path: Path) -> str: + try: + return str(path.relative_to(_BACKTEST_ROOT)) + except ValueError: + return str(path) + + +def _build_cycles_from_events( + symbols: set[str], +) -> tuple[dict[str, list[TriangularCycle]], list[str]]: + graph = CurrencyGraph() + for symbol in sorted(symbols): + if "/" not in symbol: + continue + base, quote = symbol.upper().split("/", 1) + graph.add_pair(base, quote, f"{base}/{quote}") + cycles = graph.triangular_cycles() + return graph.index_cycles_by_pair(cycles), sorted(symbols) + + +def _recent_backtest_reports(request: Request) -> list[dict[str, object]]: + reports = getattr(request.app.state, "backtest_recent_reports", []) + if isinstance(reports, list): + return cast(list[dict[str, object]], reports) + return [] + + +def _backtesting_panel_context( + request: Request, + *, + status: str = "idle", + message: str = "Configure a replay run and execute backtest.", + latest_report: dict[str, object] | None = None, + defaults: dict[str, str] | None = None, +) -> dict[str, object]: + default_values = { + "events_path": "", + "starting_balances": "USD=1000.0", + "trade_capital": "100.0", + "min_profit_threshold": "0.0005", + "fee_profile": "standard", + "custom_fee_rate": "", + "slippage_bps": "4.0", + "execution_latency_ms": "20.0", + } + if defaults is not None: + default_values.update(defaults) + + reports = _recent_backtest_reports(request) + latest = latest_report or (reports[0] if reports else None) + + return { + "status": status, + "message": message, + "latest_report": latest, + "recent_reports": reports, + "run_endpoint": "/dashboard/backtesting/run", + "reports_endpoint": "/dashboard/api/backtesting/reports", + **default_values, + } + + +async def _dashboard_response( + request: Request, template_name: str = "dashboard.html" +) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name=template_name, + context={ + "title": "Arbitrade Dashboard", + "request": request, + "metrics_endpoint": "/dashboard/fragment/metrics", + "overview_endpoint": "/dashboard/fragment/overview", + "controls_endpoint": "/dashboard/fragment/controls", + "charts_endpoint": "/dashboard/fragment/charts", + "audit_endpoint": "/dashboard/fragment/audit", + "stream_endpoint": "/dashboard/stream/metrics", + "overview_stream_endpoint": "/dashboard/stream/overview", + }, + ) + + +@router.get("/", response_class=HTMLResponse) +async def home(request: Request) -> HTMLResponse: + return await _dashboard_response(request) + + +@router.get("/dashboard", response_class=HTMLResponse) +async def dashboard(request: Request) -> HTMLResponse: + return await _dashboard_response(request) + + +@router.get("/dashboard/backtesting", response_class=HTMLResponse) +async def dashboard_backtesting_page(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="backtesting.html", + context={ + "title": "Arbitrade Backtesting", + "request": request, + "panel_endpoint": "/dashboard/fragment/backtesting", + "dashboard_endpoint": "/dashboard", + }, + ) + + +@router.get("/dashboard/fragment/backtesting", response_class=HTMLResponse) +async def dashboard_backtesting_fragment(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="partials/backtesting_panel.html", + context={"request": request, **_backtesting_panel_context(request)}, + ) + + +@router.get("/dashboard/fragment/metrics", response_class=HTMLResponse) +async def dashboard_metrics(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="partials/metrics.html", + context={"request": request, **_dashboard_metrics(request)}, + ) + + +@router.get("/dashboard/fragment/overview", response_class=HTMLResponse) +async def dashboard_overview(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="partials/overview.html", + context={"request": request, **_dashboard_overview(request)}, + ) + + +@router.get("/dashboard/fragment/controls", response_class=HTMLResponse) +async def dashboard_controls(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="partials/controls.html", + context={"request": request, **_dashboard_controls(request)}, + ) + + +@router.get("/dashboard/fragment/charts", response_class=HTMLResponse) +async def dashboard_charts(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="partials/charts.html", + context={"request": request, **_dashboard_charts(request)}, + ) + + +@router.get("/dashboard/fragment/audit", response_class=HTMLResponse) +async def dashboard_audit(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="partials/audit.html", + context={"request": request, **_dashboard_audit(request)}, + ) + + +@router.get("/dashboard/api/alerts/status", response_class=JSONResponse) +async def dashboard_alert_status(request: Request) -> JSONResponse: + return JSONResponse(_alert_status_snapshot(request)) + + +@router.get("/dashboard/api/audit/recent", response_class=JSONResponse) +async def dashboard_audit_recent(request: Request) -> JSONResponse: + return JSONResponse(_dashboard_audit(request, limit=25)) + + +@router.get("/dashboard/api/backtesting/reports", response_class=JSONResponse) +async def dashboard_backtesting_reports(request: Request) -> JSONResponse: + return JSONResponse( + { + "generated_at": datetime.now(UTC).isoformat(), + "reports": _recent_backtest_reports(request), + } + ) + + +@router.post("/dashboard/backtesting/run", response_class=HTMLResponse) +async def dashboard_backtesting_run(request: Request) -> HTMLResponse: + form = _parse_form_body(await request.body()) + defaults = { + "events_path": form.get("events_path", ""), + "starting_balances": form.get("starting_balances", "USD=1000.0"), + "trade_capital": form.get("trade_capital", "100.0"), + "min_profit_threshold": form.get("min_profit_threshold", "0.0005"), + "fee_profile": _normalize_fee_profile(form.get("fee_profile", "standard")), + "custom_fee_rate": form.get("custom_fee_rate", ""), + "slippage_bps": form.get("slippage_bps", "4.0"), + "execution_latency_ms": form.get("execution_latency_ms", "20.0"), + } + + try: + events_path = _resolve_workspace_path(defaults["events_path"]) + if not events_path.exists() or not events_path.is_file(): + raise ValueError( + "events_path must reference an existing JSONL file") + + events = load_replay_events(events_path) + if not events: + raise ValueError("events file contains no replay events") + + custom_fee_rate = ( + float(defaults["custom_fee_rate"] + ) if defaults["custom_fee_rate"].strip() else None + ) + fee_rate = _fee_rate_for_profile( + defaults["fee_profile"], custom_fee_rate) + starting_balances = _parse_balances(defaults["starting_balances"]) + + trade_capital = float(defaults["trade_capital"]) + min_profit_threshold = float(defaults["min_profit_threshold"]) + slippage_bps = float(defaults["slippage_bps"]) + execution_latency_ms = float(defaults["execution_latency_ms"]) + + cycles_by_pair, available_pairs = _build_cycles_from_events( + {event.symbol.upper() for event in events} + ) + if not cycles_by_pair: + raise ValueError( + "unable to derive triangular cycles from provided events") + + config = BacktestConfig( + fee_rate=fee_rate, + min_profit_threshold=min_profit_threshold, + trade_capital=trade_capital, + slippage_bps=slippage_bps, + execution_latency_ms=execution_latency_ms, + ) + + async with _BACKTEST_RUN_LOCK: + engine = BacktestReplayEngine( + cycles_by_pair=cycles_by_pair, + available_pairs=available_pairs, + config=config, + started_at=events[0].occurred_at, + ) + report = await engine.run(events, starting_balances=starting_balances) + + report_item: dict[str, object] = { + "run_at": datetime.now(UTC).isoformat(), + "events_path": _display_path(events_path), + "status": "completed", + "config": { + "trade_capital": trade_capital, + "min_profit_threshold": min_profit_threshold, + "fee_profile": defaults["fee_profile"], + "fee_rate": fee_rate, + "slippage_bps": slippage_bps, + "execution_latency_ms": execution_latency_ms, + }, + "report": { + "processed_events": report.processed_events, + "opportunities_seen": report.opportunities_seen, + "trades_executed": report.trades_executed, + "win_rate": report.win_rate, + "fill_rate": report.fill_rate, + "realized_pnl_usd": report.realized_pnl_usd, + "max_drawdown_usd": report.max_drawdown_usd, + "miss_reasons": dict(report.miss_reasons), + "execution_latency_p50_ms": report.execution_latency_p50_ms, + "execution_latency_p95_ms": report.execution_latency_p95_ms, + "execution_latency_p99_ms": report.execution_latency_p99_ms, + }, + } + + reports = _recent_backtest_reports(request) + reports.insert(0, report_item) + del reports[20:] + + _record_audit( + request, + actor="dashboard_user", + event_type="dashboard.backtesting.run", + decision="completed", + payload={ + "events_path": report_item["events_path"], + "processed_events": report.processed_events, + "trades_executed": report.trades_executed, + "realized_pnl_usd": report.realized_pnl_usd, + }, + ) + + context = _backtesting_panel_context( + request, + status="completed", + message="Backtest run completed successfully.", + latest_report=report_item, + defaults=defaults, + ) + except ValueError as exc: + context = _backtesting_panel_context( + request, + status="failed", + message=str(exc), + defaults=defaults, + ) + + return templates.TemplateResponse( + request=request, + name="partials/backtesting_panel.html", + context={"request": request, **context}, + ) + + +@router.post("/dashboard/control/start", response_class=HTMLResponse) +async def dashboard_control_start(request: Request) -> HTMLResponse: + controls = _dashboard_controls_state(request) + controls.is_running = True + controls.mark_updated() + notifier = _alert_notifier(request) + if notifier is not None: + await notifier.notify( + category="system", + severity="info", + title="Execution started", + message="Dashboard control started execution.", + ) + _record_audit( + request, + actor="dashboard_user", + event_type="dashboard.control.start", + decision="approved", + payload={"execution_status": "running"}, + ) + return templates.TemplateResponse( + request=request, + name="partials/controls.html", + context={"request": request, **_dashboard_controls(request)}, + ) + + +@router.post("/dashboard/control/stop", response_class=HTMLResponse) +async def dashboard_control_stop(request: Request) -> HTMLResponse: + controls = _dashboard_controls_state(request) + controls.is_running = False + controls.mark_updated() + notifier = _alert_notifier(request) + if notifier is not None: + await notifier.notify( + category="system", + severity="warning", + title="Execution stopped", + message="Dashboard control stopped execution.", + ) + _record_audit( + request, + actor="dashboard_user", + event_type="dashboard.control.stop", + decision="approved", + payload={"execution_status": "stopped"}, + ) + return templates.TemplateResponse( + request=request, + name="partials/controls.html", + context={"request": request, **_dashboard_controls(request)}, + ) + + +@router.post("/dashboard/control/kill-switch", response_class=HTMLResponse) +async def dashboard_control_kill_switch(request: Request) -> HTMLResponse: + controls = _dashboard_controls_state(request) + form = _parse_form_body(await request.body()) + reason = form.get("reason") or "manual" + controls.kill_switch.activate(reason=reason) + controls.is_running = False + controls.mark_updated() + notifier = _alert_notifier(request) + if notifier is not None: + await notifier.notify( + category="threshold", + severity="critical", + title="Kill switch activated", + message="Kill switch triggered from dashboard control.", + details={"reason": reason}, + ) + _record_audit( + request, + actor="dashboard_user", + event_type="dashboard.control.kill_switch", + decision="approved", + payload={"reason": reason}, + ) + return templates.TemplateResponse( + request=request, + name="partials/controls.html", + context={"request": request, **_dashboard_controls(request)}, + ) + + +@router.post("/dashboard/control/config", response_class=HTMLResponse) +async def dashboard_control_config(request: Request) -> HTMLResponse: + ctl = _dashboard_controls_state(request) + rs = request.app.state.settings + form = _parse_form_body(await request.body()) + + if "trade_capital_usd" in form and form["trade_capital_usd"]: + rs.trade_capital_usd = float(form["trade_capital_usd"]) + if "max_trade_capital_usd" in form: + mtcv = form["max_trade_capital_usd"].strip() + rs.max_trade_capital_usd = float(mtcv) if mtcv else None + if "max_concurrent_trades" in form: + mcv = form["max_concurrent_trades"].strip() + rs.max_concurrent_trades = int(mcv) if mcv else None + + form_pairs = form.get("tradable_pairs") + ctl.tradable_pairs = _parse_comma_separated_list(form_pairs) + if "strategy_mode" in form and form["strategy_mode"].strip(): + strategy_mode = form["strategy_mode"].strip().lower() + allowed_strategy_modes = {"incremental", "paper", "live"} + if bool(getattr(rs, "strategy_enable_stat_arb_experiment", False)): + allowed_strategy_modes.add("stat_arb_experiment") + if strategy_mode not in allowed_strategy_modes: + e = f"strategy_mode must be one of: {', '.join(sorted(allowed_strategy_modes))}" + raise ValueError(e) + ctl.strategy_mode = strategy_mode + if "strategy_profit_threshold" in form: + if form["strategy_profit_threshold"].strip(): + spt = float(form["strategy_profit_threshold"]) + ctl.strategy_profit_threshold = spt + if "strategy_max_depth_levels" in form: + if form["strategy_max_depth_levels"].strip(): + smdl = int(form["strategy_max_depth_levels"]) + ctl.strategy_max_depth_levels = smdl + + rs.paper_trading_mode = _form_bool(form.get("paper_trading_mode")) + ctl.mark_updated() + + notifier = _alert_notifier(request) + if notifier is not None: + await notifier.notify( + category="system", + severity="info", + title="Runtime config updated", + message="Dashboard control updated runtime risk and execution settings.", + details={ + "trade_capital_usd": f"{rs.trade_capital_usd}", + "max_trade_capital_usd": ( + "none" if rs.max_trade_capital_usd is None else f"{rs.max_trade_capital_usd}" + ), + "max_concurrent_trades": ( + "none" if rs.max_concurrent_trades is None else f"{rs.max_concurrent_trades}" + ), + "paper_trading_mode": "true" if rs.paper_trading_mode else "false", + }, + ) + _record_audit( + request, + actor="dashboard_user", + event_type="dashboard.control.config", + decision="approved", + payload={ + "trade_capital_usd": rs.trade_capital_usd, + "max_trade_capital_usd": rs.max_trade_capital_usd, + "max_concurrent_trades": rs.max_concurrent_trades, + "paper_trading_mode": rs.paper_trading_mode, + "tradable_pairs": ctl.tradable_pairs, + "strategy_mode": ctl.strategy_mode, + "strategy_profit_threshold": ctl.strategy_profit_threshold, + "strategy_max_depth_levels": ctl.strategy_max_depth_levels, + }, + ) + + return templates.TemplateResponse( + request=request, + name="partials/controls.html", + context={"request": request, **_dashboard_controls(request)}, + ) + + +@router.get("/dashboard/stream/metrics") +async def dashboard_metrics_stream(request: Request) -> StreamingResponse: + fragment = ( + templates.get_template("partials/metrics.html") + .render( + request=request, + **_dashboard_metrics(request), + ) + .strip() + .replace("\n", "") + ) + + async def _event_stream() -> AsyncIterator[bytes]: + payload = json.dumps(fragment) + yield f"event: metrics\ndata: {payload}\n\n".encode() + + return StreamingResponse(_event_stream(), media_type="text/event-stream") + + +@router.get("/dashboard/stream/overview") +async def dashboard_overview_stream(request: Request) -> StreamingResponse: + fragment = ( + templates.get_template("partials/overview.html") + .render(request=request, **_dashboard_overview(request)) + .strip() + .replace("\n", "") + ) + + async def _event_stream() -> AsyncIterator[bytes]: + payload = json.dumps(fragment) + yield f"event: overview\ndata: {payload}\n\n".encode() + + return StreamingResponse(_event_stream(), media_type="text/event-stream") + + +@public_router.get("/health", response_class=JSONResponse) +async def health() -> JSONResponse: + return JSONResponse({"status": "ok", "service": "arbitrade"}) diff --git a/build/lib/arbitrade/backtesting/__init__.py b/build/lib/arbitrade/backtesting/__init__.py new file mode 100644 index 0000000..e6cbc60 --- /dev/null +++ b/build/lib/arbitrade/backtesting/__init__.py @@ -0,0 +1,35 @@ +from arbitrade.backtesting.replay import ( + BacktestConfig, + BacktestReplayEngine, + BacktestReport, + ReplayBookEvent, + ReplayClock, + load_replay_events, +) +from arbitrade.backtesting.sweep import ( + PromotionCriteria, + SweepArtifacts, + SweepParameters, + SweepResult, + build_parameter_grid, + persist_sweep_results, + run_parameter_search, + split_events_time_windows, +) + +__all__ = [ + "ReplayClock", + "ReplayBookEvent", + "BacktestConfig", + "BacktestReport", + "BacktestReplayEngine", + "load_replay_events", + "SweepParameters", + "SweepResult", + "SweepArtifacts", + "PromotionCriteria", + "split_events_time_windows", + "build_parameter_grid", + "run_parameter_search", + "persist_sweep_results", +] diff --git a/build/lib/arbitrade/backtesting/replay.py b/build/lib/arbitrade/backtesting/replay.py new file mode 100644 index 0000000..0a14ec7 --- /dev/null +++ b/build/lib/arbitrade/backtesting/replay.py @@ -0,0 +1,326 @@ +from __future__ import annotations + +import asyncio +from collections import Counter +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path +from typing import Any + +import orjson + +from arbitrade.detection.engine import IncrementalCycleDetector, OpportunityEvent +from arbitrade.detection.graph import TriangularCycle +from arbitrade.exchange.models import BookLevel +from arbitrade.execution.sequencer import TriangularExecutionSequencer +from arbitrade.market_data.order_book import OrderBook +from arbitrade.risk.pre_trade import PreTradeValidator +from arbitrade.risk.trade_limits import TradeLimitsGuard + + +@dataclass(slots=True) +class ReplayClock: + _current: datetime + + @classmethod + def at(cls, started_at: datetime) -> ReplayClock: + return cls(_current=started_at.astimezone(UTC)) + + @property + def now(self) -> datetime: + return self._current + + def advance_to(self, next_time: datetime) -> None: + normalized = next_time.astimezone(UTC) + if normalized < self._current: + raise ValueError("Replay events must be monotonic by timestamp") + self._current = normalized + + def advance_ms(self, milliseconds: float) -> None: + if milliseconds < 0.0: + raise ValueError("milliseconds must be >= 0") + self._current = self._current.fromtimestamp( + self._current.timestamp() + (milliseconds / 1000.0), + tz=UTC, + ) + + +@dataclass(frozen=True, slots=True) +class ReplayBookEvent: + occurred_at: datetime + symbol: str + bids: tuple[BookLevel, ...] + asks: tuple[BookLevel, ...] + + +@dataclass(frozen=True, slots=True) +class BacktestConfig: + fee_rate: float = 0.0026 + min_profit_threshold: float = 0.0005 + trade_capital: float = 100.0 + quote_asset: str = "USD" + slippage_bps: float = 4.0 + execution_latency_ms: float = 20.0 + max_depth_levels: int = 10 + max_concurrent_trades: int = 1 + min_order_size_by_pair: Mapping[str, float] | None = None + + +@dataclass(frozen=True, slots=True) +class BacktestReport: + started_at: datetime + finished_at: datetime + processed_events: int + opportunities_seen: int + trades_executed: int + win_rate: float | None + fill_rate: float | None + realized_pnl_usd: float + max_drawdown_usd: float + miss_reasons: Mapping[str, int] + execution_latency_p50_ms: float | None + execution_latency_p95_ms: float | None + execution_latency_p99_ms: float | None + + +class _SimulatedRestClient: + def __init__( + self, clock: ReplayClock, *, slippage_bps: float, execution_latency_ms: float + ) -> None: + self._clock = clock + self._slippage_bps = slippage_bps + self._execution_latency_ms = execution_latency_ms + self._sequence = 0 + self._last_fill_ratio = 1.0 + self._last_trade_latency_ms = execution_latency_ms + + @property + def last_fill_ratio(self) -> float: + return self._last_fill_ratio + + @property + def last_trade_latency_ms(self) -> float: + return self._last_trade_latency_ms + + async def place_market_order(self, *, pair: str, side: str, volume: float) -> dict[str, Any]: + self._sequence += 1 + self._clock.advance_ms(self._execution_latency_ms) + await asyncio.sleep(0) + + normalized_fill = max(0.85, 1.0 - (self._slippage_bps / 10000.0) * 8.0) + self._last_fill_ratio = normalized_fill + self._last_trade_latency_ms = self._execution_latency_ms + + return { + "txid": [f"sim-{self._sequence}"], + "status": "closed", + "pair": pair, + "side": side, + "requested_volume": volume, + "filled_volume": volume * normalized_fill, + "simulated_at": self._clock.now.isoformat(), + } + + +def _percentile(values: Sequence[float], percentile: float) -> float | None: + if not values: + return None + + ordered = sorted(values) + if percentile <= 0.0: + return ordered[0] + if percentile >= 100.0: + return ordered[-1] + + rank = (len(ordered) - 1) * (percentile / 100.0) + lower = int(rank) + upper = min(lower + 1, len(ordered) - 1) + weight = rank - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def _parse_book_levels(raw_levels: Any) -> tuple[BookLevel, ...]: + if not isinstance(raw_levels, list): + raise ValueError("Book levels must be a list") + + levels: list[BookLevel] = [] + for raw_level in raw_levels: + if ( + not isinstance(raw_level, list) + or len(raw_level) != 2 + or not isinstance(raw_level[0], int | float) + or not isinstance(raw_level[1], int | float) + ): + raise ValueError("Each level must be [price, volume]") + levels.append(BookLevel(price=float(raw_level[0]), volume=float(raw_level[1]))) + + return tuple(levels) + + +def load_replay_events(path: Path) -> list[ReplayBookEvent]: + events: list[ReplayBookEvent] = [] + for line in path.read_text(encoding="utf-8").splitlines(): + if not line.strip(): + continue + parsed = orjson.loads(line) + if not isinstance(parsed, dict): + raise ValueError("Each JSONL row must be an object") + + timestamp_raw = parsed.get("timestamp") + symbol_raw = parsed.get("symbol") + if not isinstance(timestamp_raw, str) or not isinstance(symbol_raw, str): + raise ValueError("Each event must include timestamp and symbol") + + occurred_at = datetime.fromisoformat(timestamp_raw.replace("Z", "+00:00")).astimezone(UTC) + events.append( + ReplayBookEvent( + occurred_at=occurred_at, + symbol=symbol_raw, + bids=_parse_book_levels(parsed.get("bids")), + asks=_parse_book_levels(parsed.get("asks")), + ) + ) + + return sorted(events, key=lambda event: event.occurred_at) + + +class BacktestReplayEngine: + def __init__( + self, + *, + cycles_by_pair: Mapping[str, list[TriangularCycle]], + available_pairs: Sequence[str], + config: BacktestConfig, + started_at: datetime, + ) -> None: + self._config = config + self._clock = ReplayClock.at(started_at) + self._books: dict[str, OrderBook] = {} + + self._detector = IncrementalCycleDetector( + cycles_by_pair, + fee_rate=config.fee_rate, + max_depth_levels=config.max_depth_levels, + min_profit_threshold=config.min_profit_threshold, + min_order_size_by_pair=config.min_order_size_by_pair, + ) + self._pre_trade = PreTradeValidator() + self._trade_limits = TradeLimitsGuard(max_concurrent_trades=config.max_concurrent_trades) + self._simulated_rest = _SimulatedRestClient( + self._clock, + slippage_bps=config.slippage_bps, + execution_latency_ms=config.execution_latency_ms, + ) + self._sequencer = TriangularExecutionSequencer( + self._simulated_rest, + available_pairs=available_pairs, + ) + + @staticmethod + def _exposure_for_event(event: OpportunityEvent) -> dict[str, float]: + currencies = [part for part in event.cycle.split("->") if part] + if len(currencies) < 2: + return {} + + origin = currencies[0] + return { + currency: event.allocated_capital for currency in currencies[1:] if currency != origin + } + + async def run( + self, + events: Sequence[ReplayBookEvent], + *, + starting_balances: Mapping[str, float], + ) -> BacktestReport: + miss_reasons: Counter[str] = Counter() + + processed_events = 0 + opportunities_seen = 0 + trades_executed = 0 + + realized_pnl = 0.0 + equity = float(starting_balances.get(self._config.quote_asset.upper(), 0.0)) + peak_equity = equity + max_drawdown = 0.0 + + fill_samples: list[float] = [] + realized_samples: list[float] = [] + execution_latencies: list[float] = [] + + for event in events: + self._clock.advance_to(event.occurred_at) + processed_events += 1 + + book = self._books.setdefault(event.symbol.upper(), OrderBook()) + book.apply_bids(event.bids) + book.apply_asks(event.asks) + + opportunities = self._detector.opportunities_for_updated_pair( + event.symbol, + self._books, + base_capital=self._config.trade_capital, + ) + opportunities_seen += len(opportunities) + + for opportunity in opportunities: + required_by_asset = { + self._config.quote_asset.upper(): opportunity.allocated_capital + } + if not self._pre_trade.validate( + balances_by_asset=starting_balances, + required_by_asset=required_by_asset, + ): + miss_reasons["insufficient_balance"] += 1 + continue + + exposure = self._exposure_for_event(opportunity) + if not self._trade_limits.is_trade_allowed(exposure): + miss_reasons["trade_limit"] += 1 + continue + + self._trade_limits.open_trade(exposure) + result = await self._sequencer.execute(opportunity) + self._trade_limits.close_trade(exposure) + + execution_latencies.append(self._simulated_rest.last_trade_latency_ms) + fill_samples.append(self._simulated_rest.last_fill_ratio) + + if not result.success: + miss_reasons["execution_failed"] += 1 + continue + + slippage_cost = ( + opportunity.allocated_capital + * (self._config.slippage_bps / 10000.0) + * max(result.completed_legs, 1) + ) + realized_trade_pnl = opportunity.est_profit - slippage_cost + realized_samples.append(realized_trade_pnl) + + realized_pnl += realized_trade_pnl + equity += realized_trade_pnl + peak_equity = max(peak_equity, equity) + max_drawdown = max(max_drawdown, peak_equity - equity) + trades_executed += 1 + + wins = sum(1 for pnl in realized_samples if pnl > 0.0) + win_rate = (wins / len(realized_samples)) if realized_samples else None + fill_rate = (sum(fill_samples) / len(fill_samples)) if fill_samples else None + + return BacktestReport( + started_at=events[0].occurred_at if events else self._clock.now, + finished_at=events[-1].occurred_at if events else self._clock.now, + processed_events=processed_events, + opportunities_seen=opportunities_seen, + trades_executed=trades_executed, + win_rate=win_rate, + fill_rate=fill_rate, + realized_pnl_usd=realized_pnl, + max_drawdown_usd=max_drawdown, + miss_reasons=dict(miss_reasons), + execution_latency_p50_ms=_percentile(execution_latencies, 50.0), + execution_latency_p95_ms=_percentile(execution_latencies, 95.0), + execution_latency_p99_ms=_percentile(execution_latencies, 99.0), + ) diff --git a/build/lib/arbitrade/backtesting/sweep.py b/build/lib/arbitrade/backtesting/sweep.py new file mode 100644 index 0000000..67c44a7 --- /dev/null +++ b/build/lib/arbitrade/backtesting/sweep.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import asyncio +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from pathlib import Path + +import orjson + +from arbitrade.backtesting.replay import ( + BacktestConfig, + BacktestReplayEngine, + BacktestReport, + ReplayBookEvent, +) +from arbitrade.detection.graph import TriangularCycle + + +@dataclass(frozen=True, slots=True) +class SweepParameters: + min_profit_threshold: float + trade_capital: float + pair_universe: tuple[str, ...] + staleness_threshold_seconds: float + + +@dataclass(frozen=True, slots=True) +class PromotionCriteria: + min_test_realized_pnl_usd: float = 0.0 + min_test_win_rate: float = 0.5 + min_test_fill_rate: float = 0.9 + max_test_drawdown_usd: float = 25.0 + max_generalization_gap_ratio: float = 0.5 + + +@dataclass(frozen=True, slots=True) +class SweepResult: + parameters: SweepParameters + train_report: BacktestReport + test_report: BacktestReport + train_score: float + test_score: float + generalization_gap_ratio: float + overfit_detected: bool + promotion_ready: bool + promotion_reasons: tuple[str, ...] + train_event_count: int + test_event_count: int + + +@dataclass(frozen=True, slots=True) +class SweepArtifacts: + results: tuple[SweepResult, ...] + promoted: tuple[SweepResult, ...] + train_window: tuple[datetime, datetime] | None + test_window: tuple[datetime, datetime] | None + + +def split_events_time_windows( + events: Sequence[ReplayBookEvent], + *, + train_ratio: float, +) -> tuple[list[ReplayBookEvent], list[ReplayBookEvent]]: + if train_ratio <= 0.0 or train_ratio >= 1.0: + raise ValueError("train_ratio must be between 0 and 1") + if len(events) < 2: + raise ValueError("at least two events are required for time split") + + split_index = max(1, min(len(events) - 1, int(len(events) * train_ratio))) + return list(events[:split_index]), list(events[split_index:]) + + +def build_parameter_grid( + *, + theta_values: Sequence[float], + trade_capital_values: Sequence[float], + pair_universes: Sequence[Sequence[str]], + staleness_threshold_values: Sequence[float], +) -> list[SweepParameters]: + if not theta_values: + raise ValueError("theta_values must not be empty") + if not trade_capital_values: + raise ValueError("trade_capital_values must not be empty") + if not pair_universes: + raise ValueError("pair_universes must not be empty") + if not staleness_threshold_values: + raise ValueError("staleness_threshold_values must not be empty") + + grid: list[SweepParameters] = [] + for theta in theta_values: + for trade_capital in trade_capital_values: + for pair_universe in pair_universes: + normalized_universe = tuple( + sorted({pair.upper() for pair in pair_universe})) + for staleness_threshold in staleness_threshold_values: + grid.append( + SweepParameters( + min_profit_threshold=float(theta), + trade_capital=float(trade_capital), + pair_universe=normalized_universe, + staleness_threshold_seconds=float( + staleness_threshold), + ) + ) + return grid + + +def _filter_events_for_parameters( + events: Sequence[ReplayBookEvent], + *, + pair_universe: set[str], + staleness_threshold_seconds: float, +) -> list[ReplayBookEvent]: + if staleness_threshold_seconds <= 0.0: + raise ValueError("staleness_threshold_seconds must be > 0") + + filtered: list[ReplayBookEvent] = [] + last_seen_by_symbol: dict[str, datetime] = {} + + for event in events: + symbol = event.symbol.upper() + if symbol not in pair_universe: + continue + + previous = last_seen_by_symbol.get(symbol) + last_seen_by_symbol[symbol] = event.occurred_at + if previous is None: + filtered.append(event) + continue + + gap_seconds = (event.occurred_at - previous).total_seconds() + if gap_seconds <= staleness_threshold_seconds: + filtered.append(event) + + return filtered + + +def _restrict_cycles_by_pair( + cycles_by_pair: Mapping[str, list[TriangularCycle]], + *, + pair_universe: set[str], +) -> dict[str, list[TriangularCycle]]: + restricted: dict[str, list[TriangularCycle]] = {} + for pair_symbol, cycles in cycles_by_pair.items(): + normalized_pair = pair_symbol.upper() + if normalized_pair not in pair_universe: + continue + + kept = [cycle for cycle in cycles if all( + pair.upper() in pair_universe for pair in cycle.pairs)] + if kept: + restricted[normalized_pair] = kept + return restricted + + +def _score_report(report: BacktestReport) -> float: + win_rate_bonus = (report.win_rate or 0.0) * 100.0 + fill_rate_bonus = (report.fill_rate or 0.0) * 50.0 + return report.realized_pnl_usd + win_rate_bonus + fill_rate_bonus - report.max_drawdown_usd + + +def _safe_ratio(numerator: float, denominator: float) -> float: + if denominator <= 0.0: + return 0.0 if numerator <= 0.0 else 1.0 + return max(0.0, numerator / denominator) + + +def _evaluate_promotion( + *, + result: SweepResult, + criteria: PromotionCriteria, +) -> tuple[bool, tuple[str, ...]]: + reasons: list[str] = [] + test = result.test_report + + if test.realized_pnl_usd < criteria.min_test_realized_pnl_usd: + reasons.append( + "test_realized_pnl_below_threshold" + ) + if (test.win_rate or 0.0) < criteria.min_test_win_rate: + reasons.append("test_win_rate_below_threshold") + if (test.fill_rate or 0.0) < criteria.min_test_fill_rate: + reasons.append("test_fill_rate_below_threshold") + if test.max_drawdown_usd > criteria.max_test_drawdown_usd: + reasons.append("test_drawdown_above_threshold") + if result.generalization_gap_ratio > criteria.max_generalization_gap_ratio: + reasons.append("generalization_gap_above_threshold") + + return (not reasons), tuple(reasons) + + +def _run_backtest( + *, + events: Sequence[ReplayBookEvent], + cycles_by_pair: Mapping[str, list[TriangularCycle]], + available_pairs: Sequence[str], + config: BacktestConfig, + starting_balances: Mapping[str, float], +) -> BacktestReport: + started_at = events[0].occurred_at if events else datetime.now(UTC) + engine = BacktestReplayEngine( + cycles_by_pair=cycles_by_pair, + available_pairs=available_pairs, + config=config, + started_at=started_at, + ) + return asyncio.run(engine.run(events, starting_balances=starting_balances)) + + +def run_parameter_search( + *, + events: Sequence[ReplayBookEvent], + cycles_by_pair: Mapping[str, list[TriangularCycle]], + parameter_grid: Sequence[SweepParameters], + starting_balances: Mapping[str, float], + train_ratio: float, + promotion_criteria: PromotionCriteria | None = None, + max_concurrent_trades: int = 1, + max_depth_levels: int = 10, + quote_asset: str = "USD", +) -> SweepArtifacts: + criteria = promotion_criteria or PromotionCriteria() + train_events, test_events = split_events_time_windows( + events, train_ratio=train_ratio) + + results: list[SweepResult] = [] + promoted: list[SweepResult] = [] + + for parameters in parameter_grid: + allowed_pairs = set(parameters.pair_universe) + filtered_train = _filter_events_for_parameters( + train_events, + pair_universe=allowed_pairs, + staleness_threshold_seconds=parameters.staleness_threshold_seconds, + ) + filtered_test = _filter_events_for_parameters( + test_events, + pair_universe=allowed_pairs, + staleness_threshold_seconds=parameters.staleness_threshold_seconds, + ) + + if not filtered_train or not filtered_test: + continue + + restricted_cycles = _restrict_cycles_by_pair( + cycles_by_pair, + pair_universe=allowed_pairs, + ) + if not restricted_cycles: + continue + + config = BacktestConfig( + min_profit_threshold=parameters.min_profit_threshold, + trade_capital=parameters.trade_capital, + max_concurrent_trades=max_concurrent_trades, + max_depth_levels=max_depth_levels, + quote_asset=quote_asset, + ) + + train_report = _run_backtest( + events=filtered_train, + cycles_by_pair=restricted_cycles, + available_pairs=sorted(allowed_pairs), + config=config, + starting_balances=starting_balances, + ) + test_report = _run_backtest( + events=filtered_test, + cycles_by_pair=restricted_cycles, + available_pairs=sorted(allowed_pairs), + config=config, + starting_balances=starting_balances, + ) + + train_score = _score_report(train_report) + test_score = _score_report(test_report) + score_drop = max(0.0, train_score - test_score) + generalization_gap_ratio = _safe_ratio(score_drop, abs(train_score)) + overfit_detected = generalization_gap_ratio > criteria.max_generalization_gap_ratio + + base_result = SweepResult( + parameters=parameters, + train_report=train_report, + test_report=test_report, + train_score=train_score, + test_score=test_score, + generalization_gap_ratio=generalization_gap_ratio, + overfit_detected=overfit_detected, + promotion_ready=False, + promotion_reasons=(), + train_event_count=len(filtered_train), + test_event_count=len(filtered_test), + ) + promotion_ready, promotion_reasons = _evaluate_promotion( + result=base_result, criteria=criteria) + completed_result = SweepResult( + parameters=base_result.parameters, + train_report=base_result.train_report, + test_report=base_result.test_report, + train_score=base_result.train_score, + test_score=base_result.test_score, + generalization_gap_ratio=base_result.generalization_gap_ratio, + overfit_detected=base_result.overfit_detected, + promotion_ready=promotion_ready, + promotion_reasons=promotion_reasons, + train_event_count=base_result.train_event_count, + test_event_count=base_result.test_event_count, + ) + + results.append(completed_result) + if completed_result.promotion_ready: + promoted.append(completed_result) + + results.sort(key=lambda item: item.test_score, reverse=True) + promoted.sort(key=lambda item: item.test_score, reverse=True) + + train_window: tuple[datetime, datetime] | None = None + test_window: tuple[datetime, datetime] | None = None + if train_events: + train_window = (train_events[0].occurred_at, + train_events[-1].occurred_at) + if test_events: + test_window = (test_events[0].occurred_at, test_events[-1].occurred_at) + + return SweepArtifacts( + results=tuple(results), + promoted=tuple(promoted), + train_window=train_window, + test_window=test_window, + ) + + +def _report_to_dict(report: BacktestReport) -> dict[str, object]: + return { + "started_at": report.started_at.isoformat(), + "finished_at": report.finished_at.isoformat(), + "processed_events": report.processed_events, + "opportunities_seen": report.opportunities_seen, + "trades_executed": report.trades_executed, + "win_rate": report.win_rate, + "fill_rate": report.fill_rate, + "realized_pnl_usd": report.realized_pnl_usd, + "max_drawdown_usd": report.max_drawdown_usd, + "miss_reasons": dict(report.miss_reasons), + "execution_latency_p50_ms": report.execution_latency_p50_ms, + "execution_latency_p95_ms": report.execution_latency_p95_ms, + "execution_latency_p99_ms": report.execution_latency_p99_ms, + } + + +def persist_sweep_results(path: Path, artifacts: SweepArtifacts) -> None: + payload = { + "generated_at": datetime.now(UTC).isoformat(), + "train_window": ( + { + "started_at": artifacts.train_window[0].isoformat(), + "finished_at": artifacts.train_window[1].isoformat(), + } + if artifacts.train_window is not None + else None + ), + "test_window": ( + { + "started_at": artifacts.test_window[0].isoformat(), + "finished_at": artifacts.test_window[1].isoformat(), + } + if artifacts.test_window is not None + else None + ), + "results": [ + { + "parameters": { + "min_profit_threshold": result.parameters.min_profit_threshold, + "trade_capital": result.parameters.trade_capital, + "pair_universe": list(result.parameters.pair_universe), + "staleness_threshold_seconds": result.parameters.staleness_threshold_seconds, + }, + "train_report": _report_to_dict(result.train_report), + "test_report": _report_to_dict(result.test_report), + "train_score": result.train_score, + "test_score": result.test_score, + "generalization_gap_ratio": result.generalization_gap_ratio, + "overfit_detected": result.overfit_detected, + "promotion_ready": result.promotion_ready, + "promotion_reasons": list(result.promotion_reasons), + "train_event_count": result.train_event_count, + "test_event_count": result.test_event_count, + } + for result in artifacts.results + ], + } + + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(orjson.dumps( + payload, option=orjson.OPT_INDENT_2 | orjson.OPT_SORT_KEYS)) diff --git a/build/lib/arbitrade/config/__init__.py b/build/lib/arbitrade/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/build/lib/arbitrade/config/secrets.py b/build/lib/arbitrade/config/secrets.py new file mode 100644 index 0000000..a04d347 --- /dev/null +++ b/build/lib/arbitrade/config/secrets.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import base64 +import os +from dataclasses import dataclass + +import keyring +from cryptography.fernet import Fernet + + +@dataclass(slots=True) +class SecretStore: + service_name: str = "arbitrade" + + def _load_or_create_key(self, key_env: str | None = None) -> bytes: + if key_env: + return key_env.encode("utf-8") + + existing = keyring.get_password(self.service_name, "fernet_key") + if existing: + return existing.encode("utf-8") + + generated = Fernet.generate_key() + keyring.set_password(self.service_name, "fernet_key", generated.decode("utf-8")) + return generated + + def encrypt(self, plaintext: str, key_env: str | None = None) -> str: + key = self._load_or_create_key(key_env) + token = Fernet(key).encrypt(plaintext.encode("utf-8")) + return token.decode("utf-8") + + def decrypt(self, ciphertext: str, key_env: str | None = None) -> str: + key = self._load_or_create_key(key_env) + value = Fernet(key).decrypt(ciphertext.encode("utf-8")) + return value.decode("utf-8") + + @staticmethod + def generate_env_key() -> str: + return base64.urlsafe_b64encode(os.urandom(32)).decode("utf-8") diff --git a/build/lib/arbitrade/config/settings.py b/build/lib/arbitrade/config/settings.py new file mode 100644 index 0000000..11ac3f0 --- /dev/null +++ b/build/lib/arbitrade/config/settings.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path + +from pydantic import Field, field_validator, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + env_ignore_empty=True, + ) + + app_env: str = Field(default="dev", alias="APP_ENV") + app_host: str = Field(default="0.0.0.0", alias="APP_HOST") + app_port: int = Field(default=9090, alias="APP_PORT") + + log_level: str = Field(default="INFO", alias="LOG_LEVEL") + log_json: bool = Field(default=True, alias="LOG_JSON") + + dashboard_auth_username: str | None = Field( + default=None, + alias="DASHBOARD_AUTH_USERNAME", + ) + dashboard_auth_password: str | None = Field( + default=None, + alias="DASHBOARD_AUTH_PASSWORD", + ) + + alerts_enabled: bool = Field(default=True, alias="ALERTS_ENABLED") + alert_min_severity: str = Field( + default="warning", alias="ALERT_MIN_SEVERITY") + alert_dedup_seconds: float = Field( + default=30.0, alias="ALERT_DEDUP_SECONDS") + alert_on_trade_events: bool = Field( + default=True, alias="ALERT_ON_TRADE_EVENTS") + alert_on_error_events: bool = Field( + default=True, alias="ALERT_ON_ERROR_EVENTS") + alert_on_threshold_events: bool = Field( + default=True, alias="ALERT_ON_THRESHOLD_EVENTS") + alert_on_system_events: bool = Field( + default=True, alias="ALERT_ON_SYSTEM_EVENTS") + + telegram_alerts_enabled: bool = Field( + default=False, alias="TELEGRAM_ALERTS_ENABLED") + telegram_bot_token: str | None = Field( + default=None, alias="TELEGRAM_BOT_TOKEN") + telegram_chat_id: str | None = Field( + default=None, alias="TELEGRAM_CHAT_ID") + + discord_alerts_enabled: bool = Field( + default=False, alias="DISCORD_ALERTS_ENABLED") + discord_webhook_url: str | None = Field( + default=None, alias="DISCORD_WEBHOOK_URL") + + email_alerts_enabled: bool = Field( + default=False, alias="EMAIL_ALERTS_ENABLED") + email_smtp_host: str | None = Field(default=None, alias="EMAIL_SMTP_HOST") + email_smtp_port: int = Field(default=587, alias="EMAIL_SMTP_PORT") + email_smtp_username: str | None = Field( + default=None, alias="EMAIL_SMTP_USERNAME") + email_smtp_password: str | None = Field( + default=None, alias="EMAIL_SMTP_PASSWORD") + email_alert_from: str | None = Field( + default=None, alias="EMAIL_ALERT_FROM") + email_alert_to: str | None = Field(default=None, alias="EMAIL_ALERT_TO") + email_smtp_use_tls: bool = Field(default=True, alias="EMAIL_SMTP_USE_TLS") + + duckdb_path: Path = Field(default=Path( + "./data/arbitrade.duckdb"), alias="DUCKDB_PATH") + + kraken_rest_url: str = Field( + default="https://api.kraken.com", alias="KRAKEN_REST_URL") + kraken_ws_url: str = Field( + default="wss://ws.kraken.com/v2", alias="KRAKEN_WS_URL") + kraken_private_rate_limit_seconds: float = Field( + default=1.0, alias="KRAKEN_PRIVATE_RATE_LIMIT_SECONDS" + ) + kraken_http_timeout_seconds: float = Field( + default=10.0, alias="KRAKEN_HTTP_TIMEOUT_SECONDS") + kraken_retry_attempts: int = Field( + default=3, alias="KRAKEN_RETRY_ATTEMPTS") + kraken_retry_base_delay_seconds: float = Field( + default=0.25, alias="KRAKEN_RETRY_BASE_DELAY_SECONDS" + ) + kraken_api_key: str | None = Field(default=None, alias="KRAKEN_API_KEY") + kraken_api_secret: str | None = Field( + default=None, alias="KRAKEN_API_SECRET") + kraken_api_key_permissions: str = Field( + default="query,trade", + alias="KRAKEN_API_KEY_PERMISSIONS", + ) + ws_heartbeat_timeout_seconds: float = Field( + default=20.0, alias="WS_HEARTBEAT_TIMEOUT_SECONDS") + ws_max_staleness_seconds: float = Field( + default=5.0, alias="WS_MAX_STALENESS_SECONDS") + strategy_enable_stat_arb_experiment: bool = Field( + default=False, + alias="STRATEGY_ENABLE_STAT_ARB_EXPERIMENT", + ) + strategy_stat_arb_lookback_window: int = Field( + default=120, + alias="STRATEGY_STAT_ARB_LOOKBACK_WINDOW", + ) + strategy_stat_arb_entry_zscore: float = Field( + default=2.0, + alias="STRATEGY_STAT_ARB_ENTRY_ZSCORE", + ) + strategy_stat_arb_exit_zscore: float = Field( + default=0.5, + alias="STRATEGY_STAT_ARB_EXIT_ZSCORE", + ) + strategy_stat_arb_max_holding_seconds: float = Field( + default=900.0, + alias="STRATEGY_STAT_ARB_MAX_HOLDING_SECONDS", + ) + paper_trading_mode: bool = Field(default=True, alias="PAPER_TRADING_MODE") + trade_capital_usd: float = Field(default=100.0, alias="TRADE_CAPITAL_USD") + max_trade_capital_usd: float = Field( + default=100.0, alias="MAX_TRADE_CAPITAL_USD") + max_concurrent_trades: int | None = Field( + default=None, alias="MAX_CONCURRENT_TRADES") + max_exposure_per_asset_usd: float | None = Field( + default=None, + alias="MAX_EXPOSURE_PER_ASSET_USD", + ) + quote_balance_asset: str = Field( + default="USD", alias="QUOTE_BALANCE_ASSET") + min_order_size_usd: float | None = Field( + default=None, alias="MIN_ORDER_SIZE_USD") + kill_switch_active: bool = Field(default=False, alias="KILL_SWITCH_ACTIVE") + daily_loss_limit_usd: float | None = Field( + default=None, alias="DAILY_LOSS_LIMIT_USD") + cumulative_loss_limit_usd: float | None = Field( + default=None, alias="CUMULATIVE_LOSS_LIMIT_USD") + max_source_latency_ms: float | None = Field( + default=None, alias="MAX_SOURCE_LATENCY_MS") + max_apply_latency_ms: float | None = Field( + default=None, alias="MAX_APPLY_LATENCY_MS") + max_consecutive_failures: int | None = Field( + default=None, alias="MAX_CONSECUTIVE_FAILURES") + + fernet_key: str | None = Field(default=None, alias="FERNET_KEY") + + @field_validator("app_env") + @classmethod + def _validate_app_env(cls, value: str) -> str: + normalized = value.strip().lower() + if normalized not in {"dev", "test", "prod"}: + raise ValueError("APP_ENV must be one of: dev, test, prod") + return normalized + + @field_validator("log_level") + @classmethod + def _validate_log_level(cls, value: str) -> str: + normalized = value.strip().upper() + if normalized not in {"DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"}: + raise ValueError( + "LOG_LEVEL must be one of: DEBUG, INFO, WARNING, ERROR, CRITICAL") + return normalized + + @field_validator("alert_min_severity") + @classmethod + def _validate_alert_severity(cls, value: str) -> str: + normalized = value.strip().lower() + if normalized not in {"info", "warning", "error", "critical"}: + raise ValueError( + "ALERT_MIN_SEVERITY must be one of: info, warning, error, critical") + return normalized + + @model_validator(mode="after") + def _validate_security_constraints(self) -> Settings: + if bool(self.dashboard_auth_username) ^ bool(self.dashboard_auth_password): + raise ValueError( + "dashboard auth requires both username and password") + + if bool(self.kraken_api_key) ^ bool(self.kraken_api_secret): + raise ValueError( + "Kraken API auth requires both API key and secret") + + permissions = { + token.strip().lower() + for token in self.kraken_api_key_permissions.split(",") + if token.strip() + } + if permissions and ("query" not in permissions or "trade" not in permissions): + raise ValueError( + "KRAKEN_API_KEY_PERMISSIONS must include query and trade") + if "withdraw" in permissions or "withdrawals" in permissions: + raise ValueError( + "KRAKEN_API_KEY_PERMISSIONS must not include withdrawal scope") + + if self.alert_dedup_seconds < 0.0: + raise ValueError("ALERT_DEDUP_SECONDS must be >= 0") + + if self.strategy_stat_arb_lookback_window < 2: + raise ValueError("STRATEGY_STAT_ARB_LOOKBACK_WINDOW must be >= 2") + if self.strategy_stat_arb_entry_zscore <= 0.0: + raise ValueError("STRATEGY_STAT_ARB_ENTRY_ZSCORE must be > 0") + if self.strategy_stat_arb_exit_zscore < 0.0: + raise ValueError("STRATEGY_STAT_ARB_EXIT_ZSCORE must be >= 0") + if self.strategy_stat_arb_entry_zscore <= self.strategy_stat_arb_exit_zscore: + raise ValueError( + "STRATEGY_STAT_ARB_ENTRY_ZSCORE must be greater than STRATEGY_STAT_ARB_EXIT_ZSCORE" + ) + if self.strategy_stat_arb_max_holding_seconds <= 0.0: + raise ValueError( + "STRATEGY_STAT_ARB_MAX_HOLDING_SECONDS must be > 0") + + return self + + +@lru_cache(maxsize=1) +def get_settings() -> Settings: + return Settings() diff --git a/build/lib/arbitrade/detection/__init__.py b/build/lib/arbitrade/detection/__init__.py new file mode 100644 index 0000000..efdc829 --- /dev/null +++ b/build/lib/arbitrade/detection/__init__.py @@ -0,0 +1,12 @@ +"""Arbitrage detection package.""" + +from arbitrade.detection.engine import CycleScore, IncrementalCycleDetector, OpportunityEvent +from arbitrade.detection.graph import CurrencyGraph, TriangularCycle + +__all__ = [ + "CurrencyGraph", + "TriangularCycle", + "CycleScore", + "OpportunityEvent", + "IncrementalCycleDetector", +] diff --git a/build/lib/arbitrade/detection/benchmark.py b/build/lib/arbitrade/detection/benchmark.py new file mode 100644 index 0000000..3ec8056 --- /dev/null +++ b/build/lib/arbitrade/detection/benchmark.py @@ -0,0 +1,113 @@ +from __future__ import annotations + +import argparse +import statistics +import time +from dataclasses import asdict, dataclass + +import orjson + +from arbitrade.detection.engine import IncrementalCycleDetector +from arbitrade.detection.graph import CurrencyGraph +from arbitrade.exchange.models import BookLevel +from arbitrade.market_data.order_book import OrderBook + + +@dataclass(frozen=True, slots=True) +class DetectionBenchmarkResult: + iterations: int + total_ms: float + avg_ms: float + p50_ms: float + p95_ms: float + max_ms: float + target_ms: float + + @property + def meets_target(self) -> bool: + return self.p95_ms <= self.target_ms + + +def _make_book(*, bid: float, ask: float) -> OrderBook: + book = OrderBook() + book.apply_bids([BookLevel(price=bid, volume=10.0)]) + book.apply_asks([BookLevel(price=ask, volume=10.0)]) + return book + + +def _build_detector_and_books() -> tuple[IncrementalCycleDetector, dict[str, OrderBook]]: + asset_pairs = { + "XXBTZUSD": {"wsname": "BTC/USD"}, + "XETHXXBT": {"wsname": "ETH/BTC"}, + "XETHZUSD": {"wsname": "ETH/USD"}, + } + graph = CurrencyGraph.from_kraken_asset_pairs(asset_pairs) + cycles = graph.triangular_cycles() + index = graph.index_cycles_by_pair(cycles) + + detector = IncrementalCycleDetector( + index, + fee_rate=0.001, + min_profit_threshold=0.001, + max_depth_levels=5, + max_book_age_seconds=10.0, + ) + + books = { + "BTC/USD": _make_book(bid=99.9, ask=100.0), + "ETH/BTC": _make_book(bid=0.049, ask=0.05), + "ETH/USD": _make_book(bid=5.2, ask=5.21), + } + return detector, books + + +def run_incremental_detection_benchmark( + *, + iterations: int = 50_000, + target_ms: float = 1.0, +) -> DetectionBenchmarkResult: + if iterations <= 0: + raise ValueError("iterations must be > 0") + + detector, books = _build_detector_and_books() + + samples_ms: list[float] = [] + started_ns = time.perf_counter_ns() + for _ in range(iterations): + t0_ns = time.perf_counter_ns() + detector.score_updated_pair("ETH/BTC", books) + elapsed_ms = (time.perf_counter_ns() - t0_ns) / 1_000_000 + samples_ms.append(elapsed_ms) + + total_ms = (time.perf_counter_ns() - started_ns) / 1_000_000 + return DetectionBenchmarkResult( + iterations=iterations, + total_ms=total_ms, + avg_ms=statistics.fmean(samples_ms), + p50_ms=statistics.quantiles(samples_ms, n=100)[49], + p95_ms=statistics.quantiles(samples_ms, n=100)[94], + max_ms=max(samples_ms), + target_ms=target_ms, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Benchmark incremental detection latency") + parser.add_argument("--iterations", type=int, default=50_000) + parser.add_argument("--target-ms", type=float, default=1.0) + args = parser.parse_args() + + result = run_incremental_detection_benchmark( + iterations=args.iterations, + target_ms=args.target_ms, + ) + + payload = { + **asdict(result), + "meets_target": result.meets_target, + } + print(orjson.dumps(payload).decode("utf-8")) + + +if __name__ == "__main__": + main() diff --git a/build/lib/arbitrade/detection/engine.py b/build/lib/arbitrade/detection/engine.py new file mode 100644 index 0000000..fbed5da --- /dev/null +++ b/build/lib/arbitrade/detection/engine.py @@ -0,0 +1,295 @@ +from __future__ import annotations + +from collections.abc import Mapping +from dataclasses import dataclass +from datetime import UTC, datetime + +from arbitrade.detection.graph import TriangularCycle +from arbitrade.exchange.models import BookLevel +from arbitrade.market_data.order_book import OrderBook + + +def _normalize_pair_symbol(symbol: str) -> str: + if "/" not in symbol: + return symbol.upper() + + base, quote = symbol.split("/", 1) + return f"{base.upper()}/{quote.upper()}" + + +@dataclass(frozen=True, slots=True) +class CycleScore: + cycle: TriangularCycle + gross_rate: float + net_rate: float + min_profit_threshold: float + updated_pair: str + scored_at: datetime + + @property + def is_profitable(self) -> bool: + return (self.net_rate - 1.0) >= self.min_profit_threshold + + +@dataclass(frozen=True, slots=True) +class OpportunityEvent: + detected_at: datetime + cycle: str + updated_pair: str + gross_rate: float + net_rate: float + gross_pct: float + net_pct: float + est_profit: float + allocated_capital: float = 1.0 + + @classmethod + def from_cycle_score(cls, score: CycleScore, base_capital: float = 1.0) -> OpportunityEvent: + gross_pct = (score.gross_rate - 1.0) * 100.0 + net_pct = (score.net_rate - 1.0) * 100.0 + est_profit = (score.net_rate - 1.0) * base_capital + a, b, c = score.cycle.currencies + cycle = f"{a}->{b}->{c}->{a}" + return cls( + detected_at=score.scored_at, + cycle=cycle, + updated_pair=score.updated_pair, + gross_rate=score.gross_rate, + net_rate=score.net_rate, + gross_pct=gross_pct, + net_pct=net_pct, + est_profit=est_profit, + allocated_capital=base_capital, + ) + + +class IncrementalCycleDetector: + def __init__( + self, + cycles_by_pair: Mapping[str, list[TriangularCycle]], + *, + fee_rate: float = 0.0, + max_depth_levels: int = 10, + min_profit_threshold: float = 0.0, + min_order_size_by_pair: Mapping[str, float] | None = None, + max_book_age_seconds: float | None = None, + ) -> None: + self._cycles_by_pair = { + _normalize_pair_symbol(pair): list(cycles) for pair, cycles in cycles_by_pair.items() + } + self._fee_multiplier = 1.0 - fee_rate + self._max_depth_levels = max_depth_levels + self._min_profit_threshold = min_profit_threshold + self._max_book_age_seconds = max_book_age_seconds + self._min_order_size_by_pair = { + _normalize_pair_symbol(pair): float(min_size) + for pair, min_size in (min_order_size_by_pair or {}).items() + } + + if self._fee_multiplier < 0.0: + raise ValueError("fee_rate must be <= 1.0") + if self._max_depth_levels <= 0: + raise ValueError("max_depth_levels must be > 0") + if self._min_profit_threshold < 0.0: + raise ValueError("min_profit_threshold must be >= 0.0") + if self._max_book_age_seconds is not None and self._max_book_age_seconds <= 0.0: + raise ValueError("max_book_age_seconds must be > 0.0") + for min_size in self._min_order_size_by_pair.values(): + if min_size <= 0.0: + raise ValueError("minimum order size must be > 0.0") + + def score_updated_pair( + self, + updated_pair: str, + books: Mapping[str, OrderBook], + ) -> list[CycleScore]: + normalized_pair = _normalize_pair_symbol(updated_pair) + impacted_cycles = self._cycles_by_pair.get(normalized_pair, []) + + normalized_books = {_normalize_pair_symbol(symbol): book for symbol, book in books.items()} + + scores: list[CycleScore] = [] + scored_at = datetime.now(UTC) + for cycle in impacted_cycles: + rates = self._score_cycle(cycle, normalized_books, scored_at) + if rates is None: + continue + gross_rate, net_rate = rates + if (net_rate - 1.0) < self._min_profit_threshold: + continue + scores.append( + CycleScore( + cycle=cycle, + gross_rate=gross_rate, + net_rate=net_rate, + min_profit_threshold=self._min_profit_threshold, + updated_pair=normalized_pair, + scored_at=scored_at, + ) + ) + + return scores + + def opportunities_for_updated_pair( + self, + updated_pair: str, + books: Mapping[str, OrderBook], + *, + base_capital: float = 1.0, + ) -> list[OpportunityEvent]: + if base_capital <= 0.0: + raise ValueError("base_capital must be > 0.0") + + scores = self.score_updated_pair(updated_pair, books) + return [OpportunityEvent.from_cycle_score(score, base_capital) for score in scores] + + def _score_cycle( + self, + cycle: TriangularCycle, + books: Mapping[str, OrderBook], + scored_at: datetime, + ) -> tuple[float, float] | None: + if not self._is_cycle_fresh(cycle, books, scored_at): + return None + + a, b, c = cycle.currencies + gross_amount = 1.0 + + gross_ab = self._convert(gross_amount, a, b, cycle, books) + if gross_ab is None: + return None + net_ab = gross_ab * self._fee_multiplier + + gross_bc = self._convert(gross_ab, b, c, cycle, books) + if gross_bc is None: + return None + net_bc = self._convert(net_ab, b, c, cycle, books) + if net_bc is None: + return None + net_bc *= self._fee_multiplier + + gross_ca = self._convert(gross_bc, c, a, cycle, books) + if gross_ca is None: + return None + net_ca = self._convert(net_bc, c, a, cycle, books) + if net_ca is None: + return None + net_ca *= self._fee_multiplier + + return gross_ca, net_ca + + def _is_cycle_fresh( + self, + cycle: TriangularCycle, + books: Mapping[str, OrderBook], + scored_at: datetime, + ) -> bool: + if self._max_book_age_seconds is None: + return True + + for pair in cycle.pairs: + normalized_pair = _normalize_pair_symbol(pair) + book = books.get(normalized_pair) + if book is None: + return False + + age_seconds = (scored_at - book.updated_at).total_seconds() + if age_seconds > self._max_book_age_seconds: + return False + + return True + + @staticmethod + def _pair_for_edge(cycle: TriangularCycle, from_currency: str, to_currency: str) -> str | None: + for pair in cycle.pairs: + if "/" not in pair: + continue + base, quote = pair.split("/", 1) + base = base.upper() + quote = quote.upper() + if {base, quote} == {from_currency, to_currency}: + return f"{base}/{quote}" + return None + + def _convert( + self, + amount: float, + from_currency: str, + to_currency: str, + cycle: TriangularCycle, + books: Mapping[str, OrderBook], + ) -> float | None: + pair = self._pair_for_edge(cycle, from_currency, to_currency) + if pair is None: + return None + + book = books.get(pair) + if book is None: + return None + + bids, asks = book.top_levels(depth=self._max_depth_levels) + + base, quote = pair.split("/", 1) + base = base.upper() + quote = quote.upper() + + if from_currency == base and to_currency == quote: + quote_out = self._sell_base_for_quote(amount, bids) + if quote_out is None: + return None + if not self._is_min_order_size_satisfied(pair, amount): + return None + return quote_out + + if from_currency == quote and to_currency == base: + base_out = self._buy_base_with_quote(amount, asks) + if base_out is None: + return None + if not self._is_min_order_size_satisfied(pair, base_out): + return None + return base_out + + return None + + def _is_min_order_size_satisfied(self, pair: str, base_amount: float) -> bool: + min_size = self._min_order_size_by_pair.get(pair) + if min_size is None: + return True + return base_amount >= min_size + + @staticmethod + def _sell_base_for_quote(amount_base: float, bids: list[BookLevel]) -> float | None: + remaining = amount_base + quote_out = 0.0 + for level in bids: + if remaining <= 0.0: + break + if level.price <= 0.0 or level.volume <= 0.0: + continue + + executed = min(remaining, level.volume) + quote_out += executed * level.price + remaining -= executed + + if remaining > 0.0: + return None + return quote_out + + @staticmethod + def _buy_base_with_quote(amount_quote: float, asks: list[BookLevel]) -> float | None: + remaining_quote = amount_quote + base_out = 0.0 + for level in asks: + if remaining_quote <= 0.0: + break + if level.price <= 0.0 or level.volume <= 0.0: + continue + + level_quote_capacity = level.volume * level.price + spend = min(remaining_quote, level_quote_capacity) + base_out += spend / level.price + remaining_quote -= spend + + if remaining_quote > 0.0: + return None + return base_out diff --git a/build/lib/arbitrade/detection/graph.py b/build/lib/arbitrade/detection/graph.py new file mode 100644 index 0000000..47a1286 --- /dev/null +++ b/build/lib/arbitrade/detection/graph.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True, slots=True) +class TriangularCycle: + currencies: tuple[str, str, str] + pairs: tuple[str, str, str] + + +def _canonical_pair(base: str, quote: str) -> str: + return f"{base}/{quote}" + + +class CurrencyGraph: + def __init__(self) -> None: + self._adjacency: dict[str, set[str]] = {} + self._pair_by_direction: dict[tuple[str, str], str] = {} + + @property + def adjacency(self) -> dict[str, set[str]]: + return self._adjacency + + @property + def pair_by_direction(self) -> dict[tuple[str, str], str]: + return self._pair_by_direction + + def add_pair(self, base: str, quote: str, pair_symbol: str | None = None) -> None: + normalized_base = base.upper() + normalized_quote = quote.upper() + symbol = pair_symbol or _canonical_pair(normalized_base, normalized_quote) + + self._adjacency.setdefault(normalized_base, set()).add(normalized_quote) + self._adjacency.setdefault(normalized_quote, set()).add(normalized_base) + + self._pair_by_direction[(normalized_base, normalized_quote)] = symbol + self._pair_by_direction[(normalized_quote, normalized_base)] = symbol + + @classmethod + def from_kraken_asset_pairs(cls, asset_pairs: dict[str, Any]) -> CurrencyGraph: + graph = cls() + for value in asset_pairs.values(): + if not isinstance(value, dict): + continue + + wsname = value.get("wsname") + if isinstance(wsname, str) and "/" in wsname: + base, quote = wsname.split("/", 1) + graph.add_pair(base, quote, wsname) + continue + + raw_base = value.get("base") + raw_quote = value.get("quote") + if isinstance(raw_base, str) and isinstance(raw_quote, str): + graph.add_pair(raw_base, raw_quote) + + return graph + + def triangular_cycles(self) -> list[TriangularCycle]: + found: dict[tuple[str, str, str], TriangularCycle] = {} + + for a, neighbors_a in self._adjacency.items(): + for b in neighbors_a: + if a >= b: + continue + neighbors_b = self._adjacency.get(b, set()) + for c in neighbors_b: + if b >= c: + continue + if a not in self._adjacency.get(c, set()): + continue + + p_ab = self._pair_by_direction[(a, b)] + p_bc = self._pair_by_direction[(b, c)] + p_ca = self._pair_by_direction[(c, a)] + + key = (a, b, c) + found[key] = TriangularCycle(currencies=key, pairs=(p_ab, p_bc, p_ca)) + + return list(found.values()) + + @staticmethod + def index_cycles_by_pair(cycles: list[TriangularCycle]) -> dict[str, list[TriangularCycle]]: + index: dict[str, list[TriangularCycle]] = {} + for cycle in cycles: + for pair in cycle.pairs: + index.setdefault(pair, []).append(cycle) + return index diff --git a/build/lib/arbitrade/exchange/__init__.py b/build/lib/arbitrade/exchange/__init__.py new file mode 100644 index 0000000..26b16d5 --- /dev/null +++ b/build/lib/arbitrade/exchange/__init__.py @@ -0,0 +1 @@ +"""Kraken exchange integration package.""" diff --git a/build/lib/arbitrade/exchange/kraken_rest.py b/build/lib/arbitrade/exchange/kraken_rest.py new file mode 100644 index 0000000..b4d5816 --- /dev/null +++ b/build/lib/arbitrade/exchange/kraken_rest.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +import asyncio +import time +from typing import Any +from urllib.parse import urlencode + +import httpx +import structlog + +from arbitrade.config.settings import Settings +from arbitrade.exchange.models import KrakenApiResult, LatencySample +from arbitrade.exchange.signing import sign_kraken_private_path + +_LOG = structlog.get_logger(__name__) + + +def _result_dict(payload: dict[str, Any]) -> dict[str, Any]: + result = payload.get("result", {}) + if isinstance(result, dict): + return result + return {} + + +class KrakenRestClient: + def __init__(self, settings: Settings) -> None: + self._settings = settings + self._client = httpx.AsyncClient( + base_url=settings.kraken_rest_url, + timeout=settings.kraken_http_timeout_seconds, + limits=httpx.Limits(max_keepalive_connections=10, max_connections=50), + headers={"User-Agent": "arbitrade/0.1.0"}, + ) + self._private_lock = asyncio.Lock() + + issues = self.validate_compliance() + if issues: + _LOG.warning("kraken_compliance_issues", issues=issues) + else: + _LOG.info("kraken_compliance_ok") + + def validate_compliance(self) -> list[str]: + issues: list[str] = [] + + if not self._settings.kraken_rest_url.startswith("https://"): + issues.append("KRAKEN_REST_URL should use https://") + + if self._settings.kraken_private_rate_limit_seconds < 1.0: + issues.append("KRAKEN_PRIVATE_RATE_LIMIT_SECONDS below 1.0 may violate Kraken limits") + + if self._settings.kraken_retry_attempts < 1: + issues.append("KRAKEN_RETRY_ATTEMPTS must be >= 1") + + if self._settings.kraken_retry_base_delay_seconds < 0: + issues.append("KRAKEN_RETRY_BASE_DELAY_SECONDS must be >= 0") + + return issues + + async def close(self) -> None: + await self._client.aclose() + + async def warm_connection_pool(self) -> None: + await self.server_time() + + async def _request_with_retry( + self, + endpoint: str, + params: dict[str, Any] | None = None, + ) -> KrakenApiResult: + attempts = self._settings.kraken_retry_attempts + delay = self._settings.kraken_retry_base_delay_seconds + params = params or {} + + for attempt in range(1, attempts + 1): + t0 = time.perf_counter() + try: + response = await self._client.get(endpoint, params=params) + response.raise_for_status() + payload = response.json() + if payload.get("error"): + raise RuntimeError(f"Kraken error: {payload['error']}") + + latency = (time.perf_counter() - t0) * 1000 + _LOG.info( + "kraken_rest_request_ok", + endpoint=endpoint, + attempt=attempt, + latency_ms=latency, + sample=LatencySample.now("rest_request", latency_ms=latency).latency_ms, + ) + return KrakenApiResult(endpoint=endpoint, payload=payload) + except Exception as exc: + latency = (time.perf_counter() - t0) * 1000 + _LOG.warning( + "kraken_rest_request_failed", + endpoint=endpoint, + attempt=attempt, + latency_ms=latency, + error=str(exc), + ) + if attempt >= attempts: + raise + await asyncio.sleep(delay * (2 ** (attempt - 1))) + + raise RuntimeError("unreachable retry loop") + + async def _private_post_with_retry( + self, + endpoint: str, + data: dict[str, str] | None = None, + ) -> KrakenApiResult: + api_key = self._settings.kraken_api_key + api_secret = self._settings.kraken_api_secret + if not api_key or not api_secret: + raise RuntimeError("Missing Kraken API credentials for private endpoint") + + attempts = self._settings.kraken_retry_attempts + delay = self._settings.kraken_retry_base_delay_seconds + + for attempt in range(1, attempts + 1): + t0 = time.perf_counter() + try: + nonce = str(int(time.time() * 1000)) + payload = {"nonce": nonce} + if data is not None: + payload.update(data) + + encoded = urlencode(payload) + signature = sign_kraken_private_path(endpoint, nonce, encoded, api_secret) + + response = await self._client.post( + endpoint, + data=payload, + headers={ + "API-Key": api_key, + "API-Sign": signature, + }, + ) + response.raise_for_status() + body = response.json() + if body.get("error"): + raise RuntimeError(f"Kraken error: {body['error']}") + + latency = (time.perf_counter() - t0) * 1000 + _LOG.info( + "kraken_private_rest_request_ok", + endpoint=endpoint, + attempt=attempt, + latency_ms=latency, + sample=LatencySample.now("private_rest_request", latency_ms=latency).latency_ms, + ) + return KrakenApiResult(endpoint=endpoint, payload=body) + except Exception as exc: + latency = (time.perf_counter() - t0) * 1000 + _LOG.warning( + "kraken_private_rest_request_failed", + endpoint=endpoint, + attempt=attempt, + latency_ms=latency, + error=str(exc), + ) + if attempt >= attempts: + raise + await asyncio.sleep(delay * (2 ** (attempt - 1))) + + raise RuntimeError("unreachable retry loop") + + async def server_time(self) -> dict[str, Any]: + result = await self._request_with_retry("/0/public/Time") + return _result_dict(result.payload) + + async def assets(self) -> dict[str, Any]: + result = await self._request_with_retry("/0/public/Assets") + return _result_dict(result.payload) + + async def asset_pairs(self) -> dict[str, Any]: + result = await self._request_with_retry("/0/public/AssetPairs") + return _result_dict(result.payload) + + async def _throttled_private_call( + self, + endpoint: str, + data: dict[str, str] | None = None, + ) -> dict[str, Any]: + async with self._private_lock: + result = await self._private_post_with_retry(endpoint, data=data) + await asyncio.sleep(self._settings.kraken_private_rate_limit_seconds) + return _result_dict(result.payload) + + async def balances(self) -> dict[str, Any]: + return await self._throttled_private_call("/0/private/Balance") + + async def place_market_order( + self, + *, + pair: str, + side: str, + volume: float, + user_ref: int | None = None, + ) -> dict[str, Any]: + normalized_side = side.lower() + if normalized_side not in {"buy", "sell"}: + raise ValueError("side must be 'buy' or 'sell'") + if volume <= 0.0: + raise ValueError("volume must be > 0.0") + if user_ref is not None and user_ref < 0: + raise ValueError("user_ref must be >= 0") + + data = { + "pair": pair, + "type": normalized_side, + "ordertype": "market", + "volume": str(volume), + } + if user_ref is not None: + data["userref"] = str(user_ref) + + return await self._throttled_private_call( + "/0/private/AddOrder", + data=data, + ) + + async def place_limit_order( + self, + *, + pair: str, + side: str, + volume: float, + price: float, + user_ref: int | None = None, + ) -> dict[str, Any]: + normalized_side = side.lower() + if normalized_side not in {"buy", "sell"}: + raise ValueError("side must be 'buy' or 'sell'") + if volume <= 0.0: + raise ValueError("volume must be > 0.0") + if price <= 0.0: + raise ValueError("price must be > 0.0") + if user_ref is not None and user_ref < 0: + raise ValueError("user_ref must be >= 0") + + data = { + "pair": pair, + "type": normalized_side, + "ordertype": "limit", + "price": str(price), + "volume": str(volume), + } + if user_ref is not None: + data["userref"] = str(user_ref) + + return await self._throttled_private_call( + "/0/private/AddOrder", + data=data, + ) + + async def query_order( + self, + *, + order_id: str, + include_trades: bool = True, + ) -> dict[str, Any]: + if not order_id.strip(): + raise ValueError("order_id must be non-empty") + + return await self._throttled_private_call( + "/0/private/QueryOrders", + data={ + "txid": order_id, + "trades": "true" if include_trades else "false", + }, + ) + + async def cancel_order(self, *, order_id: str) -> dict[str, Any]: + if not order_id.strip(): + raise ValueError("order_id must be non-empty") + + return await self._throttled_private_call( + "/0/private/CancelOrder", + data={"txid": order_id}, + ) diff --git a/build/lib/arbitrade/exchange/kraken_ws.py b/build/lib/arbitrade/exchange/kraken_ws.py new file mode 100644 index 0000000..e962228 --- /dev/null +++ b/build/lib/arbitrade/exchange/kraken_ws.py @@ -0,0 +1,177 @@ +from __future__ import annotations + +import asyncio +import time +from collections.abc import AsyncIterator +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + +import orjson +import structlog +import websockets + +from arbitrade.alerting.notifier import AlertSeverity, SupportsAlerts +from arbitrade.config.settings import Settings +from arbitrade.exchange.models import BookDelta, BookLevel + +_LOG = structlog.get_logger(__name__) + + +@dataclass(slots=True) +class WsMessage: + received_at: datetime + payload: dict[str, Any] + + +class KrakenWsClient: + def __init__(self, settings: Settings, *, alert_notifier: SupportsAlerts | None = None) -> None: + self._settings = settings + self._last_message_at: datetime | None = None + self._stop = asyncio.Event() + self._alert_notifier = alert_notifier + self._has_connected_once = False + self._was_disconnected = False + + @property + def is_stale(self) -> bool: + if self._last_message_at is None: + return True + return ( + datetime.now(UTC) - self._last_message_at + ).total_seconds() > self._settings.ws_max_staleness_seconds + + async def stop(self) -> None: + self._stop.set() + + async def connect_stream(self) -> AsyncIterator[WsMessage]: + delay = 1.0 + while not self._stop.is_set(): + try: + async with websockets.connect( + self._settings.kraken_ws_url, max_size=2_000_000 + ) as ws: + _LOG.info("kraken_ws_connected", url=self._settings.kraken_ws_url) + if self._has_connected_once and self._was_disconnected: + await self._notify( + category="system", + severity="info", + title="WebSocket reconnected", + message="Kraken WebSocket connection restored.", + details={"url": self._settings.kraken_ws_url}, + ) + self._has_connected_once = True + self._was_disconnected = False + delay = 1.0 + async for raw in self._recv_loop(ws): + yield raw + except Exception as exc: + _LOG.warning("kraken_ws_disconnected", error=str(exc), reconnect_in=delay) + self._was_disconnected = True + await self._notify( + category="system", + severity="warning", + title="WebSocket disconnected", + message="Kraken WebSocket disconnected, reconnect scheduled.", + details={ + "error": str(exc), + "reconnect_in_seconds": f"{delay}", + }, + ) + await asyncio.sleep(delay) + delay = min(delay * 2, 30.0) + + async def _recv_loop(self, ws: Any) -> AsyncIterator[WsMessage]: + while not self._stop.is_set(): + t0 = time.perf_counter() + try: + raw = await asyncio.wait_for( + ws.recv(), timeout=self._settings.ws_heartbeat_timeout_seconds + ) + except TimeoutError: + await self._notify( + category="system", + severity="critical", + title="WebSocket staleness abort", + message="No WebSocket heartbeat within configured timeout; reconnecting.", + details={ + "heartbeat_timeout_seconds": ( + f"{self._settings.ws_heartbeat_timeout_seconds}" + ), + }, + ) + raise + parse_start = time.perf_counter() + payload = orjson.loads(raw) + self._last_message_at = datetime.now(UTC) + + _LOG.debug( + "kraken_ws_message", + recv_latency_ms=(parse_start - t0) * 1000, + parse_latency_ms=(time.perf_counter() - parse_start) * 1000, + ) + if isinstance(payload, dict): + yield WsMessage(received_at=self._last_message_at, payload=payload) + + async def _notify( + self, + *, + category: str, + severity: AlertSeverity, + title: str, + message: str, + details: dict[str, str] | None = None, + ) -> None: + if self._alert_notifier is None: + return + await self._alert_notifier.notify( + category=category, + severity=severity, + title=title, + message=message, + details=details, + ) + + @staticmethod + def parse_book_delta(message: dict[str, Any]) -> BookDelta | None: + # Kraken v2 book update shape can vary by channel; keep parser defensive. + channel = str(message.get("channel", "")) + if "book" not in channel: + return None + + symbol = str(message.get("symbol", "")) + data = message.get("data") + if not isinstance(data, list) or not data: + return None + + first = data[0] + if not isinstance(first, dict): + return None + + bids = [ + BookLevel(price=float(level["price"]), volume=float(level["qty"])) + for level in first.get("bids", []) + if isinstance(level, dict) and "price" in level and "qty" in level + ] + asks = [ + BookLevel(price=float(level["price"]), volume=float(level["qty"])) + for level in first.get("asks", []) + if isinstance(level, dict) and "price" in level and "qty" in level + ] + + checksum: int | None = None + raw_checksum = first.get("checksum") + if isinstance(raw_checksum, int): + checksum = raw_checksum + + source_timestamp_ms: int | None = None + if isinstance(first.get("timestamp"), int): + source_timestamp_ms = first["timestamp"] + + return BookDelta( + symbol=symbol, + bids=bids, + asks=asks, + checksum=checksum, + source_timestamp_ms=source_timestamp_ms, + ) diff --git a/build/lib/arbitrade/exchange/models.py b/build/lib/arbitrade/exchange/models.py new file mode 100644 index 0000000..27b414d --- /dev/null +++ b/build/lib/arbitrade/exchange/models.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any + + +@dataclass(slots=True) +class KrakenApiResult: + endpoint: str + payload: dict[str, Any] + + +@dataclass(slots=True) +class LatencySample: + stage: str + at: datetime + latency_ms: float + + @classmethod + def now(cls, stage: str, latency_ms: float) -> LatencySample: + return cls(stage=stage, at=datetime.now(UTC), latency_ms=latency_ms) + + +@dataclass(slots=True) +class BookLevel: + price: float + volume: float + + +@dataclass(slots=True) +class BookDelta: + symbol: str + bids: list[BookLevel] + asks: list[BookLevel] + checksum: int | None = None + source_timestamp_ms: int | None = None diff --git a/build/lib/arbitrade/exchange/signing.py b/build/lib/arbitrade/exchange/signing.py new file mode 100644 index 0000000..64648d9 --- /dev/null +++ b/build/lib/arbitrade/exchange/signing.py @@ -0,0 +1,14 @@ +from __future__ import annotations + +import base64 +import hashlib +import hmac +from functools import lru_cache + + +@lru_cache(maxsize=2048) +def sign_kraken_private_path(path: str, nonce: str, post_data: str, api_secret: str) -> str: + message = nonce.encode("utf-8") + post_data.encode("utf-8") + sha256 = hashlib.sha256(message).digest() + mac = hmac.new(base64.b64decode(api_secret), path.encode("utf-8") + sha256, hashlib.sha512) + return base64.b64encode(mac.digest()).decode("utf-8") diff --git a/build/lib/arbitrade/execution/__init__.py b/build/lib/arbitrade/execution/__init__.py new file mode 100644 index 0000000..1fdb5e3 --- /dev/null +++ b/build/lib/arbitrade/execution/__init__.py @@ -0,0 +1,32 @@ +"""Trade execution helpers.""" + +from arbitrade.execution.fill_monitor import ( + FillMonitor, + FillMonitorResult, + OrderFillState, +) +from arbitrade.execution.idempotency import ( + IdempotencyKeyFactory, + OrderReconciler, + ReconciliationReport, +) +from arbitrade.execution.recovery import PartialFillRecovery, RecoveryAction +from arbitrade.execution.sequencer import ( + ExecutionLeg, + TriangularExecutionResult, + TriangularExecutionSequencer, +) + +__all__ = [ + "ExecutionLeg", + "OrderFillState", + "FillMonitorResult", + "FillMonitor", + "IdempotencyKeyFactory", + "ReconciliationReport", + "OrderReconciler", + "RecoveryAction", + "PartialFillRecovery", + "TriangularExecutionResult", + "TriangularExecutionSequencer", +] diff --git a/build/lib/arbitrade/execution/fill_monitor.py b/build/lib/arbitrade/execution/fill_monitor.py new file mode 100644 index 0000000..306530e --- /dev/null +++ b/build/lib/arbitrade/execution/fill_monitor.py @@ -0,0 +1,133 @@ +from __future__ import annotations + +import asyncio +import time +from collections.abc import Callable +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, Protocol + + +class SupportsOrderStatusPolling(Protocol): + async def query_order( + self, *, order_id: str, include_trades: bool = True + ) -> dict[str, Any]: ... + + +@dataclass(frozen=True, slots=True) +class OrderFillState: + order_id: str + status: str + filled_volume: float | None + avg_price: float | None + updated_at: datetime + source: str + + @property + def is_terminal(self) -> bool: + return self.status in {"closed", "canceled", "expired"} + + +@dataclass(frozen=True, slots=True) +class FillMonitorResult: + order_id: str + timed_out: bool + terminal_state: OrderFillState | None + last_state: OrderFillState | None + elapsed_seconds: float + + +class FillMonitor: + def __init__( + self, + poll_client: SupportsOrderStatusPolling, + *, + poll_interval_seconds: float = 0.5, + max_wait_seconds: float = 10.0, + ws_status_provider: Callable[[str], OrderFillState | None] | None = None, + ) -> None: + if poll_interval_seconds <= 0.0: + raise ValueError("poll_interval_seconds must be > 0.0") + if max_wait_seconds <= 0.0: + raise ValueError("max_wait_seconds must be > 0.0") + + self._poll_client = poll_client + self._poll_interval_seconds = poll_interval_seconds + self._max_wait_seconds = max_wait_seconds + self._ws_status_provider = ws_status_provider + + @staticmethod + def _to_float(value: Any) -> float | None: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + @classmethod + def _state_from_payload( + cls, order_id: str, payload: dict[str, Any], *, source: str + ) -> OrderFillState: + status = str(payload.get("status", "unknown")).lower() + return OrderFillState( + order_id=order_id, + status=status, + filled_volume=cls._to_float(payload.get("vol_exec")), + avg_price=cls._to_float(payload.get("price") or payload.get("avg_price")), + updated_at=datetime.now(UTC), + source=source, + ) + + @classmethod + def _extract_order_payload(cls, order_id: str, response: dict[str, Any]) -> dict[str, Any]: + if order_id in response and isinstance(response[order_id], dict): + payload = response[order_id] + return {str(key): value for key, value in payload.items()} + return response + + async def wait_for_terminal_fill(self, order_id: str) -> FillMonitorResult: + if not order_id.strip(): + raise ValueError("order_id must be non-empty") + + started = time.monotonic() + last_state: OrderFillState | None = None + + while True: + elapsed = time.monotonic() - started + if elapsed >= self._max_wait_seconds: + return FillMonitorResult( + order_id=order_id, + timed_out=True, + terminal_state=None, + last_state=last_state, + elapsed_seconds=elapsed, + ) + + if self._ws_status_provider is not None: + ws_state = self._ws_status_provider(order_id) + if ws_state is not None: + last_state = ws_state + if ws_state.is_terminal: + return FillMonitorResult( + order_id=order_id, + timed_out=False, + terminal_state=ws_state, + last_state=ws_state, + elapsed_seconds=elapsed, + ) + + response = await self._poll_client.query_order(order_id=order_id, include_trades=True) + payload = self._extract_order_payload(order_id, response) + polled_state = self._state_from_payload(order_id, payload, source="rest_poll") + last_state = polled_state + if polled_state.is_terminal: + return FillMonitorResult( + order_id=order_id, + timed_out=False, + terminal_state=polled_state, + last_state=polled_state, + elapsed_seconds=time.monotonic() - started, + ) + + await asyncio.sleep(self._poll_interval_seconds) diff --git a/build/lib/arbitrade/execution/idempotency.py b/build/lib/arbitrade/execution/idempotency.py new file mode 100644 index 0000000..8864e2c --- /dev/null +++ b/build/lib/arbitrade/execution/idempotency.py @@ -0,0 +1,105 @@ +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from typing import Any, Protocol + +from arbitrade.detection.engine import OpportunityEvent +from arbitrade.execution.sequencer import ExecutionLeg + + +class SupportsOrderHistoryLookup(Protocol): + async def query_order( + self, *, order_id: str, include_trades: bool = True + ) -> dict[str, Any]: ... + + +@dataclass(frozen=True, slots=True) +class ReconciliationReport: + order_id: str + user_ref: int + status: str + filled_volume: float | None + avg_price: float | None + is_terminal: bool + matches_request: bool + raw_payload: dict[str, Any] + + +class IdempotencyKeyFactory: + def user_ref_for_leg(self, event: OpportunityEvent, leg: ExecutionLeg, leg_index: int) -> int: + material = "|".join( + [ + event.cycle, + event.updated_pair, + leg.from_currency, + leg.to_currency, + leg.pair, + leg.side, + f"{leg.volume:.12f}", + str(leg_index), + ] + ).encode("utf-8") + digest = hashlib.sha256(material).digest() + value = int.from_bytes(digest[:8], "big") % 2_147_483_647 + return value or 1 + + +class OrderReconciler: + def __init__(self, history_client: SupportsOrderHistoryLookup) -> None: + self._history_client = history_client + + @staticmethod + def _to_float(value: Any) -> float | None: + if value is None: + return None + try: + return float(value) + except (TypeError, ValueError): + return None + + @staticmethod + def _extract_payload(order_id: str, response: dict[str, Any]) -> dict[str, Any]: + if order_id in response and isinstance(response[order_id], dict): + payload = response[order_id] + return {str(key): value for key, value in payload.items()} + return response + + async def reconcile_order( + self, + *, + order_id: str, + user_ref: int, + expected_pair: str, + expected_side: str, + expected_volume: float, + ) -> ReconciliationReport: + if not order_id.strip(): + raise ValueError("order_id must be non-empty") + + response = await self._history_client.query_order(order_id=order_id, include_trades=True) + payload = self._extract_payload(order_id, response) + status = str(payload.get("status", "unknown")).lower() + filled_volume = self._to_float(payload.get("vol_exec")) + avg_price = self._to_float(payload.get("price") or payload.get("avg_price")) + reported_pair = str(payload.get("pair", expected_pair)) + reported_side = str(payload.get("type", expected_side)).lower() + matches_request = ( + reported_pair == expected_pair + and reported_side == expected_side.lower() + and ( + expected_volume <= 0.0 or filled_volume is None or filled_volume <= expected_volume + ) + and payload.get("userref") in {None, str(user_ref), user_ref} + ) + + return ReconciliationReport( + order_id=order_id, + user_ref=user_ref, + status=status, + filled_volume=filled_volume, + avg_price=avg_price, + is_terminal=status in {"closed", "canceled", "expired"}, + matches_request=matches_request, + raw_payload=payload, + ) diff --git a/build/lib/arbitrade/execution/recovery.py b/build/lib/arbitrade/execution/recovery.py new file mode 100644 index 0000000..b62c45b --- /dev/null +++ b/build/lib/arbitrade/execution/recovery.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any, Protocol + +from arbitrade.execution.fill_monitor import FillMonitorResult, OrderFillState + + +class SupportsOrderLifecycle(Protocol): + async def cancel_order(self, *, order_id: str) -> dict[str, Any]: ... + + async def place_market_order( + self, *, pair: str, side: str, volume: float + ) -> dict[str, Any]: ... + + +@dataclass(frozen=True, slots=True) +class RecoveryAction: + order_id: str + canceled: bool + hedged: bool + hedge_pair: str | None = None + hedge_side: str | None = None + hedge_volume: float | None = None + cancel_response: dict[str, Any] | None = None + hedge_response: dict[str, Any] | None = None + reason: str | None = None + + +class PartialFillRecovery: + def __init__(self, rest_client: SupportsOrderLifecycle) -> None: + self._rest_client = rest_client + + @staticmethod + def _counter_side(side: str) -> str: + normalized = side.lower() + if normalized == "buy": + return "sell" + if normalized == "sell": + return "buy" + raise ValueError("side must be 'buy' or 'sell'") + + @staticmethod + def _residual_volume(terminal_state: OrderFillState | None, requested_volume: float) -> float: + if requested_volume <= 0.0: + raise ValueError("requested_volume must be > 0.0") + if terminal_state is None or terminal_state.filled_volume is None: + return requested_volume + residual = requested_volume - terminal_state.filled_volume + return residual if residual > 0.0 else 0.0 + + async def recover_partial_fill( + self, + *, + order_id: str, + pair: str, + side: str, + requested_volume: float, + fill_result: FillMonitorResult, + ) -> RecoveryAction: + if not order_id.strip(): + raise ValueError("order_id must be non-empty") + + cancel_response: dict[str, Any] | None = None + hedge_response: dict[str, Any] | None = None + hedged = False + canceled = False + reason = None + + state = fill_result.terminal_state or fill_result.last_state + residual_volume = self._residual_volume(state, requested_volume) + + if state is not None and state.status in {"open", "partial"}: + cancel_response = await self._rest_client.cancel_order(order_id=order_id) + canceled = True + reason = f"canceled_{state.status}_order" + + if residual_volume > 0.0 and fill_result.timed_out: + hedge_response = await self._rest_client.place_market_order( + pair=pair, + side=self._counter_side(side), + volume=residual_volume, + ) + hedged = True + if reason is None: + reason = "hedged_timed_out_order" + + return RecoveryAction( + order_id=order_id, + canceled=canceled, + hedged=hedged, + hedge_pair=pair if hedged else None, + hedge_side=self._counter_side(side) if hedged else None, + hedge_volume=residual_volume if hedged else None, + cancel_response=cancel_response, + hedge_response=hedge_response, + reason=reason, + ) diff --git a/build/lib/arbitrade/execution/sequencer.py b/build/lib/arbitrade/execution/sequencer.py new file mode 100644 index 0000000..35f7236 --- /dev/null +++ b/build/lib/arbitrade/execution/sequencer.py @@ -0,0 +1,288 @@ +from __future__ import annotations + +from collections.abc import Callable, Sequence +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, Protocol + +from arbitrade.alerting.notifier import SupportsAlerts +from arbitrade.detection.engine import OpportunityEvent +from arbitrade.storage.executions import AsyncExecutionWriter +from arbitrade.storage.repositories import ( + AuditRecord, + AuditRepository, + OrderRecord, + PnLRecord, + TradeRecord, +) + + +class SupportsOrderPlacement(Protocol): + async def place_market_order( + self, *, pair: str, side: str, volume: float + ) -> dict[str, Any]: ... + + +@dataclass(frozen=True, slots=True) +class ExecutionLeg: + from_currency: str + to_currency: str + pair: str + side: str + volume: float + + +@dataclass(frozen=True, slots=True) +class TriangularExecutionResult: + success: bool + requested_legs: tuple[ExecutionLeg, ...] + completed_legs: int + responses: tuple[dict[str, Any], ...] + failure_reason: str | None = None + + +class TriangularExecutionSequencer: + def __init__( + self, + rest_client: SupportsOrderPlacement, + *, + available_pairs: Sequence[str], + volume_for_leg: Callable[[OpportunityEvent, ExecutionLeg, int], float] | None = None, + execution_writer: AsyncExecutionWriter | None = None, + alert_notifier: SupportsAlerts | None = None, + audit_repository: AuditRepository | None = None, + ) -> None: + self._rest_client = rest_client + self._available_pairs = {self._normalize_pair(pair) for pair in available_pairs} + self._volume_for_leg = volume_for_leg or self._default_volume_for_leg + self._execution_writer = execution_writer + self._alert_notifier = alert_notifier + self._audit_repository = audit_repository + + @staticmethod + def _normalize_pair(pair: str) -> str: + normalized = pair.strip().upper().replace("-", "/") + if "/" not in normalized: + return normalized + base, quote = normalized.split("/", 1) + return f"{base}/{quote}" + + @staticmethod + def _default_volume_for_leg(event: OpportunityEvent, _leg: ExecutionLeg, _idx: int) -> float: + if event.allocated_capital <= 0.0: + raise ValueError("allocated_capital must be > 0.0") + return event.allocated_capital + + def _resolve_leg(self, from_currency: str, to_currency: str, volume: float) -> ExecutionLeg: + from_cur = from_currency.upper() + to_cur = to_currency.upper() + + buy_pair = f"{to_cur}/{from_cur}" + if buy_pair in self._available_pairs: + return ExecutionLeg( + from_currency=from_cur, + to_currency=to_cur, + pair=buy_pair, + side="buy", + volume=volume, + ) + + sell_pair = f"{from_cur}/{to_cur}" + if sell_pair in self._available_pairs: + return ExecutionLeg( + from_currency=from_cur, + to_currency=to_cur, + pair=sell_pair, + side="sell", + volume=volume, + ) + + raise ValueError(f"No tradable pair for leg {from_cur}->{to_cur}") + + def _build_legs(self, event: OpportunityEvent) -> tuple[ExecutionLeg, ...]: + currencies = [part.strip().upper() for part in event.cycle.split("->") if part.strip()] + if len(currencies) < 4 or currencies[0] != currencies[-1]: + raise ValueError("cycle must be a closed triangular path like A->B->C->A") + + if len(currencies) != 4: + raise ValueError("cycle must contain exactly three unique currencies") + + legs: list[ExecutionLeg] = [] + for idx in range(3): + from_currency = currencies[idx] + to_currency = currencies[idx + 1] + placeholder_leg = ExecutionLeg( + from_currency=from_currency, + to_currency=to_currency, + pair="", + side="buy", + volume=0.0, + ) + volume = self._volume_for_leg(event, placeholder_leg, idx) + if volume <= 0.0: + raise ValueError("volume_for_leg must return a positive volume") + legs.append(self._resolve_leg(from_currency, to_currency, volume)) + + return tuple(legs) + + @staticmethod + def _trade_ref_for_event(event: OpportunityEvent) -> str: + material = ( + f"{event.cycle}|{event.updated_pair}|" + f"{event.detected_at.timestamp():.6f}|" + f"{event.allocated_capital:.12f}" + ) + return material.encode("utf-8").hex()[:32] + + @staticmethod + def _order_ref_from_response(response: dict[str, Any], default: str) -> str: + txid = response.get("txid") + if isinstance(txid, list) and txid: + return str(txid[0]) + if isinstance(txid, str) and txid.strip(): + return txid + return default + + async def execute(self, event: OpportunityEvent) -> TriangularExecutionResult: + legs = self._build_legs(event) + responses: list[dict[str, Any]] = [] + trade_ref = self._trade_ref_for_event(event) + started_at = datetime.now(UTC) + + for idx, leg in enumerate(legs): + try: + response = await self._rest_client.place_market_order( + pair=leg.pair, + side=leg.side, + volume=leg.volume, + ) + except Exception as exc: + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="execution_engine", + event_type="execution.trade.failed", + decision="rejected", + payload={ + "cycle": event.cycle, + "failed_leg_index": idx, + "error": str(exc), + }, + correlation_id=trade_ref, + ) + ) + if self._alert_notifier is not None: + await self._alert_notifier.notify( + category="error", + severity="error", + title="Trade execution failed", + message="Triangular execution failed before completing all legs.", + details={ + "cycle": event.cycle, + "failed_leg_index": str(idx), + "error": str(exc), + }, + ) + if self._execution_writer is not None: + await self._execution_writer.enqueue( + TradeRecord( + trade_ref=trade_ref, + started_at=started_at, + finished_at=datetime.now(UTC), + status="failed", + realized_pnl=None, + estimated_pnl=event.est_profit, + capital_used=event.allocated_capital, + cycle=event.cycle, + leg_count=len(legs), + ) + ) + return TriangularExecutionResult( + success=False, + requested_legs=legs, + completed_legs=idx, + responses=tuple(responses), + failure_reason=str(exc), + ) + + responses.append(response) + + if self._execution_writer is not None: + order_ref = self._order_ref_from_response(response, f"leg-{idx}") + await self._execution_writer.enqueue( + OrderRecord( + trade_ref=trade_ref, + order_ref=order_ref, + leg_index=idx, + pair=leg.pair, + side=leg.side, + volume=leg.volume, + user_ref=None, + status=str(response.get("status", "submitted")), + filled_volume=None, + avg_price=None, + raw_response=response, + recorded_at=datetime.now(UTC), + ) + ) + + if self._execution_writer is not None: + await self._execution_writer.enqueue( + TradeRecord( + trade_ref=trade_ref, + started_at=started_at, + finished_at=datetime.now(UTC), + status="filled", + realized_pnl=None, + estimated_pnl=event.est_profit, + capital_used=event.allocated_capital, + cycle=event.cycle, + leg_count=len(legs), + ) + ) + await self._execution_writer.enqueue( + PnLRecord( + trade_ref=trade_ref, + recorded_at=datetime.now(UTC), + kind="estimated", + pnl_usd=event.est_profit, + source="triangular_sequencer", + ) + ) + + if self._alert_notifier is not None: + await self._alert_notifier.notify( + category="trade", + severity="warning" if event.est_profit < 0.0 else "info", + title="Trade execution completed", + message="Triangular execution completed all requested legs.", + details={ + "cycle": event.cycle, + "completed_legs": str(len(legs)), + "estimated_pnl_usd": f"{event.est_profit}", + }, + ) + + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="execution_engine", + event_type="execution.trade.completed", + decision="approved", + payload={ + "cycle": event.cycle, + "completed_legs": len(legs), + "estimated_pnl_usd": event.est_profit, + }, + correlation_id=trade_ref, + ) + ) + + return TriangularExecutionResult( + success=True, + requested_legs=legs, + completed_legs=len(legs), + responses=tuple(responses), + ) diff --git a/build/lib/arbitrade/logging_setup.py b/build/lib/arbitrade/logging_setup.py new file mode 100644 index 0000000..000f9da --- /dev/null +++ b/build/lib/arbitrade/logging_setup.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import logging +import sys +from typing import Any + +import structlog + + +def configure_logging(log_level: str = "INFO", json_logs: bool = True) -> None: + level = getattr(logging, log_level.upper(), logging.INFO) + + timestamper = structlog.processors.TimeStamper(fmt="iso", utc=True) + + shared_processors: list[Any] = [ + structlog.contextvars.merge_contextvars, + structlog.stdlib.add_log_level, + structlog.stdlib.add_logger_name, + timestamper, + ] + + if json_logs: + renderer: Any = structlog.processors.JSONRenderer() + else: + renderer = structlog.dev.ConsoleRenderer() + + structlog.configure( + processors=[ + *shared_processors, + structlog.processors.dict_tracebacks, + structlog.processors.EventRenamer("message"), + renderer, + ], + wrapper_class=structlog.make_filtering_bound_logger(level), + logger_factory=structlog.stdlib.LoggerFactory(), + cache_logger_on_first_use=True, + ) + + logging.basicConfig(format="%(message)s", stream=sys.stdout, level=level, force=True) diff --git a/build/lib/arbitrade/main.py b/build/lib/arbitrade/main.py new file mode 100644 index 0000000..4b5e119 --- /dev/null +++ b/build/lib/arbitrade/main.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import platform +from importlib import import_module + +import uvicorn + +from arbitrade.api.app import create_app +from arbitrade.config.settings import get_settings + + +def _install_uvloop_if_available() -> None: + if platform.system() == "Windows": + return + + try: + uvloop = import_module("uvloop") + uvloop.install() + except Exception: + # App can still run with default asyncio loop. + return + + +def main() -> None: + _install_uvloop_if_available() + + settings = get_settings() + app = create_app(settings) + + uvicorn.run( + app, + host=settings.app_host, + port=settings.app_port, + log_level=settings.log_level.lower(), + loop="uvloop" if platform.system() != "Windows" else "asyncio", + http="httptools", + ) + + +if __name__ == "__main__": + main() diff --git a/build/lib/arbitrade/market_data/__init__.py b/build/lib/arbitrade/market_data/__init__.py new file mode 100644 index 0000000..1202ad1 --- /dev/null +++ b/build/lib/arbitrade/market_data/__init__.py @@ -0,0 +1 @@ +"""Market data ingestion and book cache package.""" diff --git a/build/lib/arbitrade/market_data/feed.py b/build/lib/arbitrade/market_data/feed.py new file mode 100644 index 0000000..a633789 --- /dev/null +++ b/build/lib/arbitrade/market_data/feed.py @@ -0,0 +1,485 @@ +from __future__ import annotations + +import time +from collections.abc import Awaitable, Callable, Mapping +from dataclasses import dataclass +from datetime import UTC, datetime + +import structlog + +from arbitrade.alerting.notifier import SupportsAlerts, dispatch_alert_nowait +from arbitrade.detection.engine import IncrementalCycleDetector, OpportunityEvent +from arbitrade.exchange.kraken_ws import KrakenWsClient +from arbitrade.market_data.order_book import OrderBook +from arbitrade.risk.kill_switch import KillSwitch +from arbitrade.risk.loss_limits import LossLimitGuard +from arbitrade.risk.pre_trade import PreTradeValidator +from arbitrade.risk.stop_conditions import StopConditionsGuard +from arbitrade.risk.trade_limits import TradeLimitsGuard +from arbitrade.storage.market_snapshots import AsyncMarketSnapshotWriter, MarketSnapshot +from arbitrade.storage.opportunities import AsyncOpportunityWriter +from arbitrade.storage.repositories import AuditRecord, AuditRepository + +_LOG = structlog.get_logger(__name__) + + +@dataclass(frozen=True, slots=True) +class ExecutionOutcome: + realized_pnl: float | None = None + close_trade: bool = True + + +class MarketDataFeed: + def __init__( + self, + ws_client: KrakenWsClient, + snapshot_writer: AsyncMarketSnapshotWriter, + detector: IncrementalCycleDetector | None = None, + opportunity_writer: AsyncOpportunityWriter | None = None, + paper_trading_mode: bool = True, + opportunity_executor: ( + Callable[[OpportunityEvent], Awaitable[ExecutionOutcome | float | None]] | None + ) = None, + trade_capital: float = 1.0, + max_trade_capital: float | None = None, + loss_limit_guard: LossLimitGuard | None = None, + trade_limits_guard: TradeLimitsGuard | None = None, + pre_trade_validator: PreTradeValidator | None = None, + balance_provider: Callable[[], Mapping[str, float]] | None = None, + quote_balance_asset: str = "USD", + kill_switch: KillSwitch | None = None, + stop_conditions_guard: StopConditionsGuard | None = None, + alert_notifier: SupportsAlerts | None = None, + audit_repository: AuditRepository | None = None, + ) -> None: + self._ws_client = ws_client + self._snapshot_writer = snapshot_writer + self._books: dict[str, OrderBook] = {} + self._detector = detector + self._opportunity_writer = opportunity_writer + self._paper_trading_mode = paper_trading_mode + self._opportunity_executor = opportunity_executor + self._trade_capital = trade_capital + self._max_trade_capital = max_trade_capital + self._loss_limit_guard = loss_limit_guard + self._trade_limits_guard = trade_limits_guard + self._pre_trade_validator = pre_trade_validator + self._balance_provider = balance_provider + self._quote_balance_asset = quote_balance_asset.upper() + self._kill_switch = kill_switch + self._stop_conditions_guard = stop_conditions_guard + self._alert_notifier = alert_notifier + self._audit_repository = audit_repository + + if self._trade_capital <= 0.0: + raise ValueError("trade_capital must be > 0.0") + if self._max_trade_capital is not None and self._max_trade_capital <= 0.0: + raise ValueError("max_trade_capital must be > 0.0") + + @property + def books(self) -> dict[str, OrderBook]: + return self._books + + def _effective_trade_capital(self) -> float: + if self._max_trade_capital is None: + return self._trade_capital + return min(self._trade_capital, self._max_trade_capital) + + @staticmethod + def _exposure_for_event(event: OpportunityEvent) -> dict[str, float]: + currencies = [part for part in event.cycle.split("->") if part] + if len(currencies) < 2: + return {} + + start = currencies[0] + exposure_assets = {currency for currency in currencies[1:] if currency != start} + return {asset: event.allocated_capital for asset in exposure_assets} + + async def run(self) -> None: + async for message in self._ws_client.connect_stream(): + parse_start = time.perf_counter() + delta = self._ws_client.parse_book_delta(message.payload) + if delta is None: + continue + + book = self._books.setdefault(delta.symbol, OrderBook()) + book.apply_bids(delta.bids) + book.apply_asks(delta.asks) + + checksum_ok = True + if delta.checksum is not None: + checksum_ok = book.compute_checksum() == delta.checksum + + apply_latency_ms = (time.perf_counter() - parse_start) * 1000 + source_latency_ms: float | None = None + if delta.source_timestamp_ms is not None: + source_latency_ms = datetime.now(UTC).timestamp() * 1000 - float( + delta.source_timestamp_ms + ) + + _LOG.info( + "book_delta_applied", + symbol=delta.symbol, + bids=len(delta.bids), + asks=len(delta.asks), + checksum_ok=checksum_ok, + apply_latency_ms=apply_latency_ms, + source_latency_ms=source_latency_ms, + ) + + if self._stop_conditions_guard is not None: + self._stop_conditions_guard.observe_latency( + source_latency_ms=source_latency_ms, + apply_latency_ms=apply_latency_ms, + ) + if self._stop_conditions_guard.is_halted: + if self._kill_switch is not None and not self._kill_switch.is_active: + self._kill_switch.activate( + reason=self._stop_conditions_guard.halted_reason + or "stop_conditions_halted", + ) + _LOG.warning( + "stop_condition_halt_triggered", + reason=self._stop_conditions_guard.halted_reason, + symbol=delta.symbol, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="risk_manager", + event_type="risk.stop_condition_halt", + decision="rejected", + payload={ + "reason": self._stop_conditions_guard.halted_reason + or "unknown", + "symbol": delta.symbol, + }, + ) + ) + + if self._detector is not None: + opportunities = self._detector.opportunities_for_updated_pair( + delta.symbol, + self._books, + base_capital=self._effective_trade_capital(), + ) + _LOG.debug( + "incremental_opportunity_scores", + symbol=delta.symbol, + opportunities=len(opportunities), + ) + + for event in opportunities: + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="detector", + event_type="detector.opportunity", + decision="scored", + payload={ + "cycle": event.cycle, + "updated_pair": event.updated_pair, + "net_pct": event.net_pct, + "est_profit": event.est_profit, + }, + ) + ) + _LOG.info( + "opportunity_detected", + cycle=event.cycle, + updated_pair=event.updated_pair, + gross_pct=event.gross_pct, + net_pct=event.net_pct, + est_profit=event.est_profit, + mode="paper" if self._paper_trading_mode else "live", + ) + + if self._opportunity_writer is not None: + await self._opportunity_writer.enqueue(event) + + if self._paper_trading_mode: + _LOG.info( + "paper_trade_simulated", + cycle=event.cycle, + updated_pair=event.updated_pair, + net_pct=event.net_pct, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="execution_engine", + event_type="execution.paper_trade", + decision="skipped", + payload={ + "cycle": event.cycle, + "updated_pair": event.updated_pair, + }, + ) + ) + continue + + if self._opportunity_executor is None: + _LOG.warning( + "live_trade_skipped_no_executor", + cycle=event.cycle, + updated_pair=event.updated_pair, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="execution_engine", + event_type="execution.live_trade", + decision="rejected", + payload={ + "reason": "missing_executor", + "cycle": event.cycle, + }, + ) + ) + continue + + if self._kill_switch is not None and self._kill_switch.is_active: + _LOG.warning( + "live_trade_skipped_kill_switch", + cycle=event.cycle, + updated_pair=event.updated_pair, + reason=self._kill_switch.reason, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="risk_manager", + event_type="risk.kill_switch", + decision="rejected", + payload={ + "reason": self._kill_switch.reason or "manual", + "cycle": event.cycle, + }, + ) + ) + continue + + if ( + self._stop_conditions_guard is not None + and self._stop_conditions_guard.is_halted + ): + _LOG.warning( + "live_trade_skipped_stop_condition_halt", + cycle=event.cycle, + updated_pair=event.updated_pair, + reason=self._stop_conditions_guard.halted_reason, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="risk_manager", + event_type="risk.stop_condition", + decision="rejected", + payload={ + "reason": self._stop_conditions_guard.halted_reason + or "halted", + "cycle": event.cycle, + }, + ) + ) + continue + + if self._loss_limit_guard is not None and self._loss_limit_guard.is_halted: + _LOG.warning( + "live_trade_skipped_loss_limit_halted", + cycle=event.cycle, + updated_pair=event.updated_pair, + reason=self._loss_limit_guard.halted_reason, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="risk_manager", + event_type="risk.loss_limit", + decision="rejected", + payload={ + "reason": self._loss_limit_guard.halted_reason or "halted", + "cycle": event.cycle, + }, + ) + ) + continue + + if self._pre_trade_validator is not None and self._balance_provider is not None: + required_balances = {self._quote_balance_asset: event.allocated_capital} + balances = { + asset.upper(): amount + for asset, amount in self._balance_provider().items() + } + if not self._pre_trade_validator.validate( + balances_by_asset=balances, + required_by_asset=required_balances, + ): + _LOG.warning( + "live_trade_skipped_pre_trade_validation", + cycle=event.cycle, + updated_pair=event.updated_pair, + required_by_asset=required_balances, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="risk_manager", + event_type="risk.pre_trade_validation", + decision="rejected", + payload={ + "cycle": event.cycle, + "required_by_asset": { + key: required_balances[key] + for key in required_balances + }, + }, + ) + ) + continue + + exposure_by_asset = self._exposure_for_event(event) + if ( + self._trade_limits_guard is not None + and not self._trade_limits_guard.is_trade_allowed(exposure_by_asset) + ): + _LOG.warning( + "live_trade_skipped_trade_limits", + cycle=event.cycle, + updated_pair=event.updated_pair, + exposure_by_asset=exposure_by_asset, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="risk_manager", + event_type="risk.trade_limits", + decision="rejected", + payload={ + "cycle": event.cycle, + "exposure_by_asset": { + key: exposure_by_asset[key] for key in exposure_by_asset + }, + }, + ) + ) + continue + + if self._trade_limits_guard is not None: + self._trade_limits_guard.open_trade(exposure_by_asset) + + try: + outcome = await self._opportunity_executor(event) + except Exception as exc: + if self._trade_limits_guard is not None: + self._trade_limits_guard.close_trade(exposure_by_asset) + + dispatch_alert_nowait( + self._alert_notifier, + category="system", + severity="critical", + title="Critical execution exception", + message="Unhandled exception raised by opportunity executor.", + details={ + "cycle": event.cycle, + "updated_pair": event.updated_pair, + "error": str(exc), + }, + ) + + if self._stop_conditions_guard is not None: + self._stop_conditions_guard.register_failure() + if self._stop_conditions_guard.is_halted: + if ( + self._kill_switch is not None + and not self._kill_switch.is_active + ): + self._kill_switch.activate( + reason=self._stop_conditions_guard.halted_reason + or "stop_conditions_halted", + ) + _LOG.warning( + "stop_condition_halt_triggered", + reason=self._stop_conditions_guard.halted_reason, + cycle=event.cycle, + updated_pair=event.updated_pair, + ) + + _LOG.exception( + "live_trade_execution_failed", + cycle=event.cycle, + updated_pair=event.updated_pair, + ) + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="execution_engine", + event_type="execution.live_trade", + decision="error", + payload={ + "cycle": event.cycle, + "updated_pair": event.updated_pair, + "error": str(exc), + }, + ) + ) + continue + + if self._stop_conditions_guard is not None: + self._stop_conditions_guard.register_success() + + realized_pnl: float | None + close_trade = True + if isinstance(outcome, ExecutionOutcome): + realized_pnl = outcome.realized_pnl + close_trade = outcome.close_trade + else: + realized_pnl = outcome + + if realized_pnl is not None and self._loss_limit_guard is not None: + self._loss_limit_guard.register_realized_pnl(realized_pnl) + if self._loss_limit_guard.is_halted: + _LOG.warning( + "loss_limit_halt_triggered", + reason=self._loss_limit_guard.halted_reason, + cumulative_pnl=self._loss_limit_guard.cumulative_pnl, + ) + + if self._trade_limits_guard is not None and close_trade: + self._trade_limits_guard.close_trade(exposure_by_asset) + + if self._audit_repository is not None: + self._audit_repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="execution_engine", + event_type="execution.live_trade", + decision="approved", + payload={ + "cycle": event.cycle, + "updated_pair": event.updated_pair, + "realized_pnl": realized_pnl, + "close_trade": close_trade, + }, + ) + ) + + await self._snapshot_writer.enqueue( + MarketSnapshot( + snapshot_at=datetime.now(UTC), + symbol=delta.symbol, + source="kraken_ws", + payload=message.payload, + latency_ms=source_latency_ms, + ) + ) diff --git a/build/lib/arbitrade/market_data/order_book.py b/build/lib/arbitrade/market_data/order_book.py new file mode 100644 index 0000000..a4ba86a --- /dev/null +++ b/build/lib/arbitrade/market_data/order_book.py @@ -0,0 +1,104 @@ +from __future__ import annotations + +import re +from collections.abc import Iterable +from dataclasses import dataclass +from datetime import UTC, datetime + +from sortedcontainers import SortedDict + +from arbitrade.exchange.models import BookLevel + +ZERO_CLEAN_RE = re.compile(r"^0+", re.ASCII) + + +def _normalize_price_for_checksum(value: float) -> str: + text = f"{value:.10f}".replace(".", "") + text = text.rstrip("0") + stripped = ZERO_CLEAN_RE.sub("", text) + return stripped or "0" + + +def _normalize_volume_for_checksum(value: float) -> str: + text = f"{value:.10f}".replace(".", "") + text = text.rstrip("0") + stripped = ZERO_CLEAN_RE.sub("", text) + return stripped or "0" + + +@dataclass(slots=True) +class BookView: + best_bid: BookLevel | None + best_ask: BookLevel | None + updated_at: datetime + + +class OrderBook: + def __init__(self) -> None: + self._bids: SortedDict[float, float] = SortedDict() + self._asks: SortedDict[float, float] = SortedDict() + self._updated_at: datetime = datetime.now(UTC) + + @property + def updated_at(self) -> datetime: + return self._updated_at + + def apply_bids(self, updates: Iterable[BookLevel]) -> None: + for level in updates: + if level.volume <= 0: + self._bids.pop(level.price, None) + else: + self._bids[level.price] = level.volume + self._updated_at = datetime.now(UTC) + + def apply_asks(self, updates: Iterable[BookLevel]) -> None: + for level in updates: + if level.volume <= 0: + self._asks.pop(level.price, None) + else: + self._asks[level.price] = level.volume + self._updated_at = datetime.now(UTC) + + def best_bid(self) -> BookLevel | None: + if not self._bids: + return None + price = self._bids.peekitem(-1)[0] + return BookLevel(price=price, volume=self._bids[price]) + + def best_ask(self) -> BookLevel | None: + if not self._asks: + return None + price = self._asks.peekitem(0)[0] + return BookLevel(price=price, volume=self._asks[price]) + + def snapshot(self) -> BookView: + return BookView( + best_bid=self.best_bid(), + best_ask=self.best_ask(), + updated_at=self._updated_at, + ) + + def top_levels(self, depth: int = 10) -> tuple[list[BookLevel], list[BookLevel]]: + bid_keys = list(self._bids.keys()) + ask_keys = list(self._asks.keys()) + + bids = [ + BookLevel(price=price, volume=self._bids[price]) + for price in reversed(bid_keys[-depth:]) + ] + asks = [BookLevel(price=price, volume=self._asks[price]) for price in ask_keys[:depth]] + return bids, asks + + def compute_checksum(self, depth: int = 10) -> int: + bids, asks = self.top_levels(depth) + combined: list[str] = [] + for level in bids: + combined.append(_normalize_price_for_checksum(level.price)) + combined.append(_normalize_volume_for_checksum(level.volume)) + for level in asks: + combined.append(_normalize_price_for_checksum(level.price)) + combined.append(_normalize_volume_for_checksum(level.volume)) + + import zlib + + return zlib.crc32("".join(combined).encode("utf-8")) diff --git a/build/lib/arbitrade/metrics.py b/build/lib/arbitrade/metrics.py new file mode 100644 index 0000000..aadaf32 --- /dev/null +++ b/build/lib/arbitrade/metrics.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime + +from arbitrade.storage.db import DuckDBStore + + +@dataclass(frozen=True, slots=True) +class PerformanceMetrics: + realized_pnl_usd: float + win_rate: float | None + avg_trade_duration_seconds: float | None + opportunities_per_minute: float | None + fill_rate: float | None + latency_p50_seconds: float | None + latency_p95_seconds: float | None + latency_p99_seconds: float | None + + +class MetricsCalculator: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def compute(self) -> PerformanceMetrics: + with self._store.connect() as conn: + tm = conn.execute(""" + SELECT + COALESCE(SUM(COALESCE(realized_pnl, 0)), 0) AS realized_pnl_usd, + COUNT(*) AS total_trades, + SUM(CASE WHEN realized_pnl > 0 THEN 1 ELSE 0 END) AS winning_trades, + AVG(EPOCH(finished_at) - EPOCH(started_at)) AS avg_trade_duration_seconds, + quantile_cont( + EPOCH(finished_at) - EPOCH(started_at), + 0.50 + ) AS latency_p50_seconds, + quantile_cont( + EPOCH(finished_at) - EPOCH(started_at), + 0.95 + ) AS latency_p95_seconds, + quantile_cont( + EPOCH(finished_at) - EPOCH(started_at), + 0.99 + ) AS latency_p99_seconds + FROM trades + WHERE finished_at IS NOT NULL + """).fetchone() + + om = conn.execute(""" + SELECT + COUNT(*) AS opportunity_count, + MIN(detected_at) AS first_detected_at, + MAX(detected_at) AS last_detected_at + FROM opportunities + """).fetchone() + + fm = conn.execute(""" + SELECT AVG(filled_volume / volume) AS fill_rate + FROM orders + WHERE volume > 0 AND filled_volume IS NOT NULL + """).fetchone() + + r_pnl_usd = float(tm[0]) if tm and tm[0] is not None else 0.0 + tt = int(tm[1]) if tm and tm[1] is not None else 0 + wt = int(tm[2]) if tm and tm[2] is not None else 0 + wr = wt / tt if tt > 0 else None + + atd = float(tm[3]) if tm and tm[3] is not None else None + + oc = int(om[0]) if om is not None and om[0] is not None else 0 + fo = om[1] if om is not None and isinstance(om[1], datetime) else None + lo = om[2] if om is not None and isinstance(om[2], datetime) else None + + opportunities_per_minute: float | None + if oc >= 2 and fo is not None and lo is not None: + span_seconds = (lo - fo).total_seconds() + opportunities_per_minute = ( + oc / (span_seconds / 60.0) if span_seconds > 0.0 else float(oc) + ) + elif oc == 1: + opportunities_per_minute = 60.0 + else: + opportunities_per_minute = None + + fill_rate = float(fm[0]) if fm and fm[0] is not None else None + + lp50 = float(tm[4]) if tm and tm[4] is not None else None + lp95 = float(tm[5]) if tm and tm[5] is not None else None + lp99 = float(tm[6]) if tm and tm[6] is not None else None + + return PerformanceMetrics( + realized_pnl_usd=r_pnl_usd, + win_rate=wr, + avg_trade_duration_seconds=atd, + opportunities_per_minute=opportunities_per_minute, + fill_rate=fill_rate, + latency_p50_seconds=lp50, + latency_p95_seconds=lp95, + latency_p99_seconds=lp99, + ) diff --git a/build/lib/arbitrade/perf/__init__.py b/build/lib/arbitrade/perf/__init__.py new file mode 100644 index 0000000..f0dbf88 --- /dev/null +++ b/build/lib/arbitrade/perf/__init__.py @@ -0,0 +1,4 @@ +from arbitrade.perf.guardrails import evaluate_guardrails +from arbitrade.perf.latency import run_latency_profile + +__all__ = ["run_latency_profile", "evaluate_guardrails"] diff --git a/build/lib/arbitrade/perf/guardrails.py b/build/lib/arbitrade/perf/guardrails.py new file mode 100644 index 0000000..51bcf4a --- /dev/null +++ b/build/lib/arbitrade/perf/guardrails.py @@ -0,0 +1,80 @@ +from __future__ import annotations + + +def evaluate_guardrails( + *, + baseline: dict[str, object], + current: dict[str, object], + thresholds: dict[str, object], +) -> list[str]: + failures: list[str] = [] + + baseline_scenarios = baseline.get("scenarios") + current_scenarios = current.get("scenarios") + if not isinstance(baseline_scenarios, dict) or not isinstance(current_scenarios, dict): + return ["invalid profile payload: missing scenarios map"] + + default_thresholds = thresholds.get("default") + if not isinstance(default_thresholds, dict): + default_thresholds = {"p95_ms": 2.5, "p99_ms": 3.0} + + scenario_thresholds = thresholds.get("scenarios") + if not isinstance(scenario_thresholds, dict): + scenario_thresholds = {} + + for scenario, baseline_payload in baseline_scenarios.items(): + current_payload = current_scenarios.get(scenario) + if not isinstance(baseline_payload, dict) or not isinstance(current_payload, dict): + failures.append(f"missing scenario in current profile: {scenario}") + continue + + baseline_stages = baseline_payload.get("stages") + current_stages = current_payload.get("stages") + if not isinstance(baseline_stages, dict) or not isinstance(current_stages, dict): + failures.append(f"missing stages map for scenario: {scenario}") + continue + + scenario_config = scenario_thresholds.get(scenario) + if not isinstance(scenario_config, dict): + scenario_config = {} + + for stage, baseline_stage in baseline_stages.items(): + current_stage = current_stages.get(stage) + if not isinstance(baseline_stage, dict) or not isinstance(current_stage, dict): + failures.append(f"missing stage in current profile: {scenario}.{stage}") + continue + + for percentile_key in ("p95_ms", "p99_ms"): + threshold_ratio_raw = scenario_config.get( + percentile_key, + default_thresholds.get(percentile_key, 3.0), + ) + threshold_ratio = ( + float(threshold_ratio_raw) + if isinstance(threshold_ratio_raw, int | float) + else 3.0 + ) + + base_value_raw = baseline_stage.get(percentile_key) + current_value_raw = current_stage.get(percentile_key) + if not isinstance(base_value_raw, int | float) or not isinstance( + current_value_raw, int | float + ): + failures.append( + f"invalid percentile value: {scenario}.{stage}.{percentile_key}" + ) + continue + + base_value = float(base_value_raw) + current_value = float(current_value_raw) + # Avoid divide-by-zero while still preserving strict checks. + max_allowed = max(base_value * threshold_ratio, 0.001) + if current_value > max_allowed: + failures.append( + f"latency regression: {scenario}.{stage}.{percentile_key} " + f"current={current_value:.4f}ms " + f"baseline={base_value:.4f}ms " + f"allowed={max_allowed:.4f}ms" + ) + + return failures diff --git a/build/lib/arbitrade/perf/latency.py b/build/lib/arbitrade/perf/latency.py new file mode 100644 index 0000000..8749a03 --- /dev/null +++ b/build/lib/arbitrade/perf/latency.py @@ -0,0 +1,195 @@ +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass +from time import perf_counter_ns + +import orjson + + +@dataclass(frozen=True, slots=True) +class PercentileSummary: + p50_ms: float + p95_ms: float + p99_ms: float + + +@dataclass(frozen=True, slots=True) +class ScenarioProfile: + scenario: str + iterations: int + stages: dict[str, PercentileSummary] + + +def _percentile(samples: list[float], percentile: float) -> float: + if not samples: + return 0.0 + ordered = sorted(samples) + if percentile <= 0.0: + return ordered[0] + if percentile >= 100.0: + return ordered[-1] + rank = (len(ordered) - 1) * (percentile / 100.0) + lower = int(rank) + upper = min(lower + 1, len(ordered) - 1) + weight = rank - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def _summarize(samples: list[float]) -> PercentileSummary: + return PercentileSummary( + p50_ms=_percentile(samples, 50.0), + p95_ms=_percentile(samples, 95.0), + p99_ms=_percentile(samples, 99.0), + ) + + +def _ingest_stage(raw_payload: bytes, state: dict[str, float]) -> None: + parsed = orjson.loads(raw_payload) + bids = parsed.get("bids", []) + asks = parsed.get("asks", []) + for price, volume in bids[:4]: + state[str(price)] = float(volume) + for price, volume in asks[:4]: + state[str(price)] = float(volume) + + +def _detect_stage(values: list[float], cycles: int) -> float: + best = 0.0 + size = len(values) + for idx in range(cycles): + a = values[idx % size] + b = values[(idx + 3) % size] + c = values[(idx + 7) % size] + gross = (a / b) * c + net = gross * 0.9975 + if net > best: + best = net + return best + + +def _risk_stage(net_edge: float, capital: float) -> float: + if net_edge < 1.0002: + return 0.0 + if capital > 500.0: + capital = 500.0 + return capital * min(net_edge - 1.0, 0.02) + + +def _execution_stage(planned_pnl: float, order_id: int) -> None: + payload = { + "order_id": order_id, + "planned_pnl": planned_pnl, + "legs": [ + {"pair": "BTC/USD", "side": "buy", "qty": 0.01}, + {"pair": "ETH/BTC", "side": "buy", "qty": 0.1}, + {"pair": "ETH/USD", "side": "sell", "qty": 0.1}, + ], + } + _ = orjson.dumps(payload) + + +def _run_scenario( + name: str, + iterations: int, + detect_cycles: int, + reconnect_every: int, +) -> ScenarioProfile: + payloads = [ + orjson.dumps( + { + "symbol": "BTC/USD", + "bids": [[100000.0 + i, 0.2 + (i % 5) * 0.01] for i in range(12)], + "asks": [[100001.0 + i, 0.2 + (i % 7) * 0.01] for i in range(12)], + } + ) + for _ in range(5) + ] + value_series = [1.0 + (idx % 31) * 0.0007 for idx in range(128)] + order_state: dict[str, float] = {} + + ingest_ms: list[float] = [] + detect_ms: list[float] = [] + risk_ms: list[float] = [] + execution_ms: list[float] = [] + end_to_end_ms: list[float] = [] + + for idx in range(iterations): + start_ns = perf_counter_ns() + payload = payloads[idx % len(payloads)] + + t0 = perf_counter_ns() + _ingest_stage(payload, order_state) + if reconnect_every > 0 and idx > 0 and idx % reconnect_every == 0: + order_state.clear() + t1 = perf_counter_ns() + + net_edge = _detect_stage(value_series, detect_cycles) + t2 = perf_counter_ns() + + planned = _risk_stage(net_edge, capital=100.0 + (idx % 50)) + t3 = perf_counter_ns() + + _execution_stage(planned, order_id=idx) + t4 = perf_counter_ns() + + ingest_ms.append((t1 - t0) / 1_000_000.0) + detect_ms.append((t2 - t1) / 1_000_000.0) + risk_ms.append((t3 - t2) / 1_000_000.0) + execution_ms.append((t4 - t3) / 1_000_000.0) + end_to_end_ms.append((t4 - start_ns) / 1_000_000.0) + + return ScenarioProfile( + scenario=name, + iterations=iterations, + stages={ + "ingest": _summarize(ingest_ms), + "detect": _summarize(detect_ms), + "risk": _summarize(risk_ms), + "execution": _summarize(execution_ms), + "end_to_end": _summarize(end_to_end_ms), + }, + ) + + +def run_latency_profile(iterations: int = 600) -> dict[str, object]: + scenarios: list[Callable[[], ScenarioProfile]] = [ + lambda: _run_scenario( + name="book_update_burst", + iterations=iterations, + detect_cycles=32, + reconnect_every=0, + ), + lambda: _run_scenario( + name="execution_spike", + iterations=iterations, + detect_cycles=96, + reconnect_every=0, + ), + lambda: _run_scenario( + name="reconnect_storm", + iterations=iterations, + detect_cycles=48, + reconnect_every=20, + ), + ] + + result: dict[str, object] = {"iterations": iterations, "scenarios": {}} + scenario_map = result["scenarios"] + assert isinstance(scenario_map, dict) + + for scenario in scenarios: + profile = scenario() + scenario_map[profile.scenario] = { + "iterations": profile.iterations, + "stages": { + stage: { + "p50_ms": summary.p50_ms, + "p95_ms": summary.p95_ms, + "p99_ms": summary.p99_ms, + } + for stage, summary in profile.stages.items() + }, + } + + return result diff --git a/build/lib/arbitrade/risk/__init__.py b/build/lib/arbitrade/risk/__init__.py new file mode 100644 index 0000000..f90c40f --- /dev/null +++ b/build/lib/arbitrade/risk/__init__.py @@ -0,0 +1,15 @@ +"""Risk management helpers.""" + +from arbitrade.risk.kill_switch import KillSwitch +from arbitrade.risk.loss_limits import LossLimitGuard +from arbitrade.risk.pre_trade import PreTradeValidator +from arbitrade.risk.stop_conditions import StopConditionsGuard +from arbitrade.risk.trade_limits import TradeLimitsGuard + +__all__ = [ + "LossLimitGuard", + "TradeLimitsGuard", + "PreTradeValidator", + "KillSwitch", + "StopConditionsGuard", +] diff --git a/build/lib/arbitrade/risk/kill_switch.py b/build/lib/arbitrade/risk/kill_switch.py new file mode 100644 index 0000000..3b3182c --- /dev/null +++ b/build/lib/arbitrade/risk/kill_switch.py @@ -0,0 +1,23 @@ +from __future__ import annotations + + +class KillSwitch: + def __init__(self, *, active: bool = False, reason: str | None = None) -> None: + self._active = active + self._reason = reason or ("manual" if active else None) + + @property + def is_active(self) -> bool: + return self._active + + @property + def reason(self) -> str | None: + return self._reason + + def activate(self, *, reason: str = "manual") -> None: + self._active = True + self._reason = reason + + def deactivate(self) -> None: + self._active = False + self._reason = None diff --git a/build/lib/arbitrade/risk/loss_limits.py b/build/lib/arbitrade/risk/loss_limits.py new file mode 100644 index 0000000..ab33199 --- /dev/null +++ b/build/lib/arbitrade/risk/loss_limits.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +from datetime import UTC, date, datetime + +from arbitrade.alerting.notifier import SupportsAlerts, dispatch_alert_nowait + + +class LossLimitGuard: + def __init__( + self, + *, + daily_loss_limit: float | None = None, + cumulative_loss_limit: float | None = None, + alert_notifier: SupportsAlerts | None = None, + ) -> None: + self._daily_loss_limit = daily_loss_limit + self._cumulative_loss_limit = cumulative_loss_limit + + if self._daily_loss_limit is not None and self._daily_loss_limit <= 0.0: + raise ValueError("daily_loss_limit must be > 0.0") + if self._cumulative_loss_limit is not None and self._cumulative_loss_limit <= 0.0: + raise ValueError("cumulative_loss_limit must be > 0.0") + + self._cumulative_pnl = 0.0 + self._daily_pnl: dict[date, float] = {} + self._halted_reason: str | None = None + self._alert_notifier = alert_notifier + + @property + def cumulative_pnl(self) -> float: + return self._cumulative_pnl + + @property + def halted_reason(self) -> str | None: + return self._halted_reason + + @property + def is_halted(self) -> bool: + return self._halted_reason is not None + + def daily_pnl(self, day: date) -> float: + return self._daily_pnl.get(day, 0.0) + + def register_realized_pnl(self, pnl: float, *, at: datetime | None = None) -> None: + if self.is_halted: + return + + timestamp = at or datetime.now(UTC) + day_key = timestamp.date() + + self._cumulative_pnl += pnl + self._daily_pnl[day_key] = self._daily_pnl.get(day_key, 0.0) + pnl + + if ( + self._daily_loss_limit is not None + and self._daily_pnl[day_key] <= -self._daily_loss_limit + ): + self._halted_reason = "daily_loss_limit_breached" + dispatch_alert_nowait( + self._alert_notifier, + category="threshold", + severity="critical", + title="Daily loss limit breached", + message="Trading halted because daily realized PnL crossed configured loss limit.", + details={ + "daily_pnl": f"{self._daily_pnl[day_key]}", + "daily_loss_limit": f"{self._daily_loss_limit}", + }, + ) + return + + if ( + self._cumulative_loss_limit is not None + and self._cumulative_pnl <= -self._cumulative_loss_limit + ): + self._halted_reason = "cumulative_loss_limit_breached" + dispatch_alert_nowait( + self._alert_notifier, + category="threshold", + severity="critical", + title="Cumulative loss limit breached", + message=( + "Trading halted because cumulative realized PnL crossed " + "configured loss limit." + ), + details={ + "cumulative_pnl": f"{self._cumulative_pnl}", + "cumulative_loss_limit": f"{self._cumulative_loss_limit}", + }, + ) diff --git a/build/lib/arbitrade/risk/pre_trade.py b/build/lib/arbitrade/risk/pre_trade.py new file mode 100644 index 0000000..74ae2ec --- /dev/null +++ b/build/lib/arbitrade/risk/pre_trade.py @@ -0,0 +1,43 @@ +from __future__ import annotations + +from collections.abc import Mapping + + +class PreTradeValidator: + def __init__( + self, + *, + min_order_size_by_asset: Mapping[str, float] | None = None, + ) -> None: + self._min_order_size_by_asset = { + asset.upper(): float(value) for asset, value in (min_order_size_by_asset or {}).items() + } + + for value in self._min_order_size_by_asset.values(): + if value <= 0.0: + raise ValueError("minimum order size must be > 0.0") + + def validate( + self, + *, + balances_by_asset: Mapping[str, float], + required_by_asset: Mapping[str, float], + ) -> bool: + # Minimum order size checks first to fail fast on structural invalid sizes. + for asset, required in required_by_asset.items(): + if required <= 0.0: + continue + + min_size = self._min_order_size_by_asset.get(asset.upper()) + if min_size is not None and required < min_size: + return False + + # Balance checks ensure required quantity is currently available. + for asset, required in required_by_asset.items(): + if required <= 0.0: + continue + available = balances_by_asset.get(asset.upper(), 0.0) + if available < required: + return False + + return True diff --git a/build/lib/arbitrade/risk/stop_conditions.py b/build/lib/arbitrade/risk/stop_conditions.py new file mode 100644 index 0000000..1691787 --- /dev/null +++ b/build/lib/arbitrade/risk/stop_conditions.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from arbitrade.alerting.notifier import SupportsAlerts, dispatch_alert_nowait + + +class StopConditionsGuard: + def __init__( + self, + *, + max_source_latency_ms: float | None = None, + max_apply_latency_ms: float | None = None, + max_consecutive_failures: int | None = None, + alert_notifier: SupportsAlerts | None = None, + ) -> None: + if max_source_latency_ms is not None and max_source_latency_ms <= 0.0: + raise ValueError("max_source_latency_ms must be > 0.0") + if max_apply_latency_ms is not None and max_apply_latency_ms <= 0.0: + raise ValueError("max_apply_latency_ms must be > 0.0") + if max_consecutive_failures is not None and max_consecutive_failures <= 0: + raise ValueError("max_consecutive_failures must be > 0") + + self._max_source_latency_ms = max_source_latency_ms + self._max_apply_latency_ms = max_apply_latency_ms + self._max_consecutive_failures = max_consecutive_failures + + self._consecutive_failures = 0 + self._halted_reason: str | None = None + self._alert_notifier = alert_notifier + + @property + def halted_reason(self) -> str | None: + return self._halted_reason + + @property + def is_halted(self) -> bool: + return self._halted_reason is not None + + @property + def consecutive_failures(self) -> int: + return self._consecutive_failures + + def observe_latency( + self, + *, + source_latency_ms: float | None, + apply_latency_ms: float, + ) -> None: + if self.is_halted: + return + + if ( + self._max_source_latency_ms is not None + and source_latency_ms is not None + and source_latency_ms > self._max_source_latency_ms + ): + self._halted_reason = "source_latency_limit_breached" + dispatch_alert_nowait( + self._alert_notifier, + category="threshold", + severity="critical", + title="Source latency limit breached", + message="Trading halted because source latency exceeded configured limit.", + details={ + "source_latency_ms": f"{source_latency_ms}", + "max_source_latency_ms": f"{self._max_source_latency_ms}", + }, + ) + return + + if self._max_apply_latency_ms is not None and apply_latency_ms > self._max_apply_latency_ms: + self._halted_reason = "apply_latency_limit_breached" + dispatch_alert_nowait( + self._alert_notifier, + category="threshold", + severity="critical", + title="Apply latency limit breached", + message="Trading halted because apply latency exceeded configured limit.", + details={ + "apply_latency_ms": f"{apply_latency_ms}", + "max_apply_latency_ms": f"{self._max_apply_latency_ms}", + }, + ) + + def register_failure(self) -> None: + if self.is_halted: + return + + self._consecutive_failures += 1 + if ( + self._max_consecutive_failures is not None + and self._consecutive_failures >= self._max_consecutive_failures + ): + self._halted_reason = "consecutive_failures_limit_breached" + dispatch_alert_nowait( + self._alert_notifier, + category="threshold", + severity="critical", + title="Consecutive failures limit breached", + message="Trading halted because consecutive failures exceeded configured limit.", + details={ + "consecutive_failures": f"{self._consecutive_failures}", + "max_consecutive_failures": f"{self._max_consecutive_failures}", + }, + ) + + def register_success(self) -> None: + if self.is_halted: + return + self._consecutive_failures = 0 diff --git a/build/lib/arbitrade/risk/trade_limits.py b/build/lib/arbitrade/risk/trade_limits.py new file mode 100644 index 0000000..978142b --- /dev/null +++ b/build/lib/arbitrade/risk/trade_limits.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from collections.abc import Mapping + +from arbitrade.alerting.notifier import SupportsAlerts, dispatch_alert_nowait + + +class TradeLimitsGuard: + def __init__( + self, + *, + max_concurrent_trades: int | None = None, + max_exposure_per_asset: float | None = None, + alert_notifier: SupportsAlerts | None = None, + ) -> None: + if max_concurrent_trades is not None and max_concurrent_trades <= 0: + raise ValueError("max_concurrent_trades must be > 0") + if max_exposure_per_asset is not None and max_exposure_per_asset <= 0.0: + raise ValueError("max_exposure_per_asset must be > 0.0") + + self._max_concurrent_trades = max_concurrent_trades + self._max_exposure_per_asset = max_exposure_per_asset + self._active_trades = 0 + self._asset_exposure: dict[str, float] = {} + self._alert_notifier = alert_notifier + + @property + def active_trades(self) -> int: + return self._active_trades + + def exposure_for_asset(self, asset: str) -> float: + return self._asset_exposure.get(asset.upper(), 0.0) + + def is_trade_allowed(self, exposure_by_asset: Mapping[str, float]) -> bool: + if ( + self._max_concurrent_trades is not None + and self._active_trades >= self._max_concurrent_trades + ): + dispatch_alert_nowait( + self._alert_notifier, + category="threshold", + severity="warning", + title="Concurrent trade limit reached", + message="Trade rejected by concurrent trade cap.", + details={ + "active_trades": f"{self._active_trades}", + "max_concurrent_trades": f"{self._max_concurrent_trades}", + }, + ) + return False + + if self._max_exposure_per_asset is None: + return True + + for asset, exposure in exposure_by_asset.items(): + if exposure <= 0.0: + continue + key = asset.upper() + next_exposure = self._asset_exposure.get(key, 0.0) + exposure + if next_exposure > self._max_exposure_per_asset: + dispatch_alert_nowait( + self._alert_notifier, + category="threshold", + severity="warning", + title="Asset exposure limit reached", + message="Trade rejected by per-asset exposure cap.", + details={ + "asset": key, + "next_exposure": f"{next_exposure}", + "max_exposure_per_asset": f"{self._max_exposure_per_asset}", + }, + ) + return False + + return True + + def open_trade(self, exposure_by_asset: Mapping[str, float]) -> None: + self._active_trades += 1 + for asset, exposure in exposure_by_asset.items(): + if exposure <= 0.0: + continue + key = asset.upper() + self._asset_exposure[key] = self._asset_exposure.get(key, 0.0) + exposure + + def close_trade(self, exposure_by_asset: Mapping[str, float]) -> None: + if self._active_trades > 0: + self._active_trades -= 1 + + for asset, exposure in exposure_by_asset.items(): + if exposure <= 0.0: + continue + key = asset.upper() + current = self._asset_exposure.get(key, 0.0) + next_exposure = max(current - exposure, 0.0) + if next_exposure == 0.0: + self._asset_exposure.pop(key, None) + else: + self._asset_exposure[key] = next_exposure diff --git a/build/lib/arbitrade/runtime/__init__.py b/build/lib/arbitrade/runtime/__init__.py new file mode 100644 index 0000000..210b16c --- /dev/null +++ b/build/lib/arbitrade/runtime/__init__.py @@ -0,0 +1,15 @@ +"""Runtime lifecycle and recovery helpers.""" + +from arbitrade.runtime.lifecycle import ( + RuntimeRecoveryReport, + graceful_shutdown, + persist_runtime_snapshot, + restore_runtime_state, +) + +__all__ = [ + "RuntimeRecoveryReport", + "graceful_shutdown", + "persist_runtime_snapshot", + "restore_runtime_state", +] diff --git a/build/lib/arbitrade/runtime/lifecycle.py b/build/lib/arbitrade/runtime/lifecycle.py new file mode 100644 index 0000000..c00a0e0 --- /dev/null +++ b/build/lib/arbitrade/runtime/lifecycle.py @@ -0,0 +1,223 @@ +from __future__ import annotations + +import inspect +from dataclasses import dataclass +from datetime import UTC, datetime +from typing import Any, cast + +from fastapi import FastAPI + +from arbitrade.api.control_state import DashboardControlState +from arbitrade.storage.db import DuckDBStore +from arbitrade.storage.repositories import ( + AuditRecord, + AuditRepository, + RuntimeStateRecord, + RuntimeStateRepository, +) + + +@dataclass(slots=True) +class RuntimeRecoveryReport: + restored_from_snapshot: bool + snapshot_at: str | None + open_trades_detected: int + restart_guard_active: bool + + +def _controls(app: FastAPI) -> DashboardControlState: + return cast(DashboardControlState, app.state.dashboard_controls) + + +def _store(app: FastAPI) -> DuckDBStore: + return cast(DuckDBStore, app.state.store) + + +def _audit_repository(app: FastAPI) -> AuditRepository | None: + repository = getattr(app.state, "audit_repository", None) + return repository if isinstance(repository, AuditRepository) else None + + +def _runtime_repository(app: FastAPI) -> RuntimeStateRepository | None: + repository = getattr(app.state, "runtime_state_repository", None) + return repository if isinstance(repository, RuntimeStateRepository) else None + + +def _open_trade_count(store: DuckDBStore) -> int: + with store.connect() as conn: + row = conn.execute(""" + SELECT COUNT(*) + FROM trades + WHERE finished_at IS NULL + """).fetchone() + return int(row[0]) if row is not None else 0 + + +def _latest_balances(store: DuckDBStore) -> dict[str, Any] | None: + with store.connect() as conn: + row = conn.execute(""" + SELECT balances + FROM portfolio_snapshots + ORDER BY snapshot_at DESC + LIMIT 1 + """).fetchone() + + if row is None or row[0] is None: + return None + raw_balances = row[0] + if isinstance(raw_balances, str): + return {"raw": raw_balances} + return {"raw": str(raw_balances)} + + +def _record_audit( + app: FastAPI, + *, + event_type: str, + decision: str, + payload: dict[str, Any] | None = None, +) -> None: + repository = _audit_repository(app) + if repository is None: + return + repository.insert( + AuditRecord( + occurred_at=datetime.now(UTC), + actor="runtime", + event_type=event_type, + decision=decision, + payload=payload, + correlation_id=None, + ) + ) + + +async def _run_startup_reconciler(app: FastAPI) -> None: + reconciler = getattr(app.state, "startup_reconciler", None) + if reconciler is None: + return + + reconcile_member = getattr(reconciler, "reconcile_open_trades", None) + if reconcile_member is None or not callable(reconcile_member): + return + + result = reconcile_member() + if inspect.isawaitable(result): + await result + + +def persist_runtime_snapshot(app: FastAPI, *, note: str | None = None) -> RuntimeStateRecord | None: + repository = _runtime_repository(app) + if repository is None: + return None + + controls = _controls(app) + store = _store(app) + snapshot = RuntimeStateRecord( + snapshot_at=datetime.now(UTC), + is_running=controls.is_running, + kill_switch_active=controls.kill_switch.is_active, + kill_switch_reason=controls.kill_switch.reason, + open_trade_count=_open_trade_count(store), + last_known_balances=_latest_balances(store), + note=note, + ) + repository.insert(snapshot) + return snapshot + + +async def restore_runtime_state(app: FastAPI) -> RuntimeRecoveryReport: + ctl = _controls(app) + store = _store(app) + repo = _runtime_repository(app) + + restored_from_snapshot = False + snapshot_at: str | None = None + + latest = repo.latest() if repo is not None else None + if latest is not None: + restored_from_snapshot = True + snapshot_at = latest.snapshot_at.isoformat() + ctl.is_running = latest.is_running + if latest.kill_switch_active: + r = latest.kill_switch_reason or "recovered" + ctl.kill_switch.activate(reason=r) + else: + ctl.kill_switch.deactivate() + ctl.mark_updated() + + open_trades = _open_trade_count(store) + restart_guard_active = False + if open_trades > 0: + ctl.is_running = False + if not ctl.kill_switch.is_active: + ctl.kill_switch.activate(reason="recovery_open_trades_detected") + ctl.mark_updated() + restart_guard_active = True + + report = RuntimeRecoveryReport( + restored_from_snapshot=restored_from_snapshot, + snapshot_at=snapshot_at, + open_trades_detected=open_trades, + restart_guard_active=restart_guard_active, + ) + app.state.recovery_report = report + + _record_audit( + app, + event_type="runtime.startup_recovery", + decision="applied", + payload={ + "restored_from_snapshot": restored_from_snapshot, + "open_trades_detected": open_trades, + "restart_guard_active": restart_guard_active, + }, + ) + + await _run_startup_reconciler(app) + + return report + + +async def drain_background_workers(app: FastAPI) -> None: + workers: list[object] = [] + + declared = getattr(app.state, "background_workers", None) + if isinstance(declared, list): + workers.extend(declared) + + for attr_name in ("execution_writer", "opportunity_writer", "snapshot_writer"): + worker = getattr(app.state, attr_name, None) + if worker is not None: + workers.append(worker) + + seen: set[int] = set() + for worker in workers: + worker_id = id(worker) + if worker_id in seen: + continue + seen.add(worker_id) + + stop_member = getattr(worker, "stop", None) + if stop_member is None or not callable(stop_member): + continue + + result = stop_member() + if inspect.isawaitable(result): + await result + + +async def graceful_shutdown(app: FastAPI) -> None: + controls = _controls(app) + controls.is_running = False + controls.mark_updated() + + _record_audit( + app, + event_type="runtime.shutdown", + decision="initiated", + payload={"execution_status": "stopped"}, + ) + + await drain_background_workers(app) + persist_runtime_snapshot(app, note="graceful_shutdown") diff --git a/build/lib/arbitrade/storage/__init__.py b/build/lib/arbitrade/storage/__init__.py new file mode 100644 index 0000000..76541db --- /dev/null +++ b/build/lib/arbitrade/storage/__init__.py @@ -0,0 +1 @@ +"""Storage helpers.""" diff --git a/build/lib/arbitrade/storage/db.py b/build/lib/arbitrade/storage/db.py new file mode 100644 index 0000000..63ce1bf --- /dev/null +++ b/build/lib/arbitrade/storage/db.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import duckdb +import structlog + +from arbitrade.config.settings import Settings + +_LOG = structlog.get_logger(__name__) + +SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TIMESTAMP DEFAULT current_timestamp +); + +CREATE TABLE IF NOT EXISTS opportunities ( + id UUID DEFAULT uuid(), + detected_at TIMESTAMP NOT NULL, + cycle VARCHAR NOT NULL, + gross_pct DOUBLE, + net_pct DOUBLE, + est_profit DOUBLE, + executed BOOLEAN DEFAULT FALSE +); + +CREATE TABLE IF NOT EXISTS trades ( + id UUID DEFAULT uuid(), + trade_ref VARCHAR NOT NULL, + started_at TIMESTAMP NOT NULL, + finished_at TIMESTAMP, + status VARCHAR NOT NULL, + realized_pnl DOUBLE, + estimated_pnl DOUBLE, + capital_used DOUBLE, + cycle VARCHAR, + leg_count INTEGER +); + +CREATE TABLE IF NOT EXISTS orders ( + id UUID DEFAULT uuid(), + trade_ref VARCHAR NOT NULL, + order_ref VARCHAR NOT NULL, + leg_index INTEGER NOT NULL, + pair VARCHAR NOT NULL, + side VARCHAR NOT NULL, + volume DOUBLE NOT NULL, + user_ref INTEGER, + status VARCHAR, + filled_volume DOUBLE, + avg_price DOUBLE, + raw_response JSON, + recorded_at TIMESTAMP NOT NULL +); + +CREATE TABLE IF NOT EXISTS pnl_events ( + id UUID DEFAULT uuid(), + trade_ref VARCHAR NOT NULL, + recorded_at TIMESTAMP NOT NULL, + kind VARCHAR NOT NULL, + pnl_usd DOUBLE NOT NULL, + source VARCHAR NOT NULL +); + +CREATE TABLE IF NOT EXISTS portfolio_snapshots ( + snapshot_at TIMESTAMP NOT NULL, + balances JSON, + total_value_usd DOUBLE +); + +CREATE TABLE IF NOT EXISTS market_snapshots ( + snapshot_at TIMESTAMP NOT NULL, + symbol VARCHAR NOT NULL, + source VARCHAR NOT NULL, + payload JSON NOT NULL, + latency_ms DOUBLE +); + +CREATE TABLE IF NOT EXISTS audit_events ( + id UUID DEFAULT uuid(), + occurred_at TIMESTAMP NOT NULL, + actor VARCHAR NOT NULL, + event_type VARCHAR NOT NULL, + decision VARCHAR NOT NULL, + payload JSON, + correlation_id VARCHAR +); + +CREATE TABLE IF NOT EXISTS runtime_state_snapshots ( + snapshot_at TIMESTAMP NOT NULL, + is_running BOOLEAN NOT NULL, + kill_switch_active BOOLEAN NOT NULL, + kill_switch_reason VARCHAR, + open_trade_count INTEGER NOT NULL, + last_known_balances JSON, + note VARCHAR +); +""" + + +class DuckDBStore: + def __init__(self, settings: Settings) -> None: + self._db_path = Path(settings.duckdb_path) + self._db_path.parent.mkdir(parents=True, exist_ok=True) + self._use_memory_fallback = False + + @contextmanager + def connect(self) -> Iterator[duckdb.DuckDBPyConnection]: + try: + conn = duckdb.connect(str(self._db_path)) + except duckdb.IOException: + if not self._use_memory_fallback: + _LOG.warning( + "duckdb_path_unavailable_falling_back_to_memory", path=str(self._db_path) + ) + self._use_memory_fallback = True + conn = duckdb.connect(":memory:") + try: + yield conn + finally: + conn.close() + + def migrate(self) -> None: + with self.connect() as conn: + conn.execute(SCHEMA_SQL) diff --git a/build/lib/arbitrade/storage/executions.py b/build/lib/arbitrade/storage/executions.py new file mode 100644 index 0000000..4262091 --- /dev/null +++ b/build/lib/arbitrade/storage/executions.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import asyncio + +import structlog + +from arbitrade.storage.repositories import ( + OrderRecord, + OrderRepository, + PnLRecord, + PnLRepository, + TradeRecord, + TradeRepository, +) + +_LOG = structlog.get_logger(__name__) + + +class AsyncExecutionWriter: + def __init__( + self, + trade_repository: TradeRepository, + order_repository: OrderRepository, + pnl_repository: PnLRepository, + max_queue_size: int = 50_000, + ) -> None: + self._trade_repository = trade_repository + self._order_repository = order_repository + self._pnl_repository = pnl_repository + self._queue: asyncio.Queue[TradeRecord | OrderRecord | PnLRecord] = asyncio.Queue( + maxsize=max_queue_size + ) + self._task: asyncio.Task[None] | None = None + self._stop = asyncio.Event() + + async def start(self) -> None: + if self._task is None or self._task.done(): + self._stop.clear() + self._task = asyncio.create_task(self._run(), name="execution-writer") + + async def stop(self) -> None: + self._stop.set() + if self._task is not None: + await self._task + + async def enqueue(self, record: TradeRecord | OrderRecord | PnLRecord) -> None: + await self._queue.put(record) + + async def _run(self) -> None: + while not (self._stop.is_set() and self._queue.empty()): + try: + record = await asyncio.wait_for(self._queue.get(), timeout=0.5) + except TimeoutError: + continue + + try: + if isinstance(record, TradeRecord): + self._trade_repository.insert(record) + elif isinstance(record, OrderRecord): + self._order_repository.insert(record) + else: + self._pnl_repository.insert(record) + except Exception as exc: + _LOG.error("execution_write_failed", error=str(exc)) + finally: + self._queue.task_done() diff --git a/build/lib/arbitrade/storage/market_snapshots.py b/build/lib/arbitrade/storage/market_snapshots.py new file mode 100644 index 0000000..c87c529 --- /dev/null +++ b/build/lib/arbitrade/storage/market_snapshots.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import asyncio +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +import structlog + +from arbitrade.storage.repositories import MarketSnapshotRecord, MarketSnapshotRepository + +_LOG = structlog.get_logger(__name__) + + +@dataclass(slots=True) +class MarketSnapshot: + snapshot_at: datetime + symbol: str + source: str + payload: dict[str, Any] + latency_ms: float | None + + +class AsyncMarketSnapshotWriter: + def __init__(self, repository: MarketSnapshotRepository, max_queue_size: int = 50_000) -> None: + self._repository = repository + self._queue: asyncio.Queue[MarketSnapshot] = asyncio.Queue(maxsize=max_queue_size) + self._task: asyncio.Task[None] | None = None + self._stop = asyncio.Event() + + async def start(self) -> None: + if self._task is None or self._task.done(): + self._stop.clear() + self._task = asyncio.create_task(self._run(), name="market-snapshot-writer") + + async def stop(self) -> None: + self._stop.set() + if self._task is not None: + await self._task + + async def enqueue(self, snapshot: MarketSnapshot) -> None: + await self._queue.put(snapshot) + + async def _run(self) -> None: + while not (self._stop.is_set() and self._queue.empty()): + try: + item = await asyncio.wait_for(self._queue.get(), timeout=0.5) + except TimeoutError: + continue + + try: + self._repository.insert( + MarketSnapshotRecord( + snapshot_at=item.snapshot_at, + symbol=item.symbol, + source=item.source, + payload=item.payload, + latency_ms=item.latency_ms, + ) + ) + except Exception as exc: + _LOG.error("market_snapshot_write_failed", error=str(exc), symbol=item.symbol) + finally: + self._queue.task_done() diff --git a/build/lib/arbitrade/storage/opportunities.py b/build/lib/arbitrade/storage/opportunities.py new file mode 100644 index 0000000..032b23a --- /dev/null +++ b/build/lib/arbitrade/storage/opportunities.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import asyncio + +import structlog + +from arbitrade.detection.engine import OpportunityEvent +from arbitrade.storage.repositories import OpportunityRecord, OpportunityRepository + +_LOG = structlog.get_logger(__name__) + + +class AsyncOpportunityWriter: + def __init__(self, repository: OpportunityRepository, max_queue_size: int = 50_000) -> None: + self._repository = repository + self._queue: asyncio.Queue[OpportunityEvent] = asyncio.Queue(maxsize=max_queue_size) + self._task: asyncio.Task[None] | None = None + self._stop = asyncio.Event() + + async def start(self) -> None: + if self._task is None or self._task.done(): + self._stop.clear() + self._task = asyncio.create_task(self._run(), name="opportunity-writer") + + async def stop(self) -> None: + self._stop.set() + if self._task is not None: + await self._task + + async def enqueue(self, event: OpportunityEvent) -> None: + await self._queue.put(event) + + async def _run(self) -> None: + while not (self._stop.is_set() and self._queue.empty()): + try: + event = await asyncio.wait_for(self._queue.get(), timeout=0.5) + except TimeoutError: + continue + + try: + self._repository.insert( + OpportunityRecord( + detected_at=event.detected_at, + cycle=event.cycle, + gross_pct=event.gross_pct, + net_pct=event.net_pct, + est_profit=event.est_profit, + ) + ) + except Exception as exc: + _LOG.error( + "opportunity_write_failed", + error=str(exc), + cycle=event.cycle, + updated_pair=event.updated_pair, + ) + finally: + self._queue.task_done() diff --git a/build/lib/arbitrade/storage/repositories.py b/build/lib/arbitrade/storage/repositories.py new file mode 100644 index 0000000..5977f61 --- /dev/null +++ b/build/lib/arbitrade/storage/repositories.py @@ -0,0 +1,378 @@ +from __future__ import annotations + +from dataclasses import dataclass +from datetime import datetime +from typing import Any + +import orjson + +from arbitrade.storage.db import DuckDBStore + + +@dataclass(slots=True) +class MarketSnapshotRecord: + snapshot_at: datetime + symbol: str + source: str + payload: dict[str, Any] + latency_ms: float | None + + +@dataclass(slots=True) +class OpportunityRecord: + detected_at: datetime + cycle: str + gross_pct: float + net_pct: float + est_profit: float + executed: bool = False + + +@dataclass(slots=True) +class TradeRecord: + trade_ref: str + started_at: datetime + finished_at: datetime | None + status: str + realized_pnl: float | None + estimated_pnl: float | None + capital_used: float | None + cycle: str | None = None + leg_count: int | None = None + + +@dataclass(slots=True) +class OrderRecord: + trade_ref: str + order_ref: str + leg_index: int + pair: str + side: str + volume: float + user_ref: int | None + status: str | None + filled_volume: float | None + avg_price: float | None + raw_response: dict[str, Any] + recorded_at: datetime + + +@dataclass(slots=True) +class PnLRecord: + trade_ref: str + recorded_at: datetime + kind: str + pnl_usd: float + source: str + + +@dataclass(slots=True) +class AuditRecord: + occurred_at: datetime + actor: str + event_type: str + decision: str + payload: dict[str, Any] | None = None + correlation_id: str | None = None + + +@dataclass(slots=True) +class RuntimeStateRecord: + snapshot_at: datetime + is_running: bool + kill_switch_active: bool + kill_switch_reason: str | None + open_trade_count: int + last_known_balances: dict[str, Any] | None = None + note: str | None = None + + +class MarketSnapshotRepository: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def insert(self, record: MarketSnapshotRecord) -> None: + with self._store.connect() as conn: + conn.execute( + """ + INSERT INTO market_snapshots (snapshot_at, symbol, source, payload, latency_ms) + VALUES (?, ?, ?, ?, ?) + """, + [ + record.snapshot_at, + record.symbol, + record.source, + orjson.dumps(record.payload).decode("utf-8"), + record.latency_ms, + ], + ) + + +class OpportunityRepository: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def insert(self, record: OpportunityRecord) -> None: + with self._store.connect() as conn: + conn.execute( + """ + INSERT INTO opportunities ( + detected_at, + cycle, + gross_pct, + net_pct, + est_profit, + executed + ) + VALUES (?, ?, ?, ?, ?, ?) + """, + [ + record.detected_at, + record.cycle, + record.gross_pct, + record.net_pct, + record.est_profit, + record.executed, + ], + ) + + +class TradeRepository: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def insert(self, record: TradeRecord) -> None: + with self._store.connect() as conn: + conn.execute( + """ + INSERT INTO trades ( + trade_ref, + started_at, + finished_at, + status, + realized_pnl, + estimated_pnl, + capital_used, + cycle, + leg_count + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + record.trade_ref, + record.started_at, + record.finished_at, + record.status, + record.realized_pnl, + record.estimated_pnl, + record.capital_used, + record.cycle, + record.leg_count, + ], + ) + + +class OrderRepository: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def insert(self, record: OrderRecord) -> None: + with self._store.connect() as conn: + conn.execute( + """ + INSERT INTO orders ( + trade_ref, + order_ref, + leg_index, + pair, + side, + volume, + user_ref, + status, + filled_volume, + avg_price, + raw_response, + recorded_at + ) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + [ + record.trade_ref, + record.order_ref, + record.leg_index, + record.pair, + record.side, + record.volume, + record.user_ref, + record.status, + record.filled_volume, + record.avg_price, + orjson.dumps(record.raw_response).decode("utf-8"), + record.recorded_at, + ], + ) + + +class PnLRepository: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def insert(self, record: PnLRecord) -> None: + with self._store.connect() as conn: + conn.execute( + """ + INSERT INTO pnl_events ( + trade_ref, + recorded_at, + kind, + pnl_usd, + source + ) + VALUES (?, ?, ?, ?, ?) + """, + [ + record.trade_ref, + record.recorded_at, + record.kind, + record.pnl_usd, + record.source, + ], + ) + + +class AuditRepository: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def insert(self, record: AuditRecord) -> None: + with self._store.connect() as conn: + conn.execute( + """ + INSERT INTO audit_events ( + occurred_at, + actor, + event_type, + decision, + payload, + correlation_id + ) + VALUES (?, ?, ?, ?, ?, ?) + """, + [ + record.occurred_at, + record.actor, + record.event_type, + record.decision, + ( + None + if record.payload is None + else orjson.dumps(record.payload).decode("utf-8") + ), + record.correlation_id, + ], + ) + + def list_recent(self, *, limit: int = 25) -> list[AuditRecord]: + with self._store.connect() as conn: + rows = conn.execute( + """ + SELECT occurred_at, actor, event_type, decision, payload, correlation_id + FROM audit_events + ORDER BY occurred_at DESC + LIMIT ? + """, + [limit], + ).fetchall() + + records: list[AuditRecord] = [] + for row in rows: + payload: dict[str, Any] | None = None + raw_payload = row[4] + if isinstance(raw_payload, str) and raw_payload: + decoded = orjson.loads(raw_payload) + if isinstance(decoded, dict): + payload = {str(k): decoded[k] for k in decoded} + + records.append( + AuditRecord( + occurred_at=row[0], + actor=str(row[1]), + event_type=str(row[2]), + decision=str(row[3]), + payload=payload, + correlation_id=str(row[5]) if row[5] is not None else None, + ) + ) + + return records + + +class RuntimeStateRepository: + def __init__(self, store: DuckDBStore) -> None: + self._store = store + + def insert(self, record: RuntimeStateRecord) -> None: + with self._store.connect() as conn: + conn.execute( + """ + INSERT INTO runtime_state_snapshots ( + snapshot_at, + is_running, + kill_switch_active, + kill_switch_reason, + open_trade_count, + last_known_balances, + note + ) + VALUES (?, ?, ?, ?, ?, ?, ?) + """, + [ + record.snapshot_at, + record.is_running, + record.kill_switch_active, + record.kill_switch_reason, + record.open_trade_count, + ( + None + if record.last_known_balances is None + else orjson.dumps(record.last_known_balances).decode("utf-8") + ), + record.note, + ], + ) + + def latest(self) -> RuntimeStateRecord | None: + with self._store.connect() as conn: + row = conn.execute(""" + SELECT + snapshot_at, + is_running, + kill_switch_active, + kill_switch_reason, + open_trade_count, + last_known_balances, + note + FROM runtime_state_snapshots + ORDER BY snapshot_at DESC + LIMIT 1 + """).fetchone() + + if row is None: + return None + + balances: dict[str, Any] | None = None + raw_balances = row[5] + if isinstance(raw_balances, str) and raw_balances: + decoded = orjson.loads(raw_balances) + if isinstance(decoded, dict): + balances = {str(key): decoded[key] for key in decoded} + + return RuntimeStateRecord( + snapshot_at=row[0], + is_running=bool(row[1]), + kill_switch_active=bool(row[2]), + kill_switch_reason=str(row[3]) if row[3] is not None else None, + open_trade_count=int(row[4]), + last_known_balances=balances, + note=str(row[6]) if row[6] is not None else None, + ) diff --git a/build/lib/arbitrade/strategy/__init__.py b/build/lib/arbitrade/strategy/__init__.py new file mode 100644 index 0000000..6d1952d --- /dev/null +++ b/build/lib/arbitrade/strategy/__init__.py @@ -0,0 +1,5 @@ +"""Experimental strategy modules.""" + +from arbitrade.strategy.stat_arb import StatArbExperiment, StatArbExperimentConfig, StatArbSignal + +__all__ = ["StatArbExperiment", "StatArbExperimentConfig", "StatArbSignal"] diff --git a/build/lib/arbitrade/strategy/stat_arb.py b/build/lib/arbitrade/strategy/stat_arb.py new file mode 100644 index 0000000..78d574d --- /dev/null +++ b/build/lib/arbitrade/strategy/stat_arb.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +from collections import deque +from dataclasses import dataclass +from datetime import UTC, datetime +from statistics import fmean, pstdev +from typing import Literal + + +@dataclass(frozen=True, slots=True) +class StatArbExperimentConfig: + pair_a: str + pair_b: str + lookback_window: int = 120 + entry_zscore: float = 2.0 + exit_zscore: float = 0.5 + max_holding_seconds: float = 900.0 + + +@dataclass(frozen=True, slots=True) +class StatArbSignal: + action: Literal[ + "warmup", + "hold", + "enter_long_spread", + "enter_short_spread", + "exit_position", + ] + observed_at: datetime + spread: float + zscore: float | None + position: Literal["long", "short", "flat"] + + +class StatArbExperiment: + """Simple mean-reversion experiment scaffold behind feature flags.""" + + def __init__(self, config: StatArbExperimentConfig) -> None: + if config.lookback_window < 2: + raise ValueError("lookback_window must be >= 2") + if config.entry_zscore <= 0.0: + raise ValueError("entry_zscore must be > 0") + if config.exit_zscore < 0.0: + raise ValueError("exit_zscore must be >= 0") + if config.entry_zscore <= config.exit_zscore: + raise ValueError("entry_zscore must be > exit_zscore") + if config.max_holding_seconds <= 0.0: + raise ValueError("max_holding_seconds must be > 0") + + self._config = config + self._spreads: deque[float] = deque(maxlen=config.lookback_window) + self._position: Literal["long", "short", "flat"] = "flat" + self._position_opened_at: datetime | None = None + + @property + def config(self) -> StatArbExperimentConfig: + return self._config + + def reset(self) -> None: + self._spreads.clear() + self._position = "flat" + self._position_opened_at = None + + def observe( + self, + *, + price_a: float, + price_b: float, + observed_at: datetime, + ) -> StatArbSignal: + if price_a <= 0.0 or price_b <= 0.0: + raise ValueError("prices must be > 0") + + at = observed_at.astimezone(UTC) + spread = price_a - price_b + self._spreads.append(spread) + + if len(self._spreads) < self._config.lookback_window: + return StatArbSignal( + action="warmup", + observed_at=at, + spread=spread, + zscore=None, + position=self._position, + ) + + mean_spread = fmean(self._spreads) + std_spread = pstdev(self._spreads) + if std_spread == 0.0: + return StatArbSignal( + action="hold", + observed_at=at, + spread=spread, + zscore=0.0, + position=self._position, + ) + + zscore = (spread - mean_spread) / std_spread + + if self._position == "flat": + if zscore >= self._config.entry_zscore: + self._position = "short" + self._position_opened_at = at + return StatArbSignal( + action="enter_short_spread", + observed_at=at, + spread=spread, + zscore=zscore, + position=self._position, + ) + if zscore <= -self._config.entry_zscore: + self._position = "long" + self._position_opened_at = at + return StatArbSignal( + action="enter_long_spread", + observed_at=at, + spread=spread, + zscore=zscore, + position=self._position, + ) + return StatArbSignal( + action="hold", + observed_at=at, + spread=spread, + zscore=zscore, + position=self._position, + ) + + assert self._position_opened_at is not None + held_seconds = (at - self._position_opened_at).total_seconds() + should_exit = abs(zscore) <= self._config.exit_zscore + if held_seconds >= self._config.max_holding_seconds: + should_exit = True + + if should_exit: + self._position = "flat" + self._position_opened_at = None + return StatArbSignal( + action="exit_position", + observed_at=at, + spread=spread, + zscore=zscore, + position=self._position, + ) + + return StatArbSignal( + action="hold", + observed_at=at, + spread=spread, + zscore=zscore, + position=self._position, + ) diff --git a/build/lib/arbitrade/web/templates/backtesting.html b/build/lib/arbitrade/web/templates/backtesting.html new file mode 100644 index 0000000..7519e75 --- /dev/null +++ b/build/lib/arbitrade/web/templates/backtesting.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} {% block title %}{{ title }}{% endblock %} {% block +content %} +
+
+

Backtesting

+

+ Replay controls, run status, and recent summary reports. +

+
+
+ Dashboard +
+
+ +
+ {% include "partials/backtesting_panel.html" %} +
+{% endblock %} diff --git a/build/lib/arbitrade/web/templates/base.html b/build/lib/arbitrade/web/templates/base.html new file mode 100644 index 0000000..4155859 --- /dev/null +++ b/build/lib/arbitrade/web/templates/base.html @@ -0,0 +1,148 @@ + + + + + + {% block title %}{{ title or "Arbitrade" }}{% endblock %} + + {% block head_scripts %}{% endblock %} + + {% block extra_style %}{% endblock %} + + +
+ {% block content %}{% endblock %} +
+ {% block scripts %}{% endblock %} + + diff --git a/build/lib/arbitrade/web/templates/dashboard.html b/build/lib/arbitrade/web/templates/dashboard.html new file mode 100644 index 0000000..b6be1fa --- /dev/null +++ b/build/lib/arbitrade/web/templates/dashboard.html @@ -0,0 +1,180 @@ +{% extends "base.html" %} {% block title %}{{ title }}{% endblock %} {% block +head_scripts %} + + +{% endblock %} {% block main_class %}shell{% endblock %} {% block content %} +
+
+

Arbitrade Dashboard

+

Live execution, P&L, and system state.

+
+
+ Refresh metrics + Health + Backtesting +
+
+ +
+ {% include "partials/metrics.html" %} +
+ +
+ {% include "partials/overview.html" %} +
+ +
+ {% include "partials/controls.html" %} +
+ +
+ {% include "partials/charts.html" %} +
+ +
+ {% include "partials/audit.html" %} +
+{% endblock %} {% block scripts %} + +{% endblock %} diff --git a/build/lib/arbitrade/web/templates/health.html b/build/lib/arbitrade/web/templates/health.html new file mode 100644 index 0000000..aa86fd0 --- /dev/null +++ b/build/lib/arbitrade/web/templates/health.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block content %} +
+

Arbitrade Bootstrap Complete

+

Status: {{ status }}

+

UTC: {{ time }}

+

+ Health JSON: + refresh +

+ 
{"status":"ok","service":"arbitrade"}
+
+{% endblock %} diff --git a/build/lib/arbitrade/web/templates/partials/audit.html b/build/lib/arbitrade/web/templates/partials/audit.html new file mode 100644 index 0000000..2aa55db --- /dev/null +++ b/build/lib/arbitrade/web/templates/partials/audit.html @@ -0,0 +1,37 @@ +
+
Audit Trail
+
Generated {{ generated_at }}
+ +
+ + + + + + + + + + + + + {% if entries %} + {% for entry in entries %} + + + + + + + + + {% endfor %} + {% else %} + + + + {% endif %} + +
TimeActorEventDecisionPayloadCorrelation
{{ entry.occurred_at }}{{ entry.actor }}{{ entry.event_type }}{{ entry.decision }}{{ entry.payload }}{{ entry.correlation_id }}
No audit entries yet.
+
+
diff --git a/build/lib/arbitrade/web/templates/partials/backtesting_panel.html b/build/lib/arbitrade/web/templates/partials/backtesting_panel.html new file mode 100644 index 0000000..15b665d --- /dev/null +++ b/build/lib/arbitrade/web/templates/partials/backtesting_panel.html @@ -0,0 +1,142 @@ +
+
+
+
Run Status
+
{{ status }}
+
{{ message }}
+
+
+
Latest Report
+ {% if latest_report %} +
Run at {{ latest_report.run_at }}
+
Events: {{ latest_report.events_path }}
+
+ Processed: {{ latest_report.report.processed_events }} +
+
+ Opportunities: {{ latest_report.report.opportunities_seen }} +
+
Trades: {{ latest_report.report.trades_executed }}
+
+ Realized P&L: {{ + '%.4f'|format(latest_report.report.realized_pnl_usd) }} USD +
+
+ Max drawdown: {{ '%.4f'|format(latest_report.report.max_drawdown_usd) }} + USD +
+ {% else %} +
No runs yet.
+ {% endif %} +
+
+ +
+
Run Backtest
+
+ + + + + + + + + +
+
+ +
+
Recent Runs
+ {% if recent_reports %} {% for item in recent_reports %} +
+ {{ item.run_at }} | {{ item.events_path }} | trades={{ + item.report.trades_executed }} | pnl={{ + '%.4f'|format(item.report.realized_pnl_usd) }} USD +
+ {% endfor %} {% else %} +
No recent reports yet.
+ {% endif %} +
+
diff --git a/build/lib/arbitrade/web/templates/partials/charts.html b/build/lib/arbitrade/web/templates/partials/charts.html new file mode 100644 index 0000000..91c51df --- /dev/null +++ b/build/lib/arbitrade/web/templates/partials/charts.html @@ -0,0 +1,37 @@ +
+
+
+
Opportunity Trend
+
Recent opportunities from DuckDB. Updated {{ generated_at }}
+
+ +
+ +
+
+ {% if has_chart_data %} + + + {% else %} +
No opportunity data yet.
+ {% endif %} +
+
+
\ No newline at end of file diff --git a/build/lib/arbitrade/web/templates/partials/controls.html b/build/lib/arbitrade/web/templates/partials/controls.html new file mode 100644 index 0000000..a5f968b --- /dev/null +++ b/build/lib/arbitrade/web/templates/partials/controls.html @@ -0,0 +1,171 @@ +
+
+
+
Runtime Status
+
{{ execution_status }}
+
Updated {{ updated_at }}
+
+
+
Kill Switch
+
{{ kill_switch_status }}
+
Reason {{ kill_switch_reason }}
+
+
+
Config Snapshot
+
Paper trading: {{ paper_trading_mode }}
+
Trade capital: {{ trade_capital_usd }}
+
Max trade capital: {{ max_trade_capital_usd }}
+
Max concurrent trades: {{ max_concurrent_trades }}
+
Tradable pairs: {{ tradable_pairs_display }}
+
Strategy mode: {{ strategy_mode }}
+
Profit threshold: {{ strategy_profit_threshold }}
+
Max depth levels: {{ strategy_max_depth_levels }}
+
+
+
Alerting
+
Status: {{ alerts_enabled }}
+
Channels: {{ alerts_channels }}
+
Min severity: {{ alerts_min_severity }}
+
Dedup window: {{ alerts_dedup_seconds }}s
+
Last result: {{ alerts_last_result }}
+
Last attempted: {{ alerts_last_attempted_at }}
+
Last success: {{ alerts_last_success_at }}
+
Last event: {{ alerts_last_event_title }}
+
Last error: {{ alerts_last_error }}
+ {% if alerts_last_channel_results %} {% for item in + alerts_last_channel_results %} +
{{ item }}
+ {% endfor %} {% endif %} +
+
+ +
+
+
Execution Controls
+
+
+ +
+
+ +
+
+ + +
+
+
+
+
Edit Config
+
+ + + + + + + + + +
+
+
+
diff --git a/build/lib/arbitrade/web/templates/partials/metrics.html b/build/lib/arbitrade/web/templates/partials/metrics.html new file mode 100644 index 0000000..1748e29 --- /dev/null +++ b/build/lib/arbitrade/web/templates/partials/metrics.html @@ -0,0 +1,31 @@ +
+
+
+
Realized P&L
+
{{ realized_pnl }}
+
+
+
Win Rate
+
{{ win_rate }}
+
+
+
Avg Trade Duration
+
{{ avg_trade_duration }}
+
+
+
Opportunities / Min
+
{{ opportunities_per_minute }}
+
+
+
Fill Rate
+
{{ fill_rate }}
+
+
+
Latency p50 / p95 / p99
+
+ {{ latency_p50 }} | {{ latency_p95 }} | {{ latency_p99 }} +
+
+
+
Updated {{ generated_at }}
+
diff --git a/build/lib/arbitrade/web/templates/partials/overview.html b/build/lib/arbitrade/web/templates/partials/overview.html new file mode 100644 index 0000000..6787b51 --- /dev/null +++ b/build/lib/arbitrade/web/templates/partials/overview.html @@ -0,0 +1,67 @@ +
+
+
+
Status
+
{{ status }}
+
+
+
Balances
+
{{ balances }}
+
+
+
Open Trades
+
{{ open_trade_count }}
+
+
+
Realized P&L
+
{{ realized_pnl_total }}
+
+
+ +
+
+
Open Trades
+
    + {% for trade in open_trades %} +
  • + {{ trade.trade_ref }} - {{ trade.status }} - {{ trade.cycle }} - {{ + trade.started_at }} +
    • + {% else %} +
  • No open trades.
    • + {% endfor %} +
+
+
+
Balances Snapshot
+
+ {{ balances }} +
+
Total value {{ total_value }}
+
+
+
Opportunity Feed
+
    + {% for opp in opportunities %} +
  • + {{ opp.cycle }} - {{ opp.net_pct }} - {{ opp.est_profit }} - {{ + opp.detected_at }} +
    • + {% else %} +
  • No opportunities.
    • + {% endfor %} +
+
+
+ +
Updated {{ generated_at }}
+
diff --git a/dist/arbitrade-0.1.0-py3-none-any.whl b/dist/arbitrade-0.1.0-py3-none-any.whl new file mode 100644 index 0000000..f88f77e Binary files /dev/null and b/dist/arbitrade-0.1.0-py3-none-any.whl differ diff --git a/pyproject.toml b/pyproject.toml index fe7512e..519d9b3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,13 @@ arbitrade-bench-detection = "arbitrade.detection.benchmark:main" [tool.setuptools] package-dir = {"" = "src"} +include-package-data = true + +[tool.setuptools.package-data] +arbitrade = [ + "web/templates/*.html", + "web/templates/partials/*.html", +] [tool.setuptools.packages.find] where = ["src"] diff --git a/src/arbitrade/api/routes.py b/src/arbitrade/api/routes.py index ff5c36d..074ac79 100644 --- a/src/arbitrade/api/routes.py +++ b/src/arbitrade/api/routes.py @@ -4,6 +4,7 @@ import json from asyncio import Lock from collections.abc import AsyncIterator from datetime import UTC, datetime +from importlib import resources from pathlib import Path from typing import cast from urllib.parse import parse_qs @@ -22,9 +23,31 @@ from arbitrade.storage.repositories import AuditRecord, AuditRepository router = APIRouter(dependencies=[Depends(require_dashboard_auth)]) public_router = APIRouter() -templates = Jinja2Templates( - directory=str(Path(__file__).resolve().parents[3] / "web" / "templates") -) + + +def _resolve_templates_directory() -> str: + # Support source layout, Docker runtime (/app), and installed package data. + source_layout_path = Path( + __file__).resolve().parents[3] / "web" / "templates" + if source_layout_path.is_dir(): + return str(source_layout_path) + + docker_runtime_path = Path.cwd() / "web" / "templates" + if docker_runtime_path.is_dir(): + return str(docker_runtime_path) + + try: + package_path = resources.files( + "arbitrade").joinpath("web", "templates") + if package_path.is_dir(): + return str(package_path) + except (ModuleNotFoundError, AttributeError): + pass + + return str(source_layout_path) + + +templates = Jinja2Templates(directory=_resolve_templates_directory()) _BACKTEST_ROOT = Path(__file__).resolve().parents[3] _BACKTEST_RUN_LOCK = Lock() diff --git a/src/arbitrade/web/templates/backtesting.html b/src/arbitrade/web/templates/backtesting.html new file mode 100644 index 0000000..7519e75 --- /dev/null +++ b/src/arbitrade/web/templates/backtesting.html @@ -0,0 +1,24 @@ +{% extends "base.html" %} {% block title %}{{ title }}{% endblock %} {% block +content %} +
+
+

Backtesting

+

+ Replay controls, run status, and recent summary reports. +

+
+
+ Dashboard +
+
+ +
+ {% include "partials/backtesting_panel.html" %} +
+{% endblock %} diff --git a/src/arbitrade/web/templates/base.html b/src/arbitrade/web/templates/base.html new file mode 100644 index 0000000..4155859 --- /dev/null +++ b/src/arbitrade/web/templates/base.html @@ -0,0 +1,148 @@ + + + + + + {% block title %}{{ title or "Arbitrade" }}{% endblock %} + + {% block head_scripts %}{% endblock %} + + {% block extra_style %}{% endblock %} + + +
+ {% block content %}{% endblock %} +
+ {% block scripts %}{% endblock %} + + diff --git a/src/arbitrade/web/templates/dashboard.html b/src/arbitrade/web/templates/dashboard.html new file mode 100644 index 0000000..b6be1fa --- /dev/null +++ b/src/arbitrade/web/templates/dashboard.html @@ -0,0 +1,180 @@ +{% extends "base.html" %} {% block title %}{{ title }}{% endblock %} {% block +head_scripts %} + + +{% endblock %} {% block main_class %}shell{% endblock %} {% block content %} +
+
+

Arbitrade Dashboard

+

Live execution, P&L, and system state.

+
+
+ Refresh metrics + Health + Backtesting +
+
+ +
+ {% include "partials/metrics.html" %} +
+ +
+ {% include "partials/overview.html" %} +
+ +
+ {% include "partials/controls.html" %} +
+ +
+ {% include "partials/charts.html" %} +
+ +
+ {% include "partials/audit.html" %} +
+{% endblock %} {% block scripts %} + +{% endblock %} diff --git a/src/arbitrade/web/templates/health.html b/src/arbitrade/web/templates/health.html new file mode 100644 index 0000000..aa86fd0 --- /dev/null +++ b/src/arbitrade/web/templates/health.html @@ -0,0 +1,14 @@ +{% extends "base.html" %} + +{% block content %} +
+

Arbitrade Bootstrap Complete

+

Status: {{ status }}

+

UTC: {{ time }}

+

+ Health JSON: + refresh +

+ 
{"status":"ok","service":"arbitrade"}
+
+{% endblock %} diff --git a/src/arbitrade/web/templates/partials/audit.html b/src/arbitrade/web/templates/partials/audit.html new file mode 100644 index 0000000..2aa55db --- /dev/null +++ b/src/arbitrade/web/templates/partials/audit.html @@ -0,0 +1,37 @@ +
+
Audit Trail
+
Generated {{ generated_at }}
+ +
+ + + + + + + + + + + + + {% if entries %} + {% for entry in entries %} + + + + + + + + + {% endfor %} + {% else %} + + + + {% endif %} + +
TimeActorEventDecisionPayloadCorrelation
{{ entry.occurred_at }}{{ entry.actor }}{{ entry.event_type }}{{ entry.decision }}{{ entry.payload }}{{ entry.correlation_id }}
No audit entries yet.
+
+
diff --git a/src/arbitrade/web/templates/partials/backtesting_panel.html b/src/arbitrade/web/templates/partials/backtesting_panel.html new file mode 100644 index 0000000..15b665d --- /dev/null +++ b/src/arbitrade/web/templates/partials/backtesting_panel.html @@ -0,0 +1,142 @@ +
+
+
+
Run Status
+
{{ status }}
+
{{ message }}
+
+
+
Latest Report
+ {% if latest_report %} +
Run at {{ latest_report.run_at }}
+
Events: {{ latest_report.events_path }}
+
+ Processed: {{ latest_report.report.processed_events }} +
+
+ Opportunities: {{ latest_report.report.opportunities_seen }} +
+
Trades: {{ latest_report.report.trades_executed }}
+
+ Realized P&L: {{ + '%.4f'|format(latest_report.report.realized_pnl_usd) }} USD +
+
+ Max drawdown: {{ '%.4f'|format(latest_report.report.max_drawdown_usd) }} + USD +
+ {% else %} +
No runs yet.
+ {% endif %} +
+
+ +
+
Run Backtest
+
+ + + + + + + + + +
+
+ +
+
Recent Runs
+ {% if recent_reports %} {% for item in recent_reports %} +
+ {{ item.run_at }} | {{ item.events_path }} | trades={{ + item.report.trades_executed }} | pnl={{ + '%.4f'|format(item.report.realized_pnl_usd) }} USD +
+ {% endfor %} {% else %} +
No recent reports yet.
+ {% endif %} +
+
diff --git a/src/arbitrade/web/templates/partials/charts.html b/src/arbitrade/web/templates/partials/charts.html new file mode 100644 index 0000000..91c51df --- /dev/null +++ b/src/arbitrade/web/templates/partials/charts.html @@ -0,0 +1,37 @@ +
+
+
+
Opportunity Trend
+
Recent opportunities from DuckDB. Updated {{ generated_at }}
+
+ +
+ +
+
+ {% if has_chart_data %} + + + {% else %} +
No opportunity data yet.
+ {% endif %} +
+
+
\ No newline at end of file diff --git a/src/arbitrade/web/templates/partials/controls.html b/src/arbitrade/web/templates/partials/controls.html new file mode 100644 index 0000000..a5f968b --- /dev/null +++ b/src/arbitrade/web/templates/partials/controls.html @@ -0,0 +1,171 @@ +
+
+
+
Runtime Status
+
{{ execution_status }}
+
Updated {{ updated_at }}
+
+
+
Kill Switch
+
{{ kill_switch_status }}
+
Reason {{ kill_switch_reason }}
+
+
+
Config Snapshot
+
Paper trading: {{ paper_trading_mode }}
+
Trade capital: {{ trade_capital_usd }}
+
Max trade capital: {{ max_trade_capital_usd }}
+
Max concurrent trades: {{ max_concurrent_trades }}
+
Tradable pairs: {{ tradable_pairs_display }}
+
Strategy mode: {{ strategy_mode }}
+
Profit threshold: {{ strategy_profit_threshold }}
+
Max depth levels: {{ strategy_max_depth_levels }}
+
+
+
Alerting
+
Status: {{ alerts_enabled }}
+
Channels: {{ alerts_channels }}
+
Min severity: {{ alerts_min_severity }}
+
Dedup window: {{ alerts_dedup_seconds }}s
+
Last result: {{ alerts_last_result }}
+
Last attempted: {{ alerts_last_attempted_at }}
+
Last success: {{ alerts_last_success_at }}
+
Last event: {{ alerts_last_event_title }}
+
Last error: {{ alerts_last_error }}
+ {% if alerts_last_channel_results %} {% for item in + alerts_last_channel_results %} +
{{ item }}
+ {% endfor %} {% endif %} +
+
+ +
+
+
Execution Controls
+
+
+ +
+
+ +
+
+ + +
+
+
+
+
Edit Config
+
+ + + + + + + + + +
+
+
+
diff --git a/src/arbitrade/web/templates/partials/metrics.html b/src/arbitrade/web/templates/partials/metrics.html new file mode 100644 index 0000000..1748e29 --- /dev/null +++ b/src/arbitrade/web/templates/partials/metrics.html @@ -0,0 +1,31 @@ +
+
+
+
Realized P&L
+
{{ realized_pnl }}
+
+
+
Win Rate
+
{{ win_rate }}
+
+
+
Avg Trade Duration
+
{{ avg_trade_duration }}
+
+
+
Opportunities / Min
+
{{ opportunities_per_minute }}
+
+
+
Fill Rate
+
{{ fill_rate }}
+
+
+
Latency p50 / p95 / p99
+
+ {{ latency_p50 }} | {{ latency_p95 }} | {{ latency_p99 }} +
+
+
+
Updated {{ generated_at }}
+
diff --git a/src/arbitrade/web/templates/partials/overview.html b/src/arbitrade/web/templates/partials/overview.html new file mode 100644 index 0000000..6787b51 --- /dev/null +++ b/src/arbitrade/web/templates/partials/overview.html @@ -0,0 +1,67 @@ +
+
+
+
Status
+
{{ status }}
+
+
+
Balances
+
{{ balances }}
+
+
+
Open Trades
+
{{ open_trade_count }}
+
+
+
Realized P&L
+
{{ realized_pnl_total }}
+
+
+ +
+
+
Open Trades
+
    + {% for trade in open_trades %} +
  • + {{ trade.trade_ref }} - {{ trade.status }} - {{ trade.cycle }} - {{ + trade.started_at }} +
    • + {% else %} +
  • No open trades.
    • + {% endfor %} +
+
+
+
Balances Snapshot
+
+ {{ balances }} +
+
Total value {{ total_value }}
+
+
+
Opportunity Feed
+
    + {% for opp in opportunities %} +
  • + {{ opp.cycle }} - {{ opp.net_pct }} - {{ opp.est_profit }} - {{ + opp.detected_at }} +
    • + {% else %} +
  • No opportunities.
    • + {% endfor %} +
+
+
+ +
Updated {{ generated_at }}
+
diff --git a/tests/unit/test_template_resolution.py b/tests/unit/test_template_resolution.py new file mode 100644 index 0000000..26cca3f --- /dev/null +++ b/tests/unit/test_template_resolution.py @@ -0,0 +1,19 @@ +from importlib import resources +from pathlib import Path + +from arbitrade.api import routes + + +def test_template_directory_resolves_to_existing_location() -> None: + template_dir = Path(routes._resolve_templates_directory()) + + assert template_dir.is_dir() + assert (template_dir / "dashboard.html").is_file() + + +def test_template_exists_in_package_resources() -> None: + template_path = resources.files("arbitrade").joinpath( + "web", "templates", "dashboard.html" + ) + + assert template_path.is_file()