diff --git a/README.md b/README.md index b0a1525..71a4e73 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,36 @@ Key features include: - Backtesting parameter configuration - Fee configuration by pairing and market type +## Templates + +Full page templates (`src/arbitrade/web/templates/`): + +| Template | Route | Purpose | +| ------------------ | ------------------------ | ------------------------------------------------------- | +| `base.html` | — (root layout) | Dark theme, `.shell` container, HTMX, CSS variables | +| `dashboard.html` | `/`, `/dashboard` | Main dashboard: metrics, overview, controls, charts | +| `config.html` | `/dashboard/config` | Full configuration: fees, runtime, alerts, Kraken, risk | +| `audit.html` | `/dashboard/audit` | Audit trail with auto-refresh via HTMX | +| `backtesting.html` | `/dashboard/backtesting` | Backtesting panel with replay/sweep forms | +| `health.html` | `/health` | System health check | + +Dashboard partials (`src/arbitrade/web/templates/partials/`): + +| Partial | In page | Content | +| ------------------------ | ---------------- | --------------------------------------------------------------------------------------------------- | +| `metrics.html` | Dashboard | 6 KPI cards: P&L, win rate, avg duration, trade count, success %, profit factor | +| `overview.html` | Dashboard | Status, balances, fee tier, open trades list, opportunity feed | +| `controls.html` | Dashboard | Runtime status, kill switch, config snapshot, alerting status, execution controls (Start/Stop/Kill) | +| `charts.html` | Dashboard | Opportunity trend chart (Chart.js, Alpine toggle) | +| `config.html` | Config page | Config form: Runtime, Alerts, Kraken, Risk, Strategy sections | +| `config_fees.html` | Config page | Pair fee table + add/edit form | +| `backtesting_panel.html` | Backtesting page | Run status, replay/sweep forms, recent runs | +| `audit.html` | Audit page | Audit trail table: time, actor, event, decision, payload | + +Legacy templates (`src/arbitrade/web/templates/dashboard/`): + +- `config_settings.html`, `config_pairs.html`, `config_fees.html` — superseded by config page; retained for reference + ## Prerequisites - Python 3.12+ diff --git a/scripts/backtest_replay.py b/scripts/backtest_replay.py index 0a0f617..e0c704f 100644 --- a/scripts/backtest_replay.py +++ b/scripts/backtest_replay.py @@ -6,10 +6,31 @@ from collections.abc import Mapping from datetime import UTC, datetime from pathlib import Path +import duckdb + from arbitrade.backtesting.replay import BacktestConfig, BacktestReplayEngine, load_replay_events from arbitrade.detection.graph import CurrencyGraph, TriangularCycle +def _resolve_fee_rate(fee_rate: float | None, db_path: str | None = None) -> float: + """Resolve fee rate from arg or DB snapshot. Falls back to 0.0026.""" + if fee_rate is not None: + return fee_rate + if db_path is not None: + try: + conn = duckdb.connect(db_path) + row = conn.execute(""" + SELECT maker_fee FROM kraken_account_snapshots + ORDER BY snapshot_at DESC LIMIT 1 + """).fetchone() + conn.close() + if row is not None and row[0] is not None: + return float(row[0]) + except Exception: + pass + return 0.0026 # ultimate fallback + + def _build_graph() -> tuple[dict[str, list[TriangularCycle]], list[str]]: graph = CurrencyGraph() graph.add_pair("USD", "BTC", "BTC/USD") @@ -30,19 +51,23 @@ def _parse_balances(raw: str) -> Mapping[str, float]: def main() -> int: - parser = argparse.ArgumentParser(description="Run a deterministic replay backtest.") + parser = argparse.ArgumentParser( + description="Run a deterministic replay backtest.") parser.add_argument("--events", type=Path, required=True) parser.add_argument("--starting-balances", type=str, default="USD=1000.0") parser.add_argument("--trade-capital", type=float, default=100.0) - parser.add_argument("--fee-rate", type=float, default=0.0026) + parser.add_argument("--fee-rate", type=float, default=None) parser.add_argument("--slippage-bps", type=float, default=4.0) parser.add_argument("--execution-latency-ms", type=float, default=20.0) + parser.add_argument("--db-path", type=str, default=None, + help="DuckDB path for fee lookup") args = parser.parse_args() cycles_by_pair, available_pairs = _build_graph() events = load_replay_events(args.events) + fee_rate = _resolve_fee_rate(args.fee_rate, args.db_path) config = BacktestConfig( - fee_rate=args.fee_rate, + fee_rate=fee_rate, trade_capital=args.trade_capital, slippage_bps=args.slippage_bps, execution_latency_ms=args.execution_latency_ms, @@ -55,15 +80,18 @@ def main() -> int: started_at=events[0].occurred_at if events else datetime.now(UTC), ) report = asyncio.run( - engine.run(events, starting_balances=_parse_balances(args.starting_balances)) + engine.run(events, starting_balances=_parse_balances( + args.starting_balances)) ) print("Backtest report:") print(f"- processed_events: {report.processed_events}") print(f"- opportunities_seen: {report.opportunities_seen}") print(f"- trades_executed: {report.trades_executed}") - print(f"- win_rate: {report.win_rate if report.win_rate is not None else 'n/a'}") - print(f"- fill_rate: {report.fill_rate if report.fill_rate is not None else 'n/a'}") + print( + f"- win_rate: {report.win_rate if report.win_rate is not None else 'n/a'}") + print( + f"- fill_rate: {report.fill_rate if report.fill_rate is not None else 'n/a'}") print(f"- realized_pnl_usd: {report.realized_pnl_usd:.4f}") print(f"- max_drawdown_usd: {report.max_drawdown_usd:.4f}") print(f"- miss_reasons: {dict(report.miss_reasons)}") diff --git a/src/arbitrade/api/routes.py b/src/arbitrade/api/routes.py index 7c84708..dcc0eb3 100644 --- a/src/arbitrade/api/routes.py +++ b/src/arbitrade/api/routes.py @@ -19,7 +19,7 @@ from arbitrade.api.auth import require_dashboard_auth from arbitrade.api.control_state import DashboardControlState from arbitrade.backtesting.replay import BacktestConfig, BacktestReplayEngine, load_replay_events from arbitrade.detection.graph import CurrencyGraph, TriangularCycle -from arbitrade.storage.repositories import AuditRecord, AuditRepository +from arbitrade.storage.repositories import AuditRecord, AuditRepository, KrakenAccountSnapshotRepository router = APIRouter(dependencies=[Depends(require_dashboard_auth)]) public_router = APIRouter() @@ -185,6 +185,7 @@ def _dashboard_overview(request: Request) -> dict[str, object]: "maker_fee": maker_fee, "taker_fee": taker_fee, "thirty_day_volume": thirty_day_volume, + "fee_source": "API" if fee_tier != "—" else "—", } @@ -569,23 +570,49 @@ def _normalize_fee_profile(profile: str) -> str: return profile.strip().lower().replace("-", "_") -def _fee_rate_for_profile(profile: str, custom_fee_rate: float | None) -> float: +def _fee_rate_for_profile( + profile: str, + custom_fee_rate: float | None, + request: Request | None = None, +) -> float: + """Resolve fee rate from profile name. + + - 'api': fetches latest maker_fee from kraken_account_snapshots (requires request) + - 'custom': uses custom_fee_rate + - legacy 'standard'/'maker_heavy'/'taker_heavy': still supported via hardcoded + fallback, logged at warning level + """ normalized = _normalize_fee_profile(profile) - profile_map = { - "standard": 0.0026, - "maker_heavy": 0.0016, - "taker_heavy": 0.0035, - } + + if normalized == "api": + if request is None: + raise ValueError("api fee profile requires request context") + store = request.app.state.store + repo = KrakenAccountSnapshotRepository(store) + latest = repo.latest_snapshot() + if latest is not None and latest.maker_fee is not None: + return latest.maker_fee + # Fallback to standard if no snapshot yet + return 0.0026 + if normalized == "custom": if custom_fee_rate is None: raise ValueError("custom fee profile requires custom_fee_rate") if custom_fee_rate < 0.0: raise ValueError("custom_fee_rate must be >= 0") return custom_fee_rate - if normalized not in profile_map: - valid = ", ".join(sorted(list(profile_map.keys()) + ["custom"])) - raise ValueError(f"fee_profile must be one of: {valid}") - return profile_map[normalized] + + # Legacy hardcoded profiles (kept for backward compat, but soft-deprecated) + profile_map = { + "standard": 0.0026, + "maker_heavy": 0.0016, + "taker_heavy": 0.0035, + } + if normalized in profile_map: + return profile_map[normalized] + + valid = ", ".join(sorted(list(profile_map.keys()) + ["api", "custom"])) + raise ValueError(f"fee_profile must be one of: {valid}") def _parse_balances(raw: str) -> dict[str, float]: @@ -652,7 +679,7 @@ def _backtesting_panel_context( "starting_balances": "USD=1000.0", "trade_capital": "100.0", "min_profit_threshold": "0.0005", - "fee_profile": "standard", + "fee_profile": "api", "custom_fee_rate": "", "slippage_bps": "4.0", "execution_latency_ms": "20.0", @@ -687,7 +714,6 @@ async def _dashboard_response( "overview_endpoint": "/dashboard/fragment/overview", "controls_endpoint": "/dashboard/fragment/controls", "charts_endpoint": "/dashboard/fragment/charts", - "audit_endpoint": "/dashboard/fragment/audit", "stream_endpoint": "/dashboard/stream/metrics", "overview_stream_endpoint": "/dashboard/stream/overview", }, @@ -763,6 +789,28 @@ async def dashboard_charts(request: Request) -> HTMLResponse: ) +@router.get("/dashboard/audit", response_class=HTMLResponse) +async def dashboard_audit_page(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="audit.html", + context={ + "title": "Arbitrade Audit Trail", + "request": request, + **_dashboard_audit(request), + }, + ) + + +@router.get("/dashboard/audit/fragment", response_class=HTMLResponse) +async def dashboard_audit_fragment(request: Request) -> HTMLResponse: + return templates.TemplateResponse( + request=request, + name="partials/audit.html", + context={"request": request, **_dashboard_audit(request)}, + ) + + @router.get("/dashboard/fragment/audit", response_class=HTMLResponse) async def dashboard_audit(request: Request) -> HTMLResponse: return templates.TemplateResponse( @@ -903,7 +951,7 @@ async def dashboard_backtesting_run(request: Request) -> HTMLResponse: "starting_balances": form.get("starting_balances", "USD=1000.0"), "trade_capital": form.get("trade_capital", "100.0"), "min_profit_threshold": form.get("min_profit_threshold", "0.0005"), - "fee_profile": _normalize_fee_profile(form.get("fee_profile", "standard")), + "fee_profile": _normalize_fee_profile(form.get("fee_profile", "api")), "custom_fee_rate": form.get("custom_fee_rate", ""), "slippage_bps": form.get("slippage_bps", "4.0"), "execution_latency_ms": form.get("execution_latency_ms", "20.0"), @@ -924,7 +972,7 @@ async def dashboard_backtesting_run(request: Request) -> HTMLResponse: ) if defaults["custom_fee_rate"].strip() else None ) fee_rate = _fee_rate_for_profile( - defaults["fee_profile"], custom_fee_rate) + defaults["fee_profile"], custom_fee_rate, request=request) starting_balances = _parse_balances(defaults["starting_balances"]) trade_capital = float(defaults["trade_capital"]) diff --git a/src/arbitrade/backtesting/replay.py b/src/arbitrade/backtesting/replay.py index 0a14ec7..4daf083 100644 --- a/src/arbitrade/backtesting/replay.py +++ b/src/arbitrade/backtesting/replay.py @@ -56,7 +56,7 @@ class ReplayBookEvent: @dataclass(frozen=True, slots=True) class BacktestConfig: - fee_rate: float = 0.0026 + fee_rate: float = 0.0 # 0.0 means "use API-sourced fee from kraken_account_snapshots" min_profit_threshold: float = 0.0005 trade_capital: float = 100.0 quote_asset: str = "USD" diff --git a/src/arbitrade/config/service.py b/src/arbitrade/config/service.py index dd40e55..ed2cfce 100644 --- a/src/arbitrade/config/service.py +++ b/src/arbitrade/config/service.py @@ -44,6 +44,7 @@ class ConfigPairFee(BaseModel): market_type: str # 'crypto_crypto' or 'crypto_fiat' maker_fee_rate: float taker_fee_rate: float + source: str = "manual" # 'manual' or 'kraken_api' updated_at: datetime | None = None @@ -247,6 +248,7 @@ class ConfigurationService: "market_type": fee.market_type, "maker_fee_rate": fee.maker_fee_rate, "taker_fee_rate": fee.taker_fee_rate, + "source": fee.source, "updated_at": fee.updated_at.isoformat() if fee.updated_at else None, }) return result @@ -282,6 +284,7 @@ class ConfigurationService: "market_type": updated.market_type, "maker_fee_rate": updated.maker_fee_rate, "taker_fee_rate": updated.taker_fee_rate, + "source": updated.source, "updated_at": updated.updated_at.isoformat() if updated.updated_at else None, } diff --git a/src/arbitrade/detection/benchmark.py b/src/arbitrade/detection/benchmark.py index 3ec8056..bfd053c 100644 --- a/src/arbitrade/detection/benchmark.py +++ b/src/arbitrade/detection/benchmark.py @@ -47,7 +47,7 @@ def _build_detector_and_books() -> tuple[IncrementalCycleDetector, dict[str, Ord detector = IncrementalCycleDetector( index, - fee_rate=0.001, + fee_rate=0.001, # synthetic benchmark: uses fixed rate, not API-sourced min_profit_threshold=0.001, max_depth_levels=5, max_book_age_seconds=10.0, @@ -92,7 +92,8 @@ def run_incremental_detection_benchmark( def main() -> None: - parser = argparse.ArgumentParser(description="Benchmark incremental detection latency") + parser = argparse.ArgumentParser( + description="Benchmark incremental detection latency") parser.add_argument("--iterations", type=int, default=50_000) parser.add_argument("--target-ms", type=float, default=1.0) args = parser.parse_args() diff --git a/src/arbitrade/storage/db.py b/src/arbitrade/storage/db.py index f01e6f0..6acfd62 100644 --- a/src/arbitrade/storage/db.py +++ b/src/arbitrade/storage/db.py @@ -61,7 +61,8 @@ CREATE TABLE IF NOT EXISTS config_backtesting_defaults ( trade_capital DOUBLE, min_profit_threshold DOUBLE, slippage_bps INTEGER, - execution_latency_ms INTEGER + execution_latency_ms INTEGER, + fee_source VARCHAR DEFAULT 'api' ); CREATE TABLE IF NOT EXISTS opportunities ( @@ -159,7 +160,7 @@ CREATE TABLE IF NOT EXISTS kraken_account_snapshots ( class DuckDBStore: - SCHEMA_VERSION = 3 + SCHEMA_VERSION = 4 def __init__(self, settings: Settings) -> None: self._db_path = Path(settings.duckdb_path) @@ -274,6 +275,14 @@ class DuckDBStore: "INSERT OR IGNORE INTO schema_migrations (version) VALUES (3)") _LOG.info("migration_applied", version=3) + if current_version < 4: + # Migration v4: Add fee_source to backtesting defaults + conn.execute( + "ALTER TABLE config_backtesting_defaults ADD COLUMN IF NOT EXISTS fee_source VARCHAR DEFAULT 'api'") + conn.execute( + "INSERT OR IGNORE INTO schema_migrations (version) VALUES (4)") + _LOG.info("migration_applied", version=4) + # Update version to current conn.execute( f"INSERT OR REPLACE INTO schema_migrations (version, applied_at) " diff --git a/src/arbitrade/storage/repositories.py b/src/arbitrade/storage/repositories.py index fbf5330..05f3ebf 100644 --- a/src/arbitrade/storage/repositories.py +++ b/src/arbitrade/storage/repositories.py @@ -738,15 +738,16 @@ class ConfigPairFeeRepository: with self._store.connect() as conn: cursor = conn.execute( """ - INSERT INTO config_pair_fees (pairing_id, market_type, maker_fee_rate, taker_fee_rate) - VALUES (?, ?, ?, ?) - RETURNING pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at + INSERT INTO config_pair_fees (pairing_id, market_type, maker_fee_rate, taker_fee_rate, source) + VALUES (?, ?, ?, ?, ?) + RETURNING pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at, source """, ( pair_fee.pairing_id, pair_fee.market_type, pair_fee.maker_fee_rate, pair_fee.taker_fee_rate, + pair_fee.source, ), ) row = cursor.fetchone() @@ -756,7 +757,8 @@ class ConfigPairFeeRepository: market_type=row[1], maker_fee_rate=row[2], taker_fee_rate=row[3], - updated_at=row[4] + updated_at=row[4], + source=row[5], ) raise ValueError("Failed to create pair fee") @@ -765,7 +767,7 @@ class ConfigPairFeeRepository: with self._store.connect() as conn: cursor = conn.execute( """ - SELECT pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at + SELECT pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at, COALESCE(source, 'manual') AS source FROM config_pair_fees WHERE pairing_id = ? AND market_type = ? """, @@ -778,7 +780,8 @@ class ConfigPairFeeRepository: market_type=row[1], maker_fee_rate=row[2], taker_fee_rate=row[3], - updated_at=row[4] + updated_at=row[4], + source=row[5], ) return None @@ -788,13 +791,14 @@ class ConfigPairFeeRepository: cursor = conn.execute( """ UPDATE config_pair_fees - SET maker_fee_rate = ?, taker_fee_rate = ? + SET maker_fee_rate = ?, taker_fee_rate = ?, source = ? WHERE pairing_id = ? AND market_type = ? - RETURNING pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at + RETURNING pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at, source """, ( pair_fee.maker_fee_rate, pair_fee.taker_fee_rate, + pair_fee.source, pairing_id, market_type, ), @@ -806,7 +810,8 @@ class ConfigPairFeeRepository: market_type=row[1], maker_fee_rate=row[2], taker_fee_rate=row[3], - updated_at=row[4] + updated_at=row[4], + source=row[5], ) raise ValueError("Failed to update pair fee") @@ -827,7 +832,7 @@ class ConfigPairFeeRepository: with self._store.connect() as conn: cursor = conn.execute( """ - SELECT pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at + SELECT pairing_id, market_type, maker_fee_rate, taker_fee_rate, updated_at, COALESCE(source, 'manual') AS source FROM config_pair_fees WHERE pairing_id = ? ORDER BY market_type @@ -840,7 +845,8 @@ class ConfigPairFeeRepository: market_type=row[1], maker_fee_rate=row[2], taker_fee_rate=row[3], - updated_at=row[4] + updated_at=row[4], + source=row[5], ) for row in cursor.fetchall() ] diff --git a/src/arbitrade/web/templates/audit.html b/src/arbitrade/web/templates/audit.html new file mode 100644 index 0000000..ca05366 --- /dev/null +++ b/src/arbitrade/web/templates/audit.html @@ -0,0 +1,27 @@ +{% extends "base.html" %} {% block title %}Audit Trail{% endblock %} {% block +main_class %}shell{% endblock %} {% block content %} +
+
+

Audit Trail

+

+ System activity, configuration changes, and execution decisions. +

+
+
+ Dashboard + Config + Backtesting + Health +
+
+ +
+ {% include "partials/audit.html" %} +
+{% endblock %} diff --git a/src/arbitrade/web/templates/base.html b/src/arbitrade/web/templates/base.html index 4155859..8ad4cf4 100644 --- a/src/arbitrade/web/templates/base.html +++ b/src/arbitrade/web/templates/base.html @@ -14,9 +14,8 @@ color: #e5eefb; } .shell { - max-width: 1120px; - margin: 0 auto; - padding: 32px 20px 48px; + max-width: none; + padding: 24px 32px 48px; } .hero { display: flex; diff --git a/src/arbitrade/web/templates/config.html b/src/arbitrade/web/templates/config.html index fa56b43..e897405 100644 --- a/src/arbitrade/web/templates/config.html +++ b/src/arbitrade/web/templates/config.html @@ -10,6 +10,7 @@ main_class %}shell{% endblock %} {% block content %}
Dashboard Backtesting + Audit Health
diff --git a/src/arbitrade/web/templates/dashboard.html b/src/arbitrade/web/templates/dashboard.html index 4751afe..0a0a19c 100644 --- a/src/arbitrade/web/templates/dashboard.html +++ b/src/arbitrade/web/templates/dashboard.html @@ -20,6 +20,7 @@ head_scripts %} Health Config Backtesting + Audit @@ -63,15 +64,6 @@ head_scripts %} {% include "partials/charts.html" %} -
- {% include "partials/audit.html" %} -
{% endblock %} {% block scripts %}