Add benchmark detection script and related tests; refactor settings and update project scripts

2026-06-01 11:01:53 +02:00
parent a89886186f
commit 9d8a8a8a45
6 changed files with 262 additions and 20 deletions
@@ -40,6 +40,7 @@ dev = [

 [project.scripts]
 arbitrade = "arbitrade.main:main"
+arbitrade-bench-detection = "arbitrade.detection.benchmark:main"

 [tool.hatch.build.targets.wheel]
 packages = ["src/arbitrade"]
@@ -8,8 +8,7 @@ from pydantic_settings import BaseSettings, SettingsConfigDict


 class Settings(BaseSettings):
-    model_config = SettingsConfigDict(
-        env_file=".env", env_file_encoding="utf-8", extra="ignore")
+    model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")

    app_env: str = Field(default="dev", alias="APP_ENV")
    app_host: str = Field(default="0.0.0.0", alias="APP_HOST")
@@ -18,30 +17,22 @@ class Settings(BaseSettings):
    log_level: str = Field(default="INFO", alias="LOG_LEVEL")
    log_json: bool = Field(default=True, alias="LOG_JSON")

-    duckdb_path: Path = Field(default=Path(
-        "./data/arbitrade.duckdb"), alias="DUCKDB_PATH")
+    duckdb_path: Path = Field(default=Path("./data/arbitrade.duckdb"), alias="DUCKDB_PATH")

-    kraken_rest_url: str = Field(
-        default="https://api.kraken.com", alias="KRAKEN_REST_URL")
-    kraken_ws_url: str = Field(
-        default="wss://ws.kraken.com/v2", alias="KRAKEN_WS_URL")
+    kraken_rest_url: str = Field(default="https://api.kraken.com", alias="KRAKEN_REST_URL")
+    kraken_ws_url: str = Field(default="wss://ws.kraken.com/v2", alias="KRAKEN_WS_URL")
    kraken_private_rate_limit_seconds: float = Field(
        default=1.0, alias="KRAKEN_PRIVATE_RATE_LIMIT_SECONDS"
    )
-    kraken_http_timeout_seconds: float = Field(
-        default=10.0, alias="KRAKEN_HTTP_TIMEOUT_SECONDS")
-    kraken_retry_attempts: int = Field(
-        default=3, alias="KRAKEN_RETRY_ATTEMPTS")
+    kraken_http_timeout_seconds: float = Field(default=10.0, alias="KRAKEN_HTTP_TIMEOUT_SECONDS")
+    kraken_retry_attempts: int = Field(default=3, alias="KRAKEN_RETRY_ATTEMPTS")
    kraken_retry_base_delay_seconds: float = Field(
        default=0.25, alias="KRAKEN_RETRY_BASE_DELAY_SECONDS"
    )
    kraken_api_key: str | None = Field(default=None, alias="KRAKEN_API_KEY")
-    kraken_api_secret: str | None = Field(
-        default=None, alias="KRAKEN_API_SECRET")
-    ws_heartbeat_timeout_seconds: float = Field(
-        default=20.0, alias="WS_HEARTBEAT_TIMEOUT_SECONDS")
-    ws_max_staleness_seconds: float = Field(
-        default=5.0, alias="WS_MAX_STALENESS_SECONDS")
+    kraken_api_secret: str | None = Field(default=None, alias="KRAKEN_API_SECRET")
+    ws_heartbeat_timeout_seconds: float = Field(default=20.0, alias="WS_HEARTBEAT_TIMEOUT_SECONDS")
+    ws_max_staleness_seconds: float = Field(default=5.0, alias="WS_MAX_STALENESS_SECONDS")
    paper_trading_mode: bool = Field(default=True, alias="PAPER_TRADING_MODE")

    fernet_key: str | None = Field(default=None, alias="FERNET_KEY")
@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import argparse
+import statistics
+import time
+from dataclasses import asdict, dataclass
+
+import orjson
+
+from arbitrade.detection.engine import IncrementalCycleDetector
+from arbitrade.detection.graph import CurrencyGraph
+from arbitrade.exchange.models import BookLevel
+from arbitrade.market_data.order_book import OrderBook
+
+
+@dataclass(frozen=True, slots=True)
+class DetectionBenchmarkResult:
+    iterations: int
+    total_ms: float
+    avg_ms: float
+    p50_ms: float
+    p95_ms: float
+    max_ms: float
+    target_ms: float
+
+    @property
+    def meets_target(self) -> bool:
+        return self.p95_ms <= self.target_ms
+
+
+def _make_book(*, bid: float, ask: float) -> OrderBook:
+    book = OrderBook()
+    book.apply_bids([BookLevel(price=bid, volume=10.0)])
+    book.apply_asks([BookLevel(price=ask, volume=10.0)])
+    return book
+
+
+def _build_detector_and_books() -> tuple[IncrementalCycleDetector, dict[str, OrderBook]]:
+    asset_pairs = {
+        "XXBTZUSD": {"wsname": "BTC/USD"},
+        "XETHXXBT": {"wsname": "ETH/BTC"},
+        "XETHZUSD": {"wsname": "ETH/USD"},
+    }
+    graph = CurrencyGraph.from_kraken_asset_pairs(asset_pairs)
+    cycles = graph.triangular_cycles()
+    index = graph.index_cycles_by_pair(cycles)
+
+    detector = IncrementalCycleDetector(
+        index,
+        fee_rate=0.001,
+        min_profit_threshold=0.001,
+        max_depth_levels=5,
+        max_book_age_seconds=10.0,
+    )
+
+    books = {
+        "BTC/USD": _make_book(bid=99.9, ask=100.0),
+        "ETH/BTC": _make_book(bid=0.049, ask=0.05),
+        "ETH/USD": _make_book(bid=5.2, ask=5.21),
+    }
+    return detector, books
+
+
+def run_incremental_detection_benchmark(
+    *,
+    iterations: int = 50_000,
+    target_ms: float = 1.0,
+) -> DetectionBenchmarkResult:
+    if iterations <= 0:
+        raise ValueError("iterations must be > 0")
+
+    detector, books = _build_detector_and_books()
+
+    samples_ms: list[float] = []
+    started_ns = time.perf_counter_ns()
+    for _ in range(iterations):
+        t0_ns = time.perf_counter_ns()
+        detector.score_updated_pair("ETH/BTC", books)
+        elapsed_ms = (time.perf_counter_ns() - t0_ns) / 1_000_000
+        samples_ms.append(elapsed_ms)
+
+    total_ms = (time.perf_counter_ns() - started_ns) / 1_000_000
+    return DetectionBenchmarkResult(
+        iterations=iterations,
+        total_ms=total_ms,
+        avg_ms=statistics.fmean(samples_ms),
+        p50_ms=statistics.quantiles(samples_ms, n=100)[49],
+        p95_ms=statistics.quantiles(samples_ms, n=100)[94],
+        max_ms=max(samples_ms),
+        target_ms=target_ms,
+    )
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Benchmark incremental detection latency")
+    parser.add_argument("--iterations", type=int, default=50_000)
+    parser.add_argument("--target-ms", type=float, default=1.0)
+    args = parser.parse_args()
+
+    result = run_incremental_detection_benchmark(
+        iterations=args.iterations,
+        target_ms=args.target_ms,
+    )
+
+    payload = {
+        **asdict(result),
+        "meets_target": result.meets_target,
+    }
+    print(orjson.dumps(payload).decode("utf-8"))
+
+
+if __name__ == "__main__":
+    main()
@@ -23,8 +23,7 @@ class MarketDataFeed:
        detector: IncrementalCycleDetector | None = None,
        opportunity_writer: AsyncOpportunityWriter | None = None,
        paper_trading_mode: bool = True,
-        opportunity_executor: Callable[[
-            OpportunityEvent], Awaitable[None]] | None = None,
+        opportunity_executor: Callable[[OpportunityEvent], Awaitable[None]] | None = None,
    ) -> None:
        self._ws_client = ws_client
        self._snapshot_writer = snapshot_writer
@@ -0,0 +1,19 @@
+import pytest
+
+from arbitrade.detection.benchmark import run_incremental_detection_benchmark
+
+
+def test_incremental_detection_benchmark_returns_metrics() -> None:
+    result = run_incremental_detection_benchmark(iterations=500)
+
+    assert result.iterations == 500
+    assert result.total_ms > 0.0
+    assert result.avg_ms > 0.0
+    assert result.p50_ms > 0.0
+    assert result.p95_ms > 0.0
+    assert result.max_ms >= result.p95_ms
+
+
+def test_incremental_detection_benchmark_rejects_invalid_iterations() -> None:
+    with pytest.raises(ValueError, match="iterations"):
+        run_incremental_detection_benchmark(iterations=0)
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import pytest
+
+from arbitrade.detection.engine import IncrementalCycleDetector
+from arbitrade.detection.graph import CurrencyGraph
+from arbitrade.exchange.models import BookLevel
+from arbitrade.market_data.order_book import OrderBook
+
+
+def _make_book(*, bid: float, ask: float) -> OrderBook:
+    book = OrderBook()
+    book.apply_bids([BookLevel(price=bid, volume=2_000.0)])
+    book.apply_asks([BookLevel(price=ask, volume=2_000.0)])
+    return book
+
+
+def _make_book_levels(
+    *, bids: list[tuple[float, float]], asks: list[tuple[float, float]]
+) -> OrderBook:
+    book = OrderBook()
+    book.apply_bids([BookLevel(price=price, volume=volume)
+                    for price, volume in bids])
+    book.apply_asks([BookLevel(price=price, volume=volume)
+                    for price, volume in asks])
+    return book
+
+
+def test_synthetic_single_cycle_known_exact_rates() -> None:
+    asset_pairs = {
+        "XXBTZUSD": {"wsname": "BTC/USD"},
+        "XETHXXBT": {"wsname": "ETH/BTC"},
+        "XETHZUSD": {"wsname": "ETH/USD"},
+    }
+    graph = CurrencyGraph.from_kraken_asset_pairs(asset_pairs)
+    detector = IncrementalCycleDetector(
+        graph.index_cycles_by_pair(graph.triangular_cycles()),
+        fee_rate=0.001,
+        max_depth_levels=1,
+    )
+
+    books = {
+        "BTC/USD": _make_book(bid=99.9, ask=100.0),
+        "ETH/BTC": _make_book(bid=0.049, ask=0.05),
+        "ETH/USD": _make_book(bid=5.20, ask=5.21),
+    }
+
+    scores = detector.score_updated_pair("ETH/BTC", books)
+
+    assert len(scores) == 1
+    score = scores[0]
+    # Known path result: 1 USD -> 0.01 BTC -> 0.2 ETH -> 1.04 USD gross.
+    assert score.gross_rate == pytest.approx(1.04)
+    assert score.net_rate == pytest.approx(1.04 * (1 - 0.001) ** 3)
+
+
+def test_synthetic_two_cycles_known_filtering_and_pair_index() -> None:
+    asset_pairs = {
+        "XXBTZUSD": {"wsname": "BTC/USD"},
+        "XETHXXBT": {"wsname": "ETH/BTC"},
+        "XETHZUSD": {"wsname": "ETH/USD"},
+        "XLTCXXBT": {"wsname": "LTC/BTC"},
+        "XLTCZUSD": {"wsname": "LTC/USD"},
+    }
+    graph = CurrencyGraph.from_kraken_asset_pairs(asset_pairs)
+    detector = IncrementalCycleDetector(
+        graph.index_cycles_by_pair(graph.triangular_cycles()),
+        min_profit_threshold=0.02,
+        max_depth_levels=3,
+    )
+
+    books = {
+        # ETH cycle expected to pass threshold (~4% gross)
+        "BTC/USD": _make_book(bid=99.9, ask=100.0),
+        "ETH/BTC": _make_book(bid=0.049, ask=0.05),
+        "ETH/USD": _make_book(bid=5.20, ask=5.21),
+        # LTC cycle expected to fail threshold (<2% gross)
+        "LTC/BTC": _make_book(bid=0.01005, ask=0.0101),
+        "LTC/USD": _make_book(bid=1.01, ask=1.011),
+    }
+
+    scores = detector.score_updated_pair("BTC/USD", books)
+
+    assert len(scores) == 1
+    assert scores[0].cycle.currencies == ("BTC", "ETH", "USD")
+    assert (scores[0].net_rate - 1.0) >= 0.02
+
+
+def test_synthetic_depth_scenario_known_no_result_when_not_fillable() -> None:
+    asset_pairs = {
+        "XXBTZUSD": {"wsname": "BTC/USD"},
+        "XETHXXBT": {"wsname": "ETH/BTC"},
+        "XETHZUSD": {"wsname": "ETH/USD"},
+    }
+    graph = CurrencyGraph.from_kraken_asset_pairs(asset_pairs)
+    detector = IncrementalCycleDetector(
+        graph.index_cycles_by_pair(graph.triangular_cycles()),
+        max_depth_levels=2,
+    )
+
+    books = {
+        # Not enough ask-side BTC/USD capacity inside top-2 levels for initial USD->BTC conversion.
+        "BTC/USD": _make_book_levels(
+            bids=[(99.9, 10.0)],
+            asks=[(100.0, 0.003), (101.0, 0.003)],
+        ),
+        "ETH/BTC": _make_book_levels(
+            bids=[(0.049, 10.0)],
+            asks=[(0.05, 10.0)],
+        ),
+        "ETH/USD": _make_book_levels(
+            bids=[(5.20, 10.0)],
+            asks=[(5.21, 10.0)],
+        ),
+    }
+
+    scores = detector.score_updated_pair("BTC/USD", books)
+
+    assert scores == []