feat: Implement idempotency and recovery mechanisms for order execution

- Add IdempotencyKeyFactory for generating unique user references based on execution legs.
- Introduce OrderReconciler to reconcile order statuses with historical data.
- Implement PartialFillRecovery to handle partial fills by canceling orders and placing hedges.
- Create TriangularExecutionSequencer for executing triangular arbitrage strategies.
- Enhance storage with new tables for trades, orders, and PnL events.
- Develop AsyncExecutionWriter for asynchronous writing of execution records to the database.
- Add unit tests for execution persistence, sequencer behavior, fill monitoring, and idempotency checks.
- Update KrakenRestClient to ensure proper payloads for order placement and querying.
This commit is contained in:
2026-06-01 11:59:13 +02:00
parent 240a591a64
commit 93f4f62d42
17 changed files with 1602 additions and 4 deletions
+90
View File
@@ -0,0 +1,90 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
import pytest
from arbitrade.config.settings import Settings
from arbitrade.detection.engine import OpportunityEvent
from arbitrade.execution.sequencer import TriangularExecutionSequencer
from arbitrade.storage.db import DuckDBStore
from arbitrade.storage.executions import AsyncExecutionWriter
from arbitrade.storage.repositories import OrderRepository, PnLRepository, TradeRepository
@dataclass(slots=True)
class _FakeRestClient:
calls: int = 0
async def place_market_order(self, *, pair: str, side: str, volume: float) -> dict[str, object]:
self.calls += 1
return {"txid": [f"tx-{self.calls}"], "status": "submitted"}
def _sample_event() -> OpportunityEvent:
return OpportunityEvent(
detected_at=datetime.now(UTC),
cycle="USD->BTC->ETH->USD",
updated_pair="BTC/USD",
gross_rate=1.04,
net_rate=1.03,
gross_pct=4.0,
net_pct=3.0,
est_profit=0.03,
)
@pytest.mark.asyncio
async def test_execution_writer_persists_trade_order_and_pnl(tmp_path) -> None:
settings = Settings(_env_file=None, DUCKDB_PATH=tmp_path / "exec.duckdb")
store = DuckDBStore(settings)
store.migrate()
writer = AsyncExecutionWriter(
TradeRepository(store),
OrderRepository(store),
PnLRepository(store),
max_queue_size=10,
)
await writer.start()
client = _FakeRestClient()
sequencer = TriangularExecutionSequencer(
client,
available_pairs=["BTC/USD", "ETH/BTC", "ETH/USD"],
execution_writer=writer,
)
result = await sequencer.execute(_sample_event())
await writer.stop()
assert result.success
assert client.calls == 3
with store.connect() as conn:
trades = conn.execute(
"SELECT trade_ref, status, estimated_pnl, capital_used, cycle, leg_count FROM trades"
).fetchall()
orders = conn.execute(
"SELECT trade_ref, order_ref, leg_index, pair, side, volume, status "
"FROM orders ORDER BY leg_index"
).fetchall()
pnls = conn.execute(
"SELECT trade_ref, kind, pnl_usd, source FROM pnl_events").fetchall()
assert len(trades) == 1
assert trades[0][1] == "filled"
assert trades[0][2] == 0.03
assert trades[0][3] == 1.0
assert trades[0][4] == "USD->BTC->ETH->USD"
assert trades[0][5] == 3
assert len(orders) == 3
assert orders[0][2] == 0
assert orders[1][2] == 1
assert orders[2][2] == 2
assert orders[0][6] == "submitted"
assert len(pnls) == 1
assert pnls[0][1] == "estimated"
assert pnls[0][2] == 0.03
+93
View File
@@ -0,0 +1,93 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
import pytest
from arbitrade.detection.engine import OpportunityEvent
from arbitrade.execution.sequencer import TriangularExecutionSequencer
@dataclass(slots=True)
class _FakeRestClient:
fail_at_call: int | None = None
calls: list[dict[str, Any]] = field(default_factory=list)
async def place_market_order(self, *, pair: str, side: str, volume: float) -> dict[str, Any]:
call_number = len(self.calls) + 1
if self.fail_at_call is not None and call_number == self.fail_at_call:
raise RuntimeError("simulated failure")
payload = {"pair": pair, "side": side, "volume": volume}
self.calls.append(payload)
return {"txid": [f"tx-{call_number}"]}
def _sample_event(cycle: str = "USD->BTC->ETH->USD") -> OpportunityEvent:
return OpportunityEvent(
detected_at=datetime.now(UTC),
cycle=cycle,
updated_pair="BTC/USD",
gross_rate=1.02,
net_rate=1.01,
gross_pct=2.0,
net_pct=1.0,
est_profit=1.0,
allocated_capital=10.0,
)
@pytest.mark.asyncio
async def test_triangular_sequencer_executes_legs_in_order() -> None:
client = _FakeRestClient()
sequencer = TriangularExecutionSequencer(
client,
available_pairs=["BTC/USD", "ETH/BTC", "ETH/USD"],
)
result = await sequencer.execute(_sample_event())
assert result.success
assert result.completed_legs == 3
assert [call["pair"] for call in client.calls] == ["BTC/USD", "ETH/BTC", "ETH/USD"]
assert [call["side"] for call in client.calls] == ["buy", "buy", "sell"]
@pytest.mark.asyncio
async def test_triangular_sequencer_stops_on_failed_leg() -> None:
client = _FakeRestClient(fail_at_call=2)
sequencer = TriangularExecutionSequencer(
client,
available_pairs=["BTC/USD", "ETH/BTC", "ETH/USD"],
)
result = await sequencer.execute(_sample_event())
assert not result.success
assert result.completed_legs == 1
assert result.failure_reason is not None
assert len(client.calls) == 1
def test_triangular_sequencer_rejects_non_closed_cycle() -> None:
client = _FakeRestClient()
sequencer = TriangularExecutionSequencer(
client,
available_pairs=["BTC/USD", "ETH/BTC", "ETH/USD"],
)
with pytest.raises(ValueError, match="closed triangular path"):
sequencer._build_legs(_sample_event(cycle="USD->BTC->ETH"))
def test_triangular_sequencer_rejects_missing_pair() -> None:
client = _FakeRestClient()
sequencer = TriangularExecutionSequencer(
client,
available_pairs=["BTC/USD", "ETH/BTC"],
)
with pytest.raises(ValueError, match="No tradable pair"):
sequencer._build_legs(_sample_event())
+112
View File
@@ -0,0 +1,112 @@
from __future__ import annotations
from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
import pytest
from arbitrade.execution.fill_monitor import FillMonitor, OrderFillState
@dataclass(slots=True)
class _FakePollClient:
responses: list[dict[str, Any]]
calls: int = 0
async def query_order(self, *, order_id: str, include_trades: bool = True) -> dict[str, Any]:
self.calls += 1
if self.responses:
return self.responses.pop(0)
return {order_id: {"status": "open", "vol_exec": "0.0", "price": "0.0"}}
@dataclass(slots=True)
class _FakeWsProvider:
states: list[OrderFillState] = field(default_factory=list)
def get(self, _order_id: str) -> OrderFillState | None:
if not self.states:
return None
return self.states.pop(0)
@pytest.mark.asyncio
async def test_fill_monitor_detects_terminal_state_via_polling() -> None:
order_id = "order-1"
client = _FakePollClient(
responses=[
{order_id: {"status": "open", "vol_exec": "0.0", "price": "0.0"}},
{order_id: {"status": "closed", "vol_exec": "1.0", "price": "100.0"}},
]
)
monitor = FillMonitor(client, poll_interval_seconds=0.001, max_wait_seconds=0.1)
result = await monitor.wait_for_terminal_fill(order_id)
assert not result.timed_out
assert result.terminal_state is not None
assert result.terminal_state.status == "closed"
assert result.terminal_state.filled_volume == 1.0
assert result.terminal_state.source == "rest_poll"
@pytest.mark.asyncio
async def test_fill_monitor_times_out_when_no_terminal_state() -> None:
order_id = "order-2"
client = _FakePollClient(
responses=[
{order_id: {"status": "open", "vol_exec": "0.1", "price": "100.0"}},
{order_id: {"status": "partial", "vol_exec": "0.2", "price": "100.0"}},
{order_id: {"status": "open", "vol_exec": "0.2", "price": "100.0"}},
]
)
monitor = FillMonitor(client, poll_interval_seconds=0.001, max_wait_seconds=0.01)
result = await monitor.wait_for_terminal_fill(order_id)
assert result.timed_out
assert result.terminal_state is None
assert result.last_state is not None
assert result.last_state.status in {"open", "partial"}
@pytest.mark.asyncio
async def test_fill_monitor_uses_ws_status_for_fast_terminal_detection() -> None:
order_id = "order-3"
ws_provider = _FakeWsProvider(
states=[
OrderFillState(
order_id=order_id,
status="closed",
filled_volume=0.5,
avg_price=200.0,
updated_at=datetime.now(UTC),
source="ws",
)
]
)
client = _FakePollClient(responses=[])
monitor = FillMonitor(
client,
poll_interval_seconds=0.001,
max_wait_seconds=0.1,
ws_status_provider=ws_provider.get,
)
result = await monitor.wait_for_terminal_fill(order_id)
assert not result.timed_out
assert result.terminal_state is not None
assert result.terminal_state.source == "ws"
assert client.calls == 0
def test_fill_monitor_rejects_invalid_configuration() -> None:
client = _FakePollClient(responses=[])
with pytest.raises(ValueError, match="poll_interval_seconds"):
FillMonitor(client, poll_interval_seconds=0.0)
with pytest.raises(ValueError, match="max_wait_seconds"):
FillMonitor(client, max_wait_seconds=0.0)
+109
View File
@@ -0,0 +1,109 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any
import pytest
from arbitrade.detection.engine import OpportunityEvent
from arbitrade.execution.idempotency import IdempotencyKeyFactory, OrderReconciler
from arbitrade.execution.sequencer import ExecutionLeg
@dataclass(slots=True)
class _FakeHistoryClient:
response: dict[str, Any]
async def query_order(self, *, order_id: str, include_trades: bool = True) -> dict[str, Any]:
return self.response
def _sample_event() -> OpportunityEvent:
return OpportunityEvent(
detected_at=datetime.now(UTC),
cycle="USD->BTC->ETH->USD",
updated_pair="BTC/USD",
gross_rate=1.02,
net_rate=1.01,
gross_pct=2.0,
net_pct=1.0,
est_profit=1.0,
allocated_capital=10.0,
)
def test_idempotency_key_factory_is_deterministic() -> None:
factory = IdempotencyKeyFactory()
event = _sample_event()
leg = ExecutionLeg(
from_currency="USD",
to_currency="BTC",
pair="BTC/USD",
side="buy",
volume=10.0,
)
first = factory.user_ref_for_leg(event, leg, 0)
second = factory.user_ref_for_leg(event, leg, 0)
assert first == second
assert first > 0
@pytest.mark.asyncio
async def test_order_reconciler_maps_query_response_to_report() -> None:
client = _FakeHistoryClient(
response={
"order-1": {
"status": "closed",
"vol_exec": "5.0",
"price": "100.0",
"pair": "BTC/USD",
"type": "buy",
"userref": 12345,
}
}
)
reconciler = OrderReconciler(client)
report = await reconciler.reconcile_order(
order_id="order-1",
user_ref=12345,
expected_pair="BTC/USD",
expected_side="buy",
expected_volume=10.0,
)
assert report.is_terminal
assert report.matches_request
assert report.status == "closed"
assert report.filled_volume == 5.0
assert report.avg_price == 100.0
@pytest.mark.asyncio
async def test_order_reconciler_marks_mismatch() -> None:
client = _FakeHistoryClient(
response={
"order-1": {
"status": "closed",
"vol_exec": "5.0",
"price": "100.0",
"pair": "ETH/USD",
"type": "sell",
"userref": 999,
}
}
)
reconciler = OrderReconciler(client)
report = await reconciler.reconcile_order(
order_id="order-1",
user_ref=12345,
expected_pair="BTC/USD",
expected_side="buy",
expected_volume=10.0,
)
assert not report.matches_request
+136
View File
@@ -110,3 +110,139 @@ def test_compliance_detects_insecure_config() -> None:
assert any("below 1.0" in issue for issue in issues)
assert any("ATTEMPTS" in issue for issue in issues)
assert any("BASE_DELAY" in issue for issue in issues)
@pytest.mark.asyncio
async def test_place_market_order_posts_add_order_payload() -> None:
settings = Settings(
_env_file=None,
KRAKEN_API_KEY="key",
KRAKEN_API_SECRET="c2VjcmV0", # base64("secret")
kraken_private_rate_limit_seconds=0.0,
)
client = KrakenRestClient(settings)
with respx.mock(base_url=settings.kraken_rest_url) as mock_router:
route = mock_router.post("/0/private/AddOrder").respond(
200,
json={"error": [], "result": {"txid": ["m1"]}},
)
payload = await client.place_market_order(
pair="XBTUSD",
side="buy",
volume=0.05,
)
await client.close()
request_body = route.calls.last.request.content.decode()
assert "pair=XBTUSD" in request_body
assert "type=buy" in request_body
assert "ordertype=market" in request_body
assert "volume=0.05" in request_body
assert payload["txid"] == ["m1"]
@pytest.mark.asyncio
async def test_place_limit_order_posts_add_order_payload() -> None:
settings = Settings(
_env_file=None,
KRAKEN_API_KEY="key",
KRAKEN_API_SECRET="c2VjcmV0", # base64("secret")
kraken_private_rate_limit_seconds=0.0,
)
client = KrakenRestClient(settings)
with respx.mock(base_url=settings.kraken_rest_url) as mock_router:
route = mock_router.post("/0/private/AddOrder").respond(
200,
json={"error": [], "result": {"txid": ["l1"]}},
)
payload = await client.place_limit_order(
pair="ETHUSD",
side="sell",
volume=1.5,
price=3500.0,
)
await client.close()
request_body = route.calls.last.request.content.decode()
assert "pair=ETHUSD" in request_body
assert "type=sell" in request_body
assert "ordertype=limit" in request_body
assert "price=3500.0" in request_body
assert "volume=1.5" in request_body
assert payload["txid"] == ["l1"]
@pytest.mark.asyncio
async def test_place_order_validates_inputs() -> None:
settings = Settings(
_env_file=None,
KRAKEN_API_KEY="key",
KRAKEN_API_SECRET="c2VjcmV0", # base64("secret")
kraken_private_rate_limit_seconds=0.0,
)
client = KrakenRestClient(settings)
with pytest.raises(ValueError, match="side"):
await client.place_market_order(pair="XBTUSD", side="hold", volume=0.1)
with pytest.raises(ValueError, match="volume"):
await client.place_market_order(pair="XBTUSD", side="buy", volume=0.0)
with pytest.raises(ValueError, match="price"):
await client.place_limit_order(
pair="XBTUSD",
side="buy",
volume=0.1,
price=0.0,
)
await client.close()
@pytest.mark.asyncio
async def test_query_order_posts_query_orders_payload() -> None:
settings = Settings(
_env_file=None,
KRAKEN_API_KEY="key",
KRAKEN_API_SECRET="c2VjcmV0", # base64("secret")
kraken_private_rate_limit_seconds=0.0,
)
client = KrakenRestClient(settings)
with respx.mock(base_url=settings.kraken_rest_url) as mock_router:
route = mock_router.post("/0/private/QueryOrders").respond(
200,
json={"error": [], "result": {"order-1": {"status": "closed"}}},
)
payload = await client.query_order(order_id="order-1", include_trades=False)
await client.close()
request_body = route.calls.last.request.content.decode()
assert "txid=order-1" in request_body
assert "trades=false" in request_body
assert payload["order-1"]["status"] == "closed"
@pytest.mark.asyncio
async def test_cancel_order_posts_cancel_order_payload() -> None:
settings = Settings(
_env_file=None,
KRAKEN_API_KEY="key",
KRAKEN_API_SECRET="c2VjcmV0", # base64("secret")
kraken_private_rate_limit_seconds=0.0,
)
client = KrakenRestClient(settings)
with respx.mock(base_url=settings.kraken_rest_url) as mock_router:
route = mock_router.post("/0/private/CancelOrder").respond(
200,
json={"error": [], "result": {"count": 1}},
)
payload = await client.cancel_order(order_id="order-1")
await client.close()
request_body = route.calls.last.request.content.decode()
assert "txid=order-1" in request_body
assert payload["count"] == 1
+96
View File
@@ -0,0 +1,96 @@
from __future__ import annotations
from dataclasses import dataclass
from datetime import UTC, datetime
from typing import Any
import pytest
from arbitrade.execution.fill_monitor import FillMonitorResult, OrderFillState
from arbitrade.execution.recovery import PartialFillRecovery
@dataclass(slots=True)
class _FakeRestClient:
cancel_calls: list[str] = None # type: ignore[assignment]
market_calls: list[dict[str, Any]] = None # type: ignore[assignment]
def __post_init__(self) -> None:
self.cancel_calls = []
self.market_calls = []
async def cancel_order(self, *, order_id: str) -> dict[str, Any]:
self.cancel_calls.append(order_id)
return {"result": {"count": 1}}
async def place_market_order(self, *, pair: str, side: str, volume: float) -> dict[str, Any]:
self.market_calls.append({"pair": pair, "side": side, "volume": volume})
return {"txid": ["hedge-1"]}
def _monitor_result(
*, status: str, filled_volume: float | None, timed_out: bool
) -> FillMonitorResult:
state = OrderFillState(
order_id="order-1",
status=status,
filled_volume=filled_volume,
avg_price=100.0,
updated_at=datetime.now(UTC),
source="rest_poll",
)
return FillMonitorResult(
order_id="order-1",
timed_out=timed_out,
terminal_state=None if status in {"open", "partial"} else state,
last_state=state,
elapsed_seconds=1.0,
)
@pytest.mark.asyncio
async def test_partial_fill_recovery_cancels_open_order_and_hedges_residual() -> None:
client = _FakeRestClient()
recovery = PartialFillRecovery(client)
result = await recovery.recover_partial_fill(
order_id="order-1",
pair="BTC/USD",
side="buy",
requested_volume=10.0,
fill_result=_monitor_result(status="partial", filled_volume=4.0, timed_out=True),
)
assert result.canceled
assert result.hedged
assert client.cancel_calls == ["order-1"]
assert client.market_calls == [{"pair": "BTC/USD", "side": "sell", "volume": 6.0}]
assert result.hedge_volume == 6.0
assert result.reason == "canceled_partial_order"
@pytest.mark.asyncio
async def test_partial_fill_recovery_no_hedge_when_no_residual() -> None:
client = _FakeRestClient()
recovery = PartialFillRecovery(client)
result = await recovery.recover_partial_fill(
order_id="order-1",
pair="BTC/USD",
side="sell",
requested_volume=5.0,
fill_result=_monitor_result(status="closed", filled_volume=5.0, timed_out=False),
)
assert not result.canceled
assert not result.hedged
assert client.cancel_calls == []
assert client.market_calls == []
def test_partial_fill_recovery_rejects_invalid_volume() -> None:
client = _FakeRestClient()
recovery = PartialFillRecovery(client)
with pytest.raises(ValueError, match="requested_volume"):
recovery._residual_volume(None, 0.0)