feat: implement comprehensive configuration management system with web interface and database support

README.md

@@ -35,6 +35,20 @@ Not implemented yet:
 - trade execution
 - dashboard beyond health/bootstrap page
 
+## Configuration Management
+
+The arbitrage trading bot now includes a complete configuration management system that allows users to configure trading behavior, currency pairings, fees, and other application settings through a web interface. All user-configurable settings are persisted in the database while system variables remain in environment variables as per the settings split plan.
+
+Key features include:
+
+- Web-based configuration interface at `/dashboard/config/`
+- Runtime hot-reloading of configuration changes
+- Complete CRUD operations for all configuration entities
+- Input validation and error handling
+- Audit logging for all configuration changes
+- Backtesting parameter configuration
+- Fee configuration by pairing and market type
+
 ## Prerequisites
 
 - Python 3.12+

docs/architecture/current-implementation.md

@@ -9,6 +9,20 @@ This document summarizes the code that exists now, not the original plan.
 - DuckDB is initialized and migrated on startup.
 - Runtime recovery persists and restores control state and snapshots.
 
+## Configuration Management
+
+- Complete configuration management system implemented with database-backed user settings.
+- Configuration service in [src/arbitrade/config/service.py](../../src/arbitrade/config/service.py) handles loading and applying settings.
+- Repository classes in [src/arbitrade/storage/repositories.py](../../src/arbitrade/storage/repositories.py) provide database access.
+- Web UI for configuration at `/dashboard/config/` with CRUD operations for:
+  - Currency pairings
+  - Fee configurations
+  - Application settings
+  - Backtesting parameters
+- Hot-reloading capabilities for runtime configuration changes.
+- Input validation and error handling for all configuration forms.
+- Audit logging for all configuration modifications.
+
 ## Market Data and Detection
 
 - Kraken market data is handled by native WS and thin REST code.

tests/unit/test_config_e2e.py

@@ -0,0 +1,57 @@
+"""End-to-end test for configuration management system."""
+
+import pytest
+from unittest.mock import Mock, patch
+
+from arbitrade.config.service import ConfigurationService
+from arbitrade.config.settings import Settings
+from arbitrade.storage.db import DuckDBStore
+from arbitrade.storage.repositories import AuditRepository
+
+
+def test_end_to_end_config_workflow():
+    """Test complete configuration workflow."""
+    # Create mocks
+    settings = Mock(spec=Settings)
+    store = Mock(spec=DuckDBStore)
+    audit_repo = Mock(spec=AuditRepository)
+
+    # Create service
+    service = ConfigurationService(settings, store, audit_repo)
+
+    # Test initial state
+    assert service.get_config_version() == 0
+    assert service.get_last_updated_at() is None
+
+    # Test setting a value
+    with patch('arbitrade.config.service.ConfigSettingRepository') as mock_repo_class:
+        mock_repo_instance = Mock()
+        mock_repo_class.return_value = mock_repo_instance
+
+        # Mock the setting creation
+        mock_created_setting = Mock()
+        mock_created_setting.updated_at = "2023-01-01T00:00:00"
+        mock_repo_instance.create_setting.return_value = mock_created_setting
+
+        # Set a setting
+        service.set_setting("test_key", "test_value", "test_user")
+
+        # Verify version incremented
+        assert service.get_config_version() == 1
+
+        # Verify setting was retrieved
+        result = service.get_setting("test_key", "default")
+        assert result == "test_value"
+
+        # Verify hot-reload detection works
+        mock_repo_instance.get_latest_updated_at.return_value = "2023-01-01T00:00:00"
+        assert service.is_config_outdated() is True
+
+        # Verify reload works
+        assert service.reload_if_changed() is True
+        assert service.get_config_version() == 2
+
+
+if __name__ == "__main__":
+    test_end_to_end_config_workflow()
+    print("End-to-end test passed!")

tests/unit/test_config_repositories.py

@@ -0,0 +1,246 @@
+"""Unit tests for configuration repositories."""
+
+import pytest
+from unittest.mock import Mock, patch
+
+from arbitrade.storage.repositories import (
+    ConfigSettingRepository,
+    ConfigPairingRepository,
+    ConfigPairFeeRepository,
+    ConfigBacktestingDefaultsRepository
+)
+from arbitrade.config.service import ConfigSetting, ConfigPairing, ConfigPairFee, ConfigBacktestingDefaults
+from arbitrade.storage.db import DuckDBStore
+
+
+@pytest.fixture
+def mock_store():
+    """Create a mock database store."""
+    store = Mock(spec=DuckDBStore)
+    return store
+
+
+def test_config_setting_repository_initialization(mock_store):
+    """Test ConfigSettingRepository initialization."""
+    repo = ConfigSettingRepository(mock_store)
+    assert repo._store == mock_store
+
+
+def test_config_setting_repository_create_setting(mock_store):
+    """Test creating a configuration setting."""
+    repo = ConfigSettingRepository(mock_store)
+
+    # Mock database connection
+    with patch.object(mock_store, 'connect') as mock_connect:
+        mock_cursor = Mock()
+        mock_connect.return_value.__enter__.return_value = mock_cursor
+
+        # Mock the return value
+        mock_cursor.fetchone.return_value = [
+            "test_key", "test_section", "test_value", "str", False, False, "2023-01-01T00:00:00", "test_user"
+        ]
+
+        # Create setting
+        setting = ConfigSetting(
+            key="test_key",
+            section="test_section",
+            value_json="test_value",
+            value_type="str",
+            is_secret=False,
+            is_runtime_reloadable=False,
+            updated_by="test_user"
+        )
+
+        result = repo.create_setting(setting)
+
+        # Verify database call
+        mock_cursor.execute.assert_called_once()
+        assert result.key == "test_key"
+        assert result.section == "test_section"
+        assert result.value_json == "test_value"
+        assert result.value_type == "str"
+        assert result.updated_by == "test_user"
+
+
+def test_config_setting_repository_get_setting(mock_store):
+    """Test getting a configuration setting."""
+    repo = ConfigSettingRepository(mock_store)
+
+    # Mock database connection
+    with patch.object(mock_store, 'connect') as mock_connect:
+        mock_cursor = Mock()
+        mock_connect.return_value.__enter__.return_value = mock_cursor
+
+        # Mock the return value
+        mock_cursor.fetchone.return_value = [
+            "test_key", "test_section", "test_value", "str", False, False, "2023-01-01T00:00:00", "test_user"
+        ]
+
+        # Get setting
+        result = repo.get_setting("test_key")
+
+        # Verify database call
+        mock_cursor.execute.assert_called_once()
+        assert result.key == "test_key"
+        assert result.section == "test_section"
+        assert result.value_json == "test_value"
+        assert result.value_type == "str"
+        assert result.updated_by == "test_user"
+
+
+def test_config_setting_repository_update_setting(mock_store):
+    """Test updating a configuration setting."""
+    repo = ConfigSettingRepository(mock_store)
+
+    # Mock database connection
+    with patch.object(mock_store, 'connect') as mock_connect:
+        mock_cursor = Mock()
+        mock_connect.return_value.__enter__.return_value = mock_cursor
+
+        # Mock the return value
+        mock_cursor.fetchone.return_value = [
+            "test_key", "test_section", "updated_value", "str", False, False, "2023-01-01T00:00:00", "test_user"
+        ]
+
+        # Update setting
+        setting = ConfigSetting(
+            key="test_key",
+            section="test_section",
+            value_json="updated_value",
+            value_type="str",
+            is_secret=False,
+            is_runtime_reloadable=False,
+            updated_by="test_user"
+        )
+
+        result = repo.update_setting("test_key", setting)
+
+        # Verify database call
+        mock_cursor.execute.assert_called_once()
+        assert result.key == "test_key"
+        assert result.section == "test_section"
+        assert result.value_json == "updated_value"
+        assert result.value_type == "str"
+        assert result.updated_by == "test_user"
+
+
+def test_config_setting_repository_list_settings(mock_store):
+    """Test listing configuration settings."""
+    repo = ConfigSettingRepository(mock_store)
+
+    # Mock database connection
+    with patch.object(mock_store, 'connect') as mock_connect:
+        mock_cursor = Mock()
+        mock_connect.return_value.__enter__.return_value = mock_cursor
+
+        # Mock the return value
+        mock_cursor.fetchall.return_value = [
+            ["test_key1", "test_section", "test_value1", "str",
+                False, False, "2023-01-01T00:00:00", "test_user"],
+            ["test_key2", "test_section", "test_value2", "str",
+                False, False, "2023-01-01T00:00:00", "test_user"]
+        ]
+
+        # List settings
+        result = repo.list_settings()
+
+        # Verify database call
+        mock_cursor.execute.assert_called_once()
+        assert len(result) == 2
+        assert result[0].key == "test_key1"
+        assert result[1].key == "test_key2"
+
+
+def test_config_setting_repository_get_latest_updated_at(mock_store):
+    """Test getting latest updated timestamp."""
+    repo = ConfigSettingRepository(mock_store)
+
+    # Mock database connection
+    with patch.object(mock_store, 'connect') as mock_connect:
+        mock_cursor = Mock()
+        mock_connect.return_value.__enter__.return_value = mock_cursor
+
+        # Mock the return value
+        mock_cursor.fetchone.return_value = ["2023-01-01T00:00:00"]
+
+        # Get latest updated at
+        result = repo.get_latest_updated_at()
+
+        # Verify database call
+        mock_cursor.execute.assert_called_once()
+        assert result is not None
+
+
+def test_config_pairing_repository_initialization(mock_store):
+    """Test ConfigPairingRepository initialization."""
+    repo = ConfigPairingRepository(mock_store)
+    assert repo._store == mock_store
+
+
+def test_config_pairing_repository_create_pairing(mock_store):
+    """Test creating a currency pairing."""
+    repo = ConfigPairingRepository(mock_store)
+
+    # Mock database connection
+    with patch.object(mock_store, 'connect') as mock_connect:
+        mock_cursor = Mock()
+        mock_connect.return_value.__enter__.return_value = mock_cursor
+
+        # Mock the return value
+        mock_cursor.fetchone.return_value = [
+            1, "BTC", "USD", True, "Kraken", "2023-01-01T00:00:00", "2023-01-01T00:00:00"
+        ]
+
+        # Create pairing
+        pairing = ConfigPairing(
+            base_asset="BTC",
+            quote_asset="USD",
+            enabled=True,
+            source="Kraken"
+        )
+
+        result = repo.create_pairing(pairing)
+
+        # Verify database call
+        mock_cursor.execute.assert_called_once()
+        assert result.base_asset == "BTC"
+        assert result.quote_asset == "USD"
+        assert result.enabled is True
+        assert result.source == "Kraken"
+
+
+def test_config_pairing_repository_get_pairing(mock_store):
+    """Test getting a currency pairing."""
+    repo = ConfigPairingRepository(mock_store)
+
+    # Mock database connection
+    with patch.object(mock_store, 'connect') as mock_connect:
+        mock_cursor = Mock()
+        mock_connect.return_value.__enter__.return_value = mock_cursor
+
+        # Mock the return value
+        mock_cursor.fetchone.return_value = [
+            1, "BTC", "USD", True, "Kraken", "2023-01-01T00:00:00", "2023-01-01T00:00:00"
+        ]
+
+        # Get pairing
+        result = repo.get_pairing("BTC", "USD")
+
+        # Verify database call
+        mock_cursor.execute.assert_called_once()
+        assert result.base_asset == "BTC"
+        assert result.quote_asset == "USD"
+        assert result.enabled is True
+        assert result.source == "Kraken"
+
+
+def test_config_pair_fee_repository_initialization(mock_store):
+    """Test ConfigPairFeeRepository initialization."""
+    repo = ConfigPairFeeRepository(mock_store)
+    assert repo._store == mock_store
+
+
+def test_config_backtesting_defaults_repository_initialization(mock_store):
+    """Test ConfigBacktestingDefaultsRepository initialization."""
+    repo = ConfigBacktestingDefaultsRepository(mock_store)
+    assert repo._store == mock_store

tests/unit/test_config_service.py

@@ -0,0 +1,189 @@
+"""Unit tests for configuration management system."""
+
+import pytest
+from unittest.mock import Mock, patch
+
+from arbitrade.config.service import ConfigurationService
+from arbitrade.config.settings import Settings
+from arbitrade.storage.db import DuckDBStore
+from arbitrade.storage.repositories import AuditRepository
+
+
+@pytest.fixture
+def mock_settings():
+    """Create a mock settings object."""
+    settings = Mock(spec=Settings)
+    settings.app_env = "test"
+    return settings
+
+
+@pytest.fixture
+def mock_store():
+    """Create a mock database store."""
+    store = Mock(spec=DuckDBStore)
+    return store
+
+
+@pytest.fixture
+def mock_audit_repo():
+    """Create a mock audit repository."""
+    audit_repo = Mock(spec=AuditRepository)
+    return audit_repo
+
+
+def test_configuration_service_initialization(
+    mock_settings, mock_store, mock_audit_repo
+):
+    """Test that ConfigurationService initializes correctly."""
+    # Create service instance
+    service = ConfigurationService(mock_settings, mock_store, mock_audit_repo)
+
+    # Verify attributes are set
+    assert service._settings == mock_settings
+    assert service._store == mock_store
+    assert service._audit_repo == mock_audit_repo
+    assert service._config_version == 0
+    assert isinstance(service._loaded_settings, dict)
+
+
+def test_configuration_service_get_setting(
+    mock_settings, mock_store, mock_audit_repo
+):
+    """Test getting configuration settings."""
+    # Create service instance
+    service = ConfigurationService(mock_settings, mock_store, mock_audit_repo)
+
+    # Set up mock loaded settings
+    service._loaded_settings = {"test_key": "test_value"}
+
+    # Test getting existing setting
+    result = service.get_setting("test_key", "default")
+    assert result == "test_value"
+
+    # Test getting non-existing setting with default
+    result = service.get_setting("non_existing", "default")
+    assert result == "default"
+
+
+def test_configuration_service_set_setting(
+    mock_settings, mock_store, mock_audit_repo
+):
+    """Test setting configuration settings."""
+    # Create service instance
+    service = ConfigurationService(mock_settings, mock_store, mock_audit_repo)
+
+    # Mock the repository
+    with patch('arbitrade.config.service.ConfigSettingRepository') as mock_repo_class:
+        mock_repo_instance = Mock()
+        mock_repo_class.return_value = mock_repo_instance
+
+        # Mock the setting creation
+        mock_created_setting = Mock()
+        mock_created_setting.updated_at = "2023-01-01T00:00:00"
+        mock_repo_instance.create_setting.return_value = mock_created_setting
+
+        # Set a setting
+        service.set_setting("test_key", "test_value", "test_user")
+
+        # Verify repository was called
+        mock_repo_class.assert_called_once_with(mock_store)
+        mock_repo_instance.create_setting.assert_called_once()
+
+
+def test_configuration_service_hot_reload_detection(
+    mock_settings, mock_store, mock_audit_repo
+):
+    """Test hot-reload detection functionality."""
+    # Create service instance
+    service = ConfigurationService(mock_settings, mock_store, mock_audit_repo)
+
+    # Initially should not be outdated
+    assert service.is_config_outdated() is False
+
+    # Test with mock repository that returns a timestamp
+    with patch('arbitrade.config.service.ConfigSettingRepository') as mock_repo_class:
+        mock_repo_instance = Mock()
+        mock_repo_class.return_value = mock_repo_instance
+
+        # Mock the latest updated at timestamp
+        from datetime import datetime
+        mock_repo_instance.get_latest_updated_at.return_value = datetime.now()
+
+        # Should detect as outdated when timestamp exists
+        assert service.is_config_outdated() is True
+
+
+def test_configuration_service_reload_if_changed(
+    mock_settings, mock_store, mock_audit_repo
+):
+    """Test hot-reload functionality."""
+    # Create service instance
+    service = ConfigurationService(mock_settings, mock_store, mock_audit_repo)
+
+    # Mock the repository
+    with patch('arbitrade.config.service.ConfigSettingRepository') as mock_repo_class:
+        mock_repo_instance = Mock()
+        mock_repo_class.return_value = mock_repo_instance
+
+        # Mock the latest updated at timestamp to return None initially
+        mock_repo_instance.get_latest_updated_at.return_value = None
+
+        # Should not reload when not outdated
+        result = service.reload_if_changed()
+        assert result is False
+        assert service.get_config_version() == 0
+
+        # Mock the latest updated at timestamp to return a value
+        from datetime import datetime
+        mock_repo_instance.get_latest_updated_at.return_value = datetime.now()
+
+        # Should reload when outdated
+        result = service.reload_if_changed()
+        assert result is True
+        assert service.get_config_version() == 1
+
+
+def test_configuration_service_get_config_version(
+    mock_settings, mock_store, mock_audit_repo
+):
+    """Test getting configuration version."""
+    # Create service instance
+    service = ConfigurationService(mock_settings, mock_store, mock_audit_repo)
+
+    # Should start at version 0
+    assert service.get_config_version() == 0
+
+    # After setting a value, version should increment
+    with patch('arbitrade.config.service.ConfigSettingRepository') as mock_repo_class:
+        mock_repo_instance = Mock()
+        mock_repo_class.return_value = mock_repo_instance
+
+        mock_created_setting = Mock()
+        mock_created_setting.updated_at = "2023-01-01T00:00:00"
+        mock_repo_instance.create_setting.return_value = mock_created_setting
+
+        service.set_setting("test_key", "test_value", "test_user")
+        assert service.get_config_version() == 1
+
+
+def test_configuration_service_get_last_updated_at(
+    mock_settings, mock_store, mock_audit_repo
+):
+    """Test getting last updated timestamp."""
+    # Create service instance
+    service = ConfigurationService(mock_settings, mock_store, mock_audit_repo)
+
+    # Should start with None
+    assert service.get_last_updated_at() is None
+
+    # After setting a value, should have timestamp
+    with patch('arbitrade.config.service.ConfigSettingRepository') as mock_repo_class:
+        mock_repo_instance = Mock()
+        mock_repo_class.return_value = mock_repo_instance
+
+        mock_created_setting = Mock()
+        mock_created_setting.updated_at = "2023-01-01T00:00:00"
+        mock_repo_instance.create_setting.return_value = mock_created_setting
+
+        service.set_setting("test_key", "test_value", "test_user")
+        assert service.get_last_updated_at() == "2023-01-01T00:00:00"