Qubx 0.6.93__tar.gz → 0.6.94__tar.gz

{qubx-0.6.93 → qubx-0.6.94}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Qubx
-Version: 0.6.93
+Version: 0.6.94
 Summary: Qubx - Quantitative Trading Framework
 License-File: LICENSE
 Author: Dmitry Marienko

{qubx-0.6.93 → qubx-0.6.94}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "Qubx"
-version = "0.6.93"
+version = "0.6.94"
 description = "Qubx - Quantitative Trading Framework"
 authors = [ "Dmitry Marienko <dmitry.marienko@xlydian.com>", "Yuriy Arabskyy <yuriy.arabskyy@xlydian.com>",]
 readme = "README.md"

{qubx-0.6.93 → qubx-0.6.94}/src/qubx/backtester/simulator.py

@@ -1,4 +1,4 @@
-from typing import Literal
+from typing import Literal, cast
 
 import pandas as pd
 from joblib import delayed
@@ -15,6 +15,7 @@ from qubx.utils.runner.configs import PrefetchConfig
 from qubx.utils.time import handle_start_stop, to_utc_naive
 
 from .runner import SimulationRunner
+from .transfers import SimulationTransferManager
 from .utils import (
     DataDecls_t,
     ExchangeName_t,
@@ -300,6 +301,18 @@ def _run_setup(
         if enable_inmemory_emitter and emitter is not None:
             emitter_data = emitter.get_dataframe()
 
+        # - get transfers log
+        transfers_log = None
+        if hasattr(runner.ctx, "_transfer_manager") and isinstance(
+            getattr(runner.ctx, "_transfer_manager"), SimulationTransferManager
+        ):
+            try:
+                transfer_manager = cast(SimulationTransferManager, getattr(runner.ctx, "_transfer_manager"))
+                transfers_log = transfer_manager.get_transfers_dataframe()
+            except Exception as e:
+                logger.error(f"Failed to get transfers log: {e}")
+                transfers_log = None
+
         return TradingSessionResult(
             setup_id,
             setup.name,
@@ -319,6 +332,7 @@ def _run_setup(
             is_simulation=True,
             author=get_current_user(),
             emitter_data=emitter_data,
+            transfers_log=transfers_log,
         )
     except Exception as e:
         logger.error(f"Simulation setup {setup_id} failed with error: {e}")

qubx-0.6.94/src/qubx/backtester/transfers.py

@@ -0,0 +1,146 @@
+import uuid
+from typing import Any
+
+import pandas as pd
+
+from qubx import logger
+from qubx.core.account import CompositeAccountProcessor
+from qubx.core.basics import ITimeProvider
+from qubx.core.interfaces import ITransferManager
+
+
+class SimulationTransferManager(ITransferManager):
+    """
+    Transfer manager for simulation mode.
+
+    Handles fund transfers between exchanges by directly manipulating account balances.
+    All transfers are instant and tracked in a DataFrame for export to results.
+    """
+
+    _account: CompositeAccountProcessor
+    _time: ITimeProvider
+    _transfers: list[dict[str, Any]]
+
+    def __init__(self, account_processor: CompositeAccountProcessor, time_provider: ITimeProvider):
+        """
+        Initialize simulation transfer manager.
+
+        Args:
+            account_processor: Account processor (typically CompositeAccountProcessor)
+            time_provider: Time provider for timestamping transfers
+        """
+        self._account = account_processor
+        self._time = time_provider
+        self._transfers = []
+
+    def transfer_funds(self, from_exchange: str, to_exchange: str, currency: str, amount: float) -> str:
+        """
+        Transfer funds between exchanges (instant in simulation).
+
+        Args:
+            from_exchange: Source exchange identifier
+            to_exchange: Destination exchange identifier
+            currency: Currency to transfer
+            amount: Amount to transfer
+
+        Returns:
+            str: Transaction ID (UUID)
+
+        Raises:
+            ValueError: If exchanges not found or insufficient funds
+        """
+        # Generate transaction ID
+        transaction_id = f"sim_{uuid.uuid4().hex[:12]}"
+
+        # Get timestamp
+        timestamp = self._time.time()
+
+        # Get individual processors
+        try:
+            from_processor = self._account.get_account_processor(from_exchange)
+            to_processor = self._account.get_account_processor(to_exchange)
+        except (KeyError, AttributeError) as e:
+            raise ValueError(f"Exchange not found: {e}")
+
+        # Validate sufficient funds
+        from_balances = from_processor.get_balances()
+        if currency not in from_balances:
+            raise ValueError(f"Currency '{currency}' not found in {from_exchange}")
+
+        available = from_balances[currency].free
+        if available < amount:
+            raise ValueError(
+                f"Insufficient funds in {from_exchange}: "
+                f"{available:.8f} {currency} available, {amount:.8f} {currency} requested"
+            )
+
+        # Execute transfer (instant balance manipulation)
+        from_balances[currency].total -= amount
+        from_balances[currency].free -= amount
+
+        to_balances = to_processor.get_balances()
+        to_balances[currency].total += amount
+        to_balances[currency].free += amount
+
+        # Record transfer
+        transfer_record = {
+            "transaction_id": transaction_id,
+            "timestamp": timestamp,
+            "from_exchange": from_exchange,
+            "to_exchange": to_exchange,
+            "currency": currency,
+            "amount": amount,
+            "status": "completed",  # Always completed in simulation
+        }
+        self._transfers.append(transfer_record)
+
+        logger.debug(f"[SimTransfer] {amount:.8f} {currency} {from_exchange} → {to_exchange} (ID: {transaction_id})")
+
+        return transaction_id
+
+    def get_transfer_status(self, transaction_id: str) -> dict[str, Any]:
+        """
+        Get the status of a transfer.
+
+        Args:
+            transaction_id: Transaction ID
+
+        Returns:
+            dict[str, Any]: Transfer status information
+        """
+        # Find transfer
+        for transfer in self._transfers:
+            if transfer["transaction_id"] == transaction_id:
+                return transfer.copy()
+
+        # Not found
+        return {
+            "transaction_id": transaction_id,
+            "status": "not_found",
+            "error": f"Transaction {transaction_id} not found",
+        }
+
+    def get_transfers(self) -> dict[str, dict[str, Any]]:
+        """
+        Get all transfers as a dictionary.
+
+        Returns:
+            dict[str, dict[str, Any]]: Dictionary mapping transaction IDs to transfer info
+        """
+        return {t["transaction_id"]: t for t in self._transfers}
+
+    def get_transfers_dataframe(self) -> pd.DataFrame:
+        """
+        Get all transfers as a pandas DataFrame.
+
+        Returns:
+            pd.DataFrame: DataFrame with columns [transaction_id, timestamp, from_exchange,
+                         to_exchange, currency, amount, status]
+        """
+        if not self._transfers:
+            # Return empty DataFrame with correct schema
+            return pd.DataFrame(
+                columns=["transaction_id", "from_exchange", "to_exchange", "currency", "amount", "status"]  # type: ignore
+            )
+
+        return pd.DataFrame(self._transfers).set_index("timestamp")

{qubx-0.6.93 → qubx-0.6.94}/src/qubx/connectors/xlighter/account.py

@@ -19,6 +19,7 @@ import numpy as np
 from qubx import logger
 from qubx.core.account import BasicAccountProcessor
 from qubx.core.basics import (
+    ZERO_COSTS,
     CtrlChannel,
     DataType,
     Deal,
@@ -65,7 +66,7 @@ class LighterAccountProcessor(BasicAccountProcessor):
         time_provider: ITimeProvider,
         loop: asyncio.AbstractEventLoop,
         base_currency: str = "USDC",
-        tcc: TransactionCostsCalculator = None,
+        tcc: TransactionCostsCalculator | None = None,
         initial_capital: float = 100_000,
         max_retries: int = 10,
         connection_timeout: int = 30,
@@ -88,8 +89,6 @@ class LighterAccountProcessor(BasicAccountProcessor):
             connection_timeout: Connection timeout in seconds
         """
         if tcc is None:
-            from qubx.core.basics import ZERO_COSTS
-
             tcc = ZERO_COSTS
 
         super().__init__(
@@ -121,12 +120,20 @@ class LighterAccountProcessor(BasicAccountProcessor):
         self._auth_token_expiry: Optional[int] = None
         self._processed_tx_hashes: set[str] = set()  # Track processed transaction hashes
 
+        self._account_stats_initialized = False
+
         logger.info(f"Initialized LighterAccountProcessor for account {account_id}")
 
     def set_subscription_manager(self, manager: ISubscriptionManager) -> None:
         """Set the subscription manager (required by interface)"""
         self._subscription_manager = manager
 
+    def get_total_capital(self, exchange: str | None = None) -> float:
+        if not self._account_stats_initialized:
+            self._async_loop.submit(self._start_subscriptions())
+            self._wait_for_account_stats_initialized()
+        return super().get_total_capital(exchange)
+
     def start(self):
         """Start WebSocket subscriptions for account data"""
         if self._is_running:
@@ -142,33 +149,13 @@ class LighterAccountProcessor(BasicAccountProcessor):
         # Start subscription tasks using AsyncThreadLoop
         logger.info("Starting Lighter account subscriptions")
 
-        # Submit connection and subscription tasks to the event loop
-        self._async_loop.submit(self._start_subscriptions())
+        if not self._account_stats_initialized:
+            # Submit connection and subscription tasks to the event loop
+            self._async_loop.submit(self._start_subscriptions())
+            self._wait_for_account_stats_initialized()
 
         logger.info("Lighter account subscriptions started")
 
-    async def _start_subscriptions(self):
-        """Connect to WebSocket and start all subscriptions"""
-        try:
-            # Ensure WebSocket is connected
-            if not self.ws_manager.is_connected:
-                logger.info("Connecting to Lighter WebSocket...")
-                await self.ws_manager.connect()
-                logger.info("Connected to Lighter WebSocket")
-
-            # Generate auth token for authenticated channels
-            await self._generate_auth_token()
-
-            # Start all subscriptions
-            await self._subscribe_account_all()
-            await self._subscribe_account_all_orders()
-            await self._subscribe_user_stats()
-
-        except Exception as e:
-            logger.error(f"Failed to start subscriptions: {e}")
-            self._is_running = False
-            raise
-
     def stop(self):
         """Stop all WebSocket subscriptions"""
         if not self._is_running:
@@ -183,164 +170,134 @@ class LighterAccountProcessor(BasicAccountProcessor):
 
         self._subscription_tasks.clear()
         self._is_running = False
-
         logger.info("Lighter account subscriptions stopped")
 
-    async def _generate_auth_token(self):
+    def process_deals(self, instrument: Instrument, deals: list[Deal], is_snapshot: bool = False) -> None:
         """
-        Generate authentication token for WebSocket subscriptions.
+        Override process_deals to track fees WITHOUT updating positions.
+
+        In Lighter, positions are synced directly from account_all channel
+        (single source of truth for quantity and avg_entry_price). However,
+        we still need to track fees/commissions from deals.
 
-        Auth tokens are required for account-specific channels:
-        - account_all/{account_id}
-        - account_all_orders/{account_id}
-        - user_stats/{account_id}
+        This prevents double position updates while ensuring commission tracking.
 
-        Note: create_auth_token_with_expiry() returns (auth_token_string, error_string)
-        where auth_token_string is the token itself (not an object).
-        Default expiry is 10 minutes from creation time.
+        Args:
+            instrument: The instrument for the deals
+            deals: List of Deal objects
+            is_snapshot: Whether this is a snapshot or incremental update
         """
-        try:
-            logger.info("Generating auth token...")
+        # Do NOT call super().process_deals() - that would update positions
+        # Instead, manually track fees for the position
+        if not deals:
+            return
+
+        position = self.get_position(instrument)
+
+        for deal in deals:
+            # Track commission from the deal
+            if deal.fee_amount and deal.fee_amount > 0:
+                # Add fee to position's commission tracking
+                position.commissions += deal.fee_amount
 
-            # Use SignerClient to create auth token
-            # Returns (token_string, error_string)
-            signer = self.client.signer_client
-            auth_token, error = signer.create_auth_token_with_expiry()
+                logger.debug(
+                    f"Tracked fee for {instrument.symbol}: {deal.fee_amount:.6f} {deal.fee_currency} "
+                    f"(total commissions: {position.commissions:.6f})"
+                )
+
+        logger.debug(
+            f"Processed {len(deals)} deal(s) for {instrument.symbol} - fees tracked, positions synced from account_all"
+        )
+
+    def process_order(self, order: Order, update_locked_value: bool = True) -> None:
+        """
+        Override process_order to handle Lighter's server-assigned order IDs.
+
+        Lighter assigns server IDs different from our client_id. When an order
+        update arrives with a new server ID but matching client_id, we need to
+        migrate the order from client_id key to server_id key while preserving
+        the same object instance (for external references).
+
+        Args:
+            order: Order update from WebSocket
+            update_locked_value: Whether to update locked capital tracking
+        """
+        # Check if order exists under client_id (migration case)
+        if order.client_id and order.client_id in self._active_orders:
+            # Get the existing order stored under client_id
+            existing_order = self._active_orders[order.client_id]
+
+            logger.debug(f"Migrating order: client_id={order.client_id} → server_id={order.id}")
+
+            # Remove from old location
+            self._active_orders.pop(order.client_id)
 
-            if error:
-                raise RuntimeError(f"Failed to generate auth token: {error}")
+            # Store it under the new server ID before base class processing
+            # This allows base class merge logic to find and update it in place
+            self._active_orders[order.id] = existing_order
 
-            if not auth_token:
-                raise RuntimeError("Auth token is empty")
+            # Also migrate locked capital tracking if present
+            if order.client_id in self._locked_capital_by_order:
+                locked_value = self._locked_capital_by_order.pop(order.client_id)
+                self._locked_capital_by_order[order.id] = locked_value
 
-            # Store token (it's already a string, not an object)
-            self._auth_token = auth_token
-            # Calculate expiry (default is 10 minutes from now)
-            self._auth_token_expiry = int(time.time()) + 10 * 60
+        # Let base class handle the rest (merge, store, lock/unlock, etc.)
+        # The base class will now find the existing order under order.id and merge in place
+        super().process_order(order, update_locked_value)
 
-            logger.info(f"Auth token generated successfully (expires at: {self._auth_token_expiry})")
+    def _wait_for_account_stats_initialized(self):
+        max_wait_time = 20.0  # seconds
+        elapsed = 0.0
+        interval = 0.1
+        while not self._account_stats_initialized:
+            if elapsed >= max_wait_time:
+                raise TimeoutError(f"Account stats were not initialized within {max_wait_time} seconds")
+            time.sleep(interval)
+            elapsed += interval
+
+    async def _start_subscriptions(self):
+        """Connect to WebSocket and start all subscriptions"""
+        try:
+            # Ensure WebSocket is connected
+            if not self.ws_manager.is_connected:
+                logger.info("Connecting to Lighter WebSocket...")
+                await self.ws_manager.connect()
+                logger.info("Connected to Lighter WebSocket")
+
+            # Start all subscriptions
+            await self._subscribe_account_all()
+            await self._subscribe_account_all_orders()
+            await self._subscribe_user_stats()
 
         except Exception as e:
-            logger.error(f"Failed to generate auth token: {e}")
+            logger.error(f"Failed to start subscriptions: {e}")
+            self._is_running = False
             raise
 
     async def _subscribe_account_all(self):
-        """
-        Subscribe to account_all channel for positions and trades (primary channel).
-
-        Requires authentication token.
-
-        This is the single source of truth for positions and trade history.
-        Updates positions directly from position data, sends trades as Deals
-        through channel for strategy notification.
-
-        Message format:
-        {
-            "account": 225671,
-            "channel": "account_all:225671",
-            "type": "update/account_all",
-            "positions": {
-                "24": {
-                    "market_id": 24,
-                    "sign": -1,  # 1 for long, -1 for short
-                    "position": "1.00",
-                    "avg_entry_price": "40.1342",
-                    ...
-                }
-            },
-            "trades": {
-                "24": [
-                    {
-                        "trade_id": 225067334,
-                        "market_id": 24,
-                        "size": "1.00",
-                        "price": "40.1342",
-                        "timestamp": 1760287839079,
-                        ...
-                    }
-                ]
-            },
-            "funding_histories": {}
-        }
-        """
-        channel = f"account_all/{self._lighter_account_index}"
-        logger.info(f"Subscribing to {channel} (with auth)")
-
         try:
-            # Subscribe with auth token
-            await self.ws_manager.subscribe(
-                channel=channel, handler=self._handle_account_all_message, auth=self._auth_token
-            )
-            logger.info(f"Successfully subscribed to {channel}")
+            await self.ws_manager.subscribe_account_all(self._lighter_account_index, self._handle_account_all_message)
+            logger.info(f"Subscribed to account_all for account {self._lighter_account_index}")
         except Exception as e:
-            logger.error(f"Failed to subscribe to {channel}: {e}")
+            logger.error(f"Failed to subscribe to account_all for account {self._lighter_account_index}: {e}")
             raise
 
     async def _subscribe_account_all_orders(self):
-        """
-        Subscribe to account_all_orders channel for order updates across all markets.
-
-        Requires authentication token.
-
-        Message format:
-        {
-            "channel": "account_all_orders:225671",
-            "type": "update/account_all_orders",
-            "orders": {
-                "24": [  # market_index
-                    {
-                        "order_id": "7036874567748225",
-                        "status": "filled",
-                        ...
-                    }
-                ]
-            }
-        }
-        """
-        channel = f"account_all_orders/{self._lighter_account_index}"
-        logger.info(f"Subscribing to {channel} (with auth)")
-
         try:
-            # Subscribe with auth token
-            await self.ws_manager.subscribe(
-                channel=channel, handler=self._handle_account_all_orders_message, auth=self._auth_token
+            await self.ws_manager.subscribe_account_all_orders(
+                self._lighter_account_index, self._handle_account_all_orders_message
             )
-            logger.info(f"Successfully subscribed to {channel}")
+            logger.info(f"Subscribed to account_all_orders for account {self._lighter_account_index}")
         except Exception as e:
-            logger.error(f"Failed to subscribe to {channel}: {e}")
+            logger.error(f"Failed to subscribe to account_all_orders for account {self._lighter_account_index}: {e}")
             raise
 
     async def _subscribe_user_stats(self):
-        """
-        Subscribe to user_stats channel for account statistics.
-
-        Requires authentication token.
-
-        Message format:
-        {
-            "channel": "user_stats:225671",
-            "type": "update/user_stats",
-            "stats": {
-                "collateral": "998.888700",
-                "portfolio_value": "998.901500",
-                "available_balance": "990.920600",
-                "leverage": "0.04",
-                "margin_usage": "0.80",
-                ...
-            }
-        }
-        """
-        channel = f"user_stats/{self._lighter_account_index}"
-        logger.info(f"Subscribing to {channel} (with auth)")
-
         try:
-            # Subscribe with auth token
-            await self.ws_manager.subscribe(
-                channel=channel, handler=self._handle_user_stats_message, auth=self._auth_token
-            )
-            logger.info(f"Successfully subscribed to {channel}")
+            await self.ws_manager.subscribe_user_stats(self._lighter_account_index, self._handle_user_stats_message)
+            logger.info(f"Subscribed to user_stats for account {self._lighter_account_index}")
         except Exception as e:
-            logger.error(f"Failed to subscribe to {channel}: {e}")
+            logger.error(f"Failed to subscribe to user_stats for account {self._lighter_account_index}: {e}")
             raise
 
     async def _handle_account_all_message(self, message: dict):
@@ -369,6 +326,8 @@ class LighterAccountProcessor(BasicAccountProcessor):
                 # Sync quantity and position_avg_price from Lighter's authoritative data
                 position.quantity = pos_state.quantity
                 position.position_avg_price = pos_state.avg_entry_price
+                position.position_avg_price_funds = pos_state.avg_entry_price
+                position.r_pnl = pos_state.realized_pnl
 
                 # Update market price for unrealized PnL recalculation
                 # Use the avg_entry_price as a reference if no better price available
@@ -393,15 +352,15 @@ class LighterAccountProcessor(BasicAccountProcessor):
                     f"fee={deal.fee_amount:.6f} (id={deal.id})"
                 )
 
+            # Funding payments are handled by the data provider, so I commented them here to avoid double sending
             # Send funding payments through channel
-            for instrument, payments in funding_payments.items():
-                for payment in payments:
-                    # Send: (instrument, DataType.FUNDING_PAYMENT, payment, False)
-                    self.channel.send((instrument, DataType.FUNDING_PAYMENT, payment, False))
-
-                    logger.debug(
-                        f"Sent funding payment: {instrument.symbol} rate={payment.funding_rate:.6f} at {payment.time}"
-                    )
+            # for instrument, payments in funding_payments.items():
+            #     for payment in payments:
+            #         # Send: (instrument, DataType.FUNDING_PAYMENT, payment, False)
+            #         self.channel.send((instrument, DataType.FUNDING_PAYMENT, payment, False))
+            #         logger.debug(
+            #             f"Sent funding payment: {instrument.symbol} rate={payment.funding_rate:.6f} at {payment.time}"
+            #         )
 
         except Exception as e:
             logger.error(f"Error handling account_all message: {e}")
@@ -453,79 +412,10 @@ class LighterAccountProcessor(BasicAccountProcessor):
                     f"free={balance.free:.2f}, locked={balance.locked:.2f}"
                 )
 
+            if not self._account_stats_initialized:
+                self._account_stats_initialized = True
+                logger.debug("Account stats initialized")
+
         except Exception as e:
             logger.error(f"Error handling user_stats message: {e}")
             logger.exception(e)
-
-    def process_deals(self, instrument: Instrument, deals: list[Deal], is_snapshot: bool = False) -> None:
-        """
-        Override process_deals to track fees WITHOUT updating positions.
-
-        In Lighter, positions are synced directly from account_all channel
-        (single source of truth for quantity and avg_entry_price). However,
-        we still need to track fees/commissions from deals.
-
-        This prevents double position updates while ensuring commission tracking.
-
-        Args:
-            instrument: The instrument for the deals
-            deals: List of Deal objects
-            is_snapshot: Whether this is a snapshot or incremental update
-        """
-        # Do NOT call super().process_deals() - that would update positions
-        # Instead, manually track fees for the position
-        if not deals:
-            return
-
-        position = self.get_position(instrument)
-
-        for deal in deals:
-            # Track commission from the deal
-            if deal.fee_amount and deal.fee_amount > 0:
-                # Add fee to position's commission tracking
-                position.commissions += deal.fee_amount
-
-                logger.debug(
-                    f"Tracked fee for {instrument.symbol}: {deal.fee_amount:.6f} {deal.fee_currency} "
-                    f"(total commissions: {position.commissions:.6f})"
-                )
-
-        logger.debug(
-            f"Processed {len(deals)} deal(s) for {instrument.symbol} - fees tracked, positions synced from account_all"
-        )
-
-    def process_order(self, order: Order, update_locked_value: bool = True) -> None:
-        """
-        Override process_order to handle Lighter's server-assigned order IDs.
-
-        Lighter assigns server IDs different from our client_id. When an order
-        update arrives with a new server ID but matching client_id, we need to
-        migrate the order from client_id key to server_id key while preserving
-        the same object instance (for external references).
-
-        Args:
-            order: Order update from WebSocket
-            update_locked_value: Whether to update locked capital tracking
-        """
-        # Check if order exists under client_id (migration case)
-        if order.client_id and order.client_id in self._active_orders:
-            # Get the existing order stored under client_id
-            existing_order = self._active_orders[order.client_id]
-
-            logger.debug(f"Migrating order: client_id={order.client_id} → server_id={order.id}")
-
-            # Remove from old location
-            self._active_orders.pop(order.client_id)
-
-            # Store it under the new server ID before base class processing
-            # This allows base class merge logic to find and update it in place
-            self._active_orders[order.id] = existing_order
-
-            # Also migrate locked capital tracking if present
-            if order.client_id in self._locked_capital_by_order:
-                locked_value = self._locked_capital_by_order.pop(order.client_id)
-                self._locked_capital_by_order[order.id] = locked_value
-
-        # Let base class handle the rest (merge, store, lock/unlock, etc.)
-        # The base class will now find the existing order under order.id and merge in place
-        super().process_order(order, update_locked_value)