Qubx 0.6.94__cp312-cp312-manylinux_2_39_x86_64.whl → 0.7.3__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/utils.py

@@ -155,7 +155,7 @@ class SimulatedLogFormatter:
             now = self.time_provider.time().astype("datetime64[us]").item().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
 
         # prefix = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> [ <level>%s</level> ] " % record["level"].icon
-        prefix = f"<lc>{now}</lc> [<level>{record['level'].icon}</level>] "
+        prefix = f"<lc>{now}</lc> [<level>{record['level'].icon}</level>] <cyan>({{module}})</cyan> "
 
         if record["exception"] is not None:
             record["extra"]["stack"] = stackprinter.format(record["exception"], style="darkbg3")

qubx/connectors/ccxt/data.py

@@ -43,18 +43,15 @@ class CcxtDataProvider(IDataProvider):
     ):
         # Store the exchange manager (always ExchangeManager now)
         self._exchange_manager = exchange_manager
-        
+
         self.time_provider = time_provider
         self.channel = channel
         self.max_ws_retries = max_ws_retries
         self._warmup_timeout = warmup_timeout
         self._health_monitor = health_monitor or DummyHealthMonitor()
 
-        self._data_arrival_listeners: List[IDataArrivalListener] = [
-            self._health_monitor,
-            self._exchange_manager
-        ]
-        
+        self._data_arrival_listeners: List[IDataArrivalListener] = [self._health_monitor, self._exchange_manager]
+
         logger.debug(f"Registered {len(self._data_arrival_listeners)} data arrival listeners")
 
         # Core components - access exchange directly via exchange_manager.exchange
@@ -106,10 +103,10 @@ class CcxtDataProvider(IDataProvider):
     def _loop(self) -> AsyncThreadLoop:
         """Get current AsyncThreadLoop for the exchange."""
         return AsyncThreadLoop(self._exchange_manager.exchange.asyncio_loop)
-    
+
     def notify_data_arrival(self, event_type: str, event_time: dt_64) -> None:
         """Notify all registered listeners about data arrival.
-        
+
         Args:
             event_type: Type of data event (e.g., "ohlcv:BTC/USDT:1m")
             event_time: Timestamp of the data event
@@ -120,10 +117,6 @@ class CcxtDataProvider(IDataProvider):
             except Exception as e:
                 logger.error(f"Error notifying data arrival listener {type(listener).__name__}: {e}")
 
-
-
-
-
     @property
     def is_simulation(self) -> bool:
         return False
@@ -219,7 +212,7 @@ class CcxtDataProvider(IDataProvider):
         self._warmup_service.execute_warmup(warmups)
 
     def get_quote(self, instrument: Instrument) -> Quote | None:
-        return self._last_quotes[instrument]
+        return self._last_quotes.get(instrument, None)
 
     def get_ohlc(self, instrument: Instrument, timeframe: str, nbarsback: int) -> List[Bar]:
         """Get historical OHLC data (delegated to OhlcDataHandler)."""
@@ -259,9 +252,9 @@ class CcxtDataProvider(IDataProvider):
                 except Exception as e:
                     logger.error(f"Error stopping subscription {subscription_type}: {e}")
 
-            # Stop ExchangeManager monitoring  
+            # Stop ExchangeManager monitoring
             self._exchange_manager.stop_monitoring()
-            
+
             # Close exchange connection
             if hasattr(self._exchange_manager.exchange, "close"):
                 future = self._loop.submit(self._exchange_manager.exchange.close())  # type: ignore
@@ -277,47 +270,57 @@ class CcxtDataProvider(IDataProvider):
 
     def _handle_exchange_recreation(self) -> None:
         """Handle exchange recreation by resubscribing to all active subscriptions."""
-        logger.info(f"<yellow>{self._exchange_id}</yellow> Handling exchange recreation - resubscribing to active subscriptions")
-        
+        logger.info(
+            f"<yellow>{self._exchange_id}</yellow> Handling exchange recreation - resubscribing to active subscriptions"
+        )
+
         # Get snapshot of current subscriptions before cleanup
         active_subscriptions = self._subscription_manager.get_subscriptions()
-        
+
         resubscription_data = []
         for subscription_type in active_subscriptions:
             instruments = self._subscription_manager.get_subscribed_instruments(subscription_type)
             if instruments:
                 resubscription_data.append((subscription_type, instruments))
-        
-        logger.info(f"<yellow>{self._exchange_id}</yellow> Found {len(resubscription_data)} active subscriptions to recreate")
-        
+
+        logger.info(
+            f"<yellow>{self._exchange_id}</yellow> Found {len(resubscription_data)} active subscriptions to recreate"
+        )
+
         # Track success/failure counts for reporting
         successful_resubscriptions = 0
         failed_resubscriptions = 0
-        
+
         # Clean resubscription: unsubscribe then subscribe for each subscription type
         for subscription_type, instruments in resubscription_data:
             try:
-                logger.info(f"<yellow>{self._exchange_id}</yellow> Resubscribing to {subscription_type} with {len(instruments)} instruments")
-                
+                logger.info(
+                    f"<yellow>{self._exchange_id}</yellow> Resubscribing to {subscription_type} with {len(instruments)} instruments"
+                )
+
                 self.unsubscribe(subscription_type, instruments)
 
                 # Resubscribe with reset=True to ensure clean state
                 self.subscribe(subscription_type, instruments, reset=True)
-                
+
                 successful_resubscriptions += 1
                 logger.debug(f"<yellow>{self._exchange_id}</yellow> Successfully resubscribed to {subscription_type}")
-                
+
             except Exception as e:
                 failed_resubscriptions += 1
                 logger.error(f"<yellow>{self._exchange_id}</yellow> Failed to resubscribe to {subscription_type}: {e}")
                 # Continue with other subscriptions even if one fails
-        
+
         # Report final status
         total_subscriptions = len(resubscription_data)
         if failed_resubscriptions == 0:
-            logger.info(f"<yellow>{self._exchange_id}</yellow> Exchange recreation resubscription completed successfully ({total_subscriptions}/{total_subscriptions})")
+            logger.info(
+                f"<yellow>{self._exchange_id}</yellow> Exchange recreation resubscription completed successfully ({total_subscriptions}/{total_subscriptions})"
+            )
         else:
-            logger.warning(f"<yellow>{self._exchange_id}</yellow> Exchange recreation resubscription completed with errors ({successful_resubscriptions}/{total_subscriptions} successful)")
+            logger.warning(
+                f"<yellow>{self._exchange_id}</yellow> Exchange recreation resubscription completed with errors ({successful_resubscriptions}/{total_subscriptions} successful)"
+            )
 
     @property
     def subscribed_instruments(self) -> Set[Instrument]:

qubx/connectors/xlighter/account.py

@@ -21,7 +21,6 @@ from qubx.core.account import BasicAccountProcessor
 from qubx.core.basics import (
     ZERO_COSTS,
     CtrlChannel,
-    DataType,
     Deal,
     Instrument,
     ITimeProvider,
@@ -121,8 +120,9 @@ class LighterAccountProcessor(BasicAccountProcessor):
         self._processed_tx_hashes: set[str] = set()  # Track processed transaction hashes
 
         self._account_stats_initialized = False
+        self._account_positions_initialized = False
 
-        logger.info(f"Initialized LighterAccountProcessor for account {account_id}")
+        self.__info(f"Initialized LighterAccountProcessor for account {account_id}")
 
     def set_subscription_manager(self, manager: ISubscriptionManager) -> None:
         """Set the subscription manager (required by interface)"""
@@ -131,37 +131,37 @@ class LighterAccountProcessor(BasicAccountProcessor):
     def get_total_capital(self, exchange: str | None = None) -> float:
         if not self._account_stats_initialized:
             self._async_loop.submit(self._start_subscriptions())
-            self._wait_for_account_stats_initialized()
+            self._wait_for_account_initialized()
         return super().get_total_capital(exchange)
 
     def start(self):
         """Start WebSocket subscriptions for account data"""
         if self._is_running:
-            logger.debug("Account processor is already running")
+            self.__debug("Account processor is already running")
             return
 
         if not self.channel or not self.channel.control.is_set():
-            logger.warning("Channel not set or control not active, cannot start")
+            self.__warning("Channel not set or control not active, cannot start")
             return
 
         self._is_running = True
 
         # Start subscription tasks using AsyncThreadLoop
-        logger.info("Starting Lighter account subscriptions")
+        self.__info("Starting Lighter account subscriptions")
 
         if not self._account_stats_initialized:
             # Submit connection and subscription tasks to the event loop
             self._async_loop.submit(self._start_subscriptions())
-            self._wait_for_account_stats_initialized()
+            self._wait_for_account_initialized()
 
-        logger.info("Lighter account subscriptions started")
+        self.__info("Lighter account subscriptions started")
 
     def stop(self):
         """Stop all WebSocket subscriptions"""
         if not self._is_running:
             return
 
-        logger.info("Stopping Lighter account subscriptions")
+        self.__info("Stopping Lighter account subscriptions")
 
         # Cancel all subscription tasks
         for task in self._subscription_tasks:
@@ -170,7 +170,7 @@ class LighterAccountProcessor(BasicAccountProcessor):
 
         self._subscription_tasks.clear()
         self._is_running = False
-        logger.info("Lighter account subscriptions stopped")
+        self.__info("Lighter account subscriptions stopped")
 
     def process_deals(self, instrument: Instrument, deals: list[Deal], is_snapshot: bool = False) -> None:
         """
@@ -200,12 +200,12 @@ class LighterAccountProcessor(BasicAccountProcessor):
                 # Add fee to position's commission tracking
                 position.commissions += deal.fee_amount
 
-                logger.debug(
+                self.__debug(
                     f"Tracked fee for {instrument.symbol}: {deal.fee_amount:.6f} {deal.fee_currency} "
                     f"(total commissions: {position.commissions:.6f})"
                 )
 
-        logger.debug(
+        self.__debug(
             f"Processed {len(deals)} deal(s) for {instrument.symbol} - fees tracked, positions synced from account_all"
         )
 
@@ -227,7 +227,7 @@ class LighterAccountProcessor(BasicAccountProcessor):
             # Get the existing order stored under client_id
             existing_order = self._active_orders[order.client_id]
 
-            logger.debug(f"Migrating order: client_id={order.client_id} → server_id={order.id}")
+            self.__debug(f"Migrating order: client_id={order.client_id} → server_id={order.id}")
 
             # Remove from old location
             self._active_orders.pop(order.client_id)
@@ -245,13 +245,13 @@ class LighterAccountProcessor(BasicAccountProcessor):
         # The base class will now find the existing order under order.id and merge in place
         super().process_order(order, update_locked_value)
 
-    def _wait_for_account_stats_initialized(self):
+    def _wait_for_account_initialized(self):
         max_wait_time = 20.0  # seconds
         elapsed = 0.0
         interval = 0.1
-        while not self._account_stats_initialized:
+        while not self._account_stats_initialized or not self._account_positions_initialized:
             if elapsed >= max_wait_time:
-                raise TimeoutError(f"Account stats were not initialized within {max_wait_time} seconds")
+                raise TimeoutError(f"Account was not initialized within {max_wait_time} seconds")
             time.sleep(interval)
             elapsed += interval
 
@@ -260,9 +260,9 @@ class LighterAccountProcessor(BasicAccountProcessor):
         try:
             # Ensure WebSocket is connected
             if not self.ws_manager.is_connected:
-                logger.info("Connecting to Lighter WebSocket...")
+                self.__info("Connecting to Lighter WebSocket...")
                 await self.ws_manager.connect()
-                logger.info("Connected to Lighter WebSocket")
+                self.__info("Connected to Lighter WebSocket")
 
             # Start all subscriptions
             await self._subscribe_account_all()
@@ -270,16 +270,16 @@ class LighterAccountProcessor(BasicAccountProcessor):
             await self._subscribe_user_stats()
 
         except Exception as e:
-            logger.error(f"Failed to start subscriptions: {e}")
+            self.__error(f"Failed to start subscriptions: {e}")
             self._is_running = False
             raise
 
     async def _subscribe_account_all(self):
         try:
             await self.ws_manager.subscribe_account_all(self._lighter_account_index, self._handle_account_all_message)
-            logger.info(f"Subscribed to account_all for account {self._lighter_account_index}")
+            self.__info(f"Subscribed to account_all for account {self._lighter_account_index}")
         except Exception as e:
-            logger.error(f"Failed to subscribe to account_all for account {self._lighter_account_index}: {e}")
+            self.__error(f"Failed to subscribe to account_all for account {self._lighter_account_index}: {e}")
             raise
 
     async def _subscribe_account_all_orders(self):
@@ -287,17 +287,17 @@ class LighterAccountProcessor(BasicAccountProcessor):
             await self.ws_manager.subscribe_account_all_orders(
                 self._lighter_account_index, self._handle_account_all_orders_message
             )
-            logger.info(f"Subscribed to account_all_orders for account {self._lighter_account_index}")
+            self.__info(f"Subscribed to account_all_orders for account {self._lighter_account_index}")
         except Exception as e:
-            logger.error(f"Failed to subscribe to account_all_orders for account {self._lighter_account_index}: {e}")
+            self.__error(f"Failed to subscribe to account_all_orders for account {self._lighter_account_index}: {e}")
             raise
 
     async def _subscribe_user_stats(self):
         try:
             await self.ws_manager.subscribe_user_stats(self._lighter_account_index, self._handle_user_stats_message)
-            logger.info(f"Subscribed to user_stats for account {self._lighter_account_index}")
+            self.__info(f"Subscribed to user_stats for account {self._lighter_account_index}")
         except Exception as e:
-            logger.error(f"Failed to subscribe to user_stats for account {self._lighter_account_index}: {e}")
+            self.__error(f"Failed to subscribe to user_stats for account {self._lighter_account_index}: {e}")
             raise
 
     async def _handle_account_all_message(self, message: dict):
@@ -335,11 +335,15 @@ class LighterAccountProcessor(BasicAccountProcessor):
                 if position.last_update_price == 0 or np.isnan(position.last_update_price):
                     position.last_update_price = pos_state.avg_entry_price
 
-                logger.debug(
+                self.__debug(
                     f"Synced position: {instrument.symbol} = {pos_state.quantity:+.4f} "
                     f"@ avg_price={pos_state.avg_entry_price:.4f}"
                 )
 
+            if not self._account_positions_initialized:
+                self._account_positions_initialized = True
+                self.__info("Account positions initialized")
+
             # Send deals through channel for strategy notification
             # Note: process_deals is overridden to track fees without updating positions
             for instrument, deal in deals:
@@ -347,7 +351,7 @@ class LighterAccountProcessor(BasicAccountProcessor):
                 # False means not a snapshot
                 self.channel.send((instrument, "deals", [deal], False))
 
-                logger.debug(
+                self.__debug(
                     f"Sent deal: {instrument.symbol} {deal.amount:+.4f} @ {deal.price:.4f} "
                     f"fee={deal.fee_amount:.6f} (id={deal.id})"
                 )
@@ -363,7 +367,7 @@ class LighterAccountProcessor(BasicAccountProcessor):
             #         )
 
         except Exception as e:
-            logger.error(f"Error handling account_all message: {e}")
+            self.__error(f"Error handling account_all message: {e}")
             logger.exception(e)
 
     async def _handle_account_all_orders_message(self, message: dict):
@@ -384,13 +388,13 @@ class LighterAccountProcessor(BasicAccountProcessor):
                 # False means not a snapshot
                 self.channel.send((instrument, "order", order, False))
 
-                logger.debug(
+                self.__debug(
                     f"Sent order: {instrument.symbol} {order.side} {order.quantity:+.4f} @ {order.price:.4f} "
                     f"[{order.status}] (order_id={order.id})"
                 )
 
         except Exception as e:
-            logger.error(f"Error handling account_all_orders message: {e}")
+            self.__error(f"Error handling account_all_orders message: {e}")
             logger.exception(e)
 
     async def _handle_user_stats_message(self, message: dict):
@@ -407,15 +411,30 @@ class LighterAccountProcessor(BasicAccountProcessor):
             for currency, balance in balances.items():
                 self.update_balance(currency, balance.total, balance.locked)
 
-                logger.debug(
+                self.__debug(
                     f"Updated balance - {currency}: total={balance.total:.2f}, "
                     f"free={balance.free:.2f}, locked={balance.locked:.2f}"
                 )
 
             if not self._account_stats_initialized:
                 self._account_stats_initialized = True
-                logger.debug("Account stats initialized")
+                self.__info("Account stats initialized")
 
         except Exception as e:
-            logger.error(f"Error handling user_stats message: {e}")
+            self.__error(f"Error handling user_stats message: {e}")
             logger.exception(e)
+
+    def __info(self, msg: str):
+        logger.info(self.__format(msg))
+
+    def __debug(self, msg: str):
+        logger.debug(self.__format(msg))
+
+    def __warning(self, msg: str):
+        logger.warning(self.__format(msg))
+
+    def __error(self, msg: str):
+        logger.error(self.__format(msg))
+
+    def __format(self, msg: str):
+        return f"<green>[Lighter]</green> {msg}"

qubx/connectors/xlighter/client.py

@@ -4,9 +4,11 @@ import asyncio
 import logging
 import threading
 import time
-from typing import Awaitable, Optional, TypeVar
+from typing import Awaitable, Optional, TypeVar, cast
 
+import pandas as pd
 from lighter import (  # type: ignore
+    AccountApi,
     ApiClient,
     CandlestickApi,
     Configuration,
@@ -20,8 +22,15 @@ from lighter import (  # type: ignore
 logging.root.setLevel(logging.WARNING)
 
 from qubx import logger
+from qubx.utils.rate_limiter import rate_limited
 
 from .constants import API_BASE_MAINNET, API_BASE_TESTNET
+from .rate_limits import (
+    WEIGHT_CANDLESTICKS,
+    WEIGHT_DEFAULT,
+    WEIGHT_FUNDING,
+    create_lighter_rate_limiters,
+)
 
 T = TypeVar("T")
 
@@ -54,6 +63,7 @@ class LighterClient:
     """
 
     _config: Configuration
+    _account_api: AccountApi
     _api_client: ApiClient
     _info_api: InfoApi
     _order_api: OrderApi
@@ -69,6 +79,8 @@ class LighterClient:
         api_key_index: int = 0,
         testnet: bool = False,
         loop: asyncio.AbstractEventLoop | None = None,
+        account_type: str = "premium",
+        rest_rate_limit: int | None = None,
     ):
         """
         Initialize Lighter client.
@@ -79,6 +91,9 @@ class LighterClient:
             account_index: Lighter account index
             api_key_index: API key index for the account
             testnet: If True, use testnet. Otherwise mainnet.
+            loop: Event loop to use (optional)
+            account_type: "premium" or "standard" for rate limiting (default: "premium")
+            rest_rate_limit: Override REST API rate limit in requests/minute (optional)
         """
         self.api_key = api_key
         self.private_key = private_key.replace("0x", "")  # Remove 0x prefix if present
@@ -104,6 +119,7 @@ class LighterClient:
         async def _init_sdks():
             self._config = Configuration(host=self.api_url)
             self._api_client = ApiClient(configuration=self._config)
+            self._account_api = AccountApi(self._api_client)
             self._info_api = InfoApi(self._api_client)
             self._order_api = OrderApi(self._api_client)
             self._candlestick_api = CandlestickApi(self._api_client)
@@ -117,8 +133,15 @@ class LighterClient:
 
         asyncio.run_coroutine_threadsafe(_init_sdks(), self._loop).result()
 
+        # Initialize rate limiters
+        self._rate_limiters = create_lighter_rate_limiters(
+            account_type=account_type,
+            rest_rate_limit=rest_rate_limit,
+        )
+
         logger.info(
-            f"Initialized LighterClient (testnet={testnet}, account_index={account_index}, api_key_index={api_key_index})"
+            f"Initialized LighterClient (testnet={testnet}, account_index={account_index}, "
+            f"api_key_index={api_key_index}, account_type={account_type})"
         )
 
     def _run_event_loop(self):
@@ -140,6 +163,7 @@ class LighterClient:
         # Make it awaitable from the *caller*'s loop:
         return await asyncio.wrap_future(cfut)
 
+    @rate_limited("rest", weight=WEIGHT_DEFAULT)
     async def get_markets(self) -> list[dict]:
         """
         Get list of all markets.
@@ -184,6 +208,7 @@ class LighterClient:
                 return market
         return None
 
+    @rate_limited("rest", weight=WEIGHT_CANDLESTICKS)
     async def get_candlesticks(
         self,
         market_id: int,
@@ -218,8 +243,19 @@ class LighterClient:
                 resolution_ms = self._resolution_to_milliseconds(resolution)
                 start_timestamp = end_timestamp - (count_back * resolution_ms)
 
+            start_timestamp_str = (
+                cast(pd.Timestamp, pd.Timestamp(start_timestamp, unit="ms")).strftime("%Y-%m-%d %H:%M:%S")
+                if start_timestamp is not None
+                else None
+            )
+            end_timestamp_str = (
+                cast(pd.Timestamp, pd.Timestamp(end_timestamp, unit="ms")).strftime("%Y-%m-%d %H:%M:%S")
+                if end_timestamp is not None
+                else None
+            )
+
             logger.debug(
-                f"Fetching candlesticks for market {market_id}: {resolution}, from {start_timestamp} to {end_timestamp}"
+                f"[Lighter] Fetching candlesticks for market {market_id}: {resolution}, from {start_timestamp_str} to {end_timestamp_str}"
             )
 
             response = await self._run_on_client_loop(
@@ -245,6 +281,7 @@ class LighterClient:
             logger.error(f"Failed to get candlesticks for market {market_id}: {e}")
             raise
 
+    @rate_limited("rest", weight=WEIGHT_FUNDING)
     async def get_fundings(
         self,
         market_id: int,
@@ -304,6 +341,7 @@ class LighterClient:
             logger.error(f"Failed to get funding data for market {market_id}: {e}")
             raise
 
+    @rate_limited("rest", weight=WEIGHT_DEFAULT)
     async def get_funding_rates(self) -> dict[int, float]:
         """
         Get current funding rates for all markets.

qubx/connectors/xlighter/constants.py

@@ -53,6 +53,8 @@ WS_CHANNEL_ACCOUNT_ALL = "account_all"
 WS_CHANNEL_USER_STATS = "user_stats"
 WS_CHANNEL_EXECUTED_TX = "executed_transaction"
 
+WS_RESUBSCRIBE_DELAY = 2.0  # seconds to wait before resubscribing
+
 # WebSocket message types
 WS_MSG_TYPE_CONNECTED = "connected"
 WS_MSG_TYPE_SUBSCRIBE = "subscribe"
@@ -70,9 +72,11 @@ API_BASE_TESTNET = "https://testnet.zklighter.elliot.ai"
 WS_BASE_MAINNET = "wss://mainnet.zklighter.elliot.ai/stream"
 WS_BASE_TESTNET = "wss://testnet.zklighter.elliot.ai/stream"
 
-DEFAULT_PING_INTERVAL = 200
-DEFAULT_PING_TIMEOUT = 30
+DEFAULT_PING_INTERVAL = None
+DEFAULT_PING_TIMEOUT = None
 DEFAULT_MAX_RETRIES = 10
+DEFAULT_MAX_SIZE = None
+DEFAULT_MAX_QUEUE = 5000
 
 
 # Enums for type safety (kept for backward compatibility)