Qubx 0.6.94__tar.gz → 0.6.95__tar.gz

{qubx-0.6.94 → qubx-0.6.95}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Qubx
-Version: 0.6.94
+Version: 0.6.95
 Summary: Qubx - Quantitative Trading Framework
 License-File: LICENSE
 Author: Dmitry Marienko

{qubx-0.6.94 → qubx-0.6.95}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "Qubx"
-version = "0.6.94"
+version = "0.6.95"
 description = "Qubx - Quantitative Trading Framework"
 authors = [ "Dmitry Marienko <dmitry.marienko@xlydian.com>", "Yuriy Arabskyy <yuriy.arabskyy@xlydian.com>",]
 readme = "README.md"

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/connectors/xlighter/constants.py

@@ -70,9 +70,11 @@ API_BASE_TESTNET = "https://testnet.zklighter.elliot.ai"
 WS_BASE_MAINNET = "wss://mainnet.zklighter.elliot.ai/stream"
 WS_BASE_TESTNET = "wss://testnet.zklighter.elliot.ai/stream"
 
-DEFAULT_PING_INTERVAL = 200
-DEFAULT_PING_TIMEOUT = 30
+DEFAULT_PING_INTERVAL = None
+DEFAULT_PING_TIMEOUT = None
 DEFAULT_MAX_RETRIES = 10
+DEFAULT_MAX_SIZE = None
+DEFAULT_MAX_QUEUE = 5000
 
 
 # Enums for type safety (kept for backward compatibility)

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/connectors/xlighter/data.py

@@ -82,6 +82,9 @@ class LighterDataProvider(IDataProvider):
         # Track if market_stats:all is subscribed (single subscription for all instruments)
         self._market_stats_subscribed: bool = False
 
+        # Track if reconnection callback has been registered
+        self._reconnection_callback_registered: bool = False
+
         logger.info("LighterDataProvider initialized")
 
     @property
@@ -160,6 +163,25 @@ class LighterDataProvider(IDataProvider):
             f"Subscribed to {subscription_type} for {len(instruments)} instruments: {[i.symbol for i in instruments]}"
         )
 
+    async def _on_reconnected(self) -> None:
+        """
+        Callback invoked after WebSocket reconnection.
+
+        Resets all handler states to ensure clean state after reconnection.
+        This is particularly important for stateful handlers like OrderbookHandler
+        which maintain incremental state that becomes invalid after disconnection.
+        """
+        logger.info("WebSocket reconnected, resetting all handler states")
+
+        # Reset all handlers (stateless handlers have empty reset() implementation)
+        for handler in self._handlers.values():
+            try:
+                handler.reset()
+            except Exception as e:
+                logger.error(f"Error resetting handler {handler.__class__.__name__}: {e}")
+
+        logger.debug(f"Reset {len(self._handlers)} handlers after reconnection")
+
     def _ensure_websocket_connected(self, timeout: float = 5.0) -> None:
         """
         Ensure WebSocket is connected, wait if necessary.
@@ -192,6 +214,12 @@ class LighterDataProvider(IDataProvider):
             await self._ws_manager.connect()
             self._ws_connected = True
 
+            # Register reconnection callback (one-time setup)
+            if not self._reconnection_callback_registered:
+                self._ws_manager.on_reconnected(self._on_reconnected)
+                self._reconnection_callback_registered = True
+                logger.debug("Registered reconnection callback for handler state reset")
+
         # Submit and WAIT for connection
         future = self._async_loop.submit(_connect())
         try:
@@ -302,7 +330,8 @@ class LighterDataProvider(IDataProvider):
             handler = cast(OrderbookHandler, self._handlers.get(handler_key))
 
             if handler and handler.can_handle(message):
-                orderbook = cast(OrderBook, handler.handle(message))
+                orderbook = handler.handle(message)
+                orderbook = cast(OrderBook, orderbook)
                 if orderbook:
                     # Send to channel
                     self.channel.send((instrument, "orderbook", orderbook, False))

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/connectors/xlighter/handlers/base.py

@@ -102,3 +102,15 @@ class BaseHandler(ABC, Generic[T]):
         """Reset statistics counters"""
         self._message_count = 0
         self._error_count = 0
+
+    def reset(self):
+        """
+        Reset handler internal state.
+
+        This method is called when the WebSocket connection is reestablished
+        to ensure handlers start with clean state. Handlers with stateful
+        components should override this method to reset their state.
+
+        The default implementation does nothing (stateless handlers).
+        """
+        pass

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/connectors/xlighter/handlers/orderbook.py

@@ -58,7 +58,11 @@ class OrderbookHandler(BaseHandler[OrderBook]):
         self.max_levels = max_levels
         self.tick_size_pct = tick_size_pct
         self.instrument = instrument
-        self._state_manager = OrderBookStateManager()
+
+        # Initialize state manager with sufficient buffer size
+        # Use larger buffer to accommodate raw orderbook before aggregation
+        state_manager_max_levels = max(1000, 2 * max_levels)
+        self._state_manager = OrderBookStateManager(max_levels=state_manager_max_levels)
 
     def can_handle(self, message: dict[str, Any]) -> bool:
         channel = message.get("channel", "")
@@ -96,6 +100,10 @@ class OrderbookHandler(BaseHandler[OrderBook]):
             logger.warning("Missing order_book in message")
             return None
 
+        is_update = message.get("type") == "update/order_book"
+        if not is_update:
+            self._state_manager.reset()
+
         bids = book.get("bids", [])
         asks = book.get("asks", [])
         bids = [(float(bid["price"]), float(bid["size"])) for bid in bids]

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/connectors/xlighter/websocket.py

@@ -13,7 +13,6 @@ from qubx.utils.websocket_manager import BaseWebSocketManager, ReconnectionConfi
 from .client import LighterClient
 from .constants import (
     DEFAULT_MAX_RETRIES,
-    DEFAULT_PING_INTERVAL,
     DEFAULT_PING_TIMEOUT,
     WS_BASE_MAINNET,
     WS_BASE_TESTNET,
@@ -43,8 +42,9 @@ class LighterWebSocketManager(BaseWebSocketManager):
         client: LighterClient,
         testnet: bool = False,
         reconnection_config: Optional[ReconnectionConfig] = None,
-        ping_interval: float | None = DEFAULT_PING_INTERVAL,
+        ping_interval: float | None = None,
         ping_timeout: float | None = DEFAULT_PING_TIMEOUT,
+        application_ping_interval: float | None = 20.0,
     ):
         """
         Initialize Lighter WebSocket manager.
@@ -52,19 +52,25 @@ class LighterWebSocketManager(BaseWebSocketManager):
         Args:
             testnet: If True, connect to testnet. Otherwise mainnet.
             reconnection_config: Configuration for reconnection behavior
-            ping_interval: Interval between pings (seconds)
+            ping_interval: Interval for protocol-level pings (seconds), or None to disable (default: None)
             ping_timeout: Timeout for ping response (seconds)
+            application_ping_interval: Interval for application-level pings (seconds), or None to disable (default: 20.0)
         """
-        url = WS_BASE_TESTNET if testnet else WS_BASE_MAINNET
-
         if reconnection_config is None:
             reconnection_config = ReconnectionConfig(max_retries=DEFAULT_MAX_RETRIES)
 
         super().__init__(
-            url=url,
-            reconnection_config=reconnection_config,
-            ping_interval=ping_interval,
-            ping_timeout=ping_timeout,
+            url=WS_BASE_TESTNET if testnet else WS_BASE_MAINNET,
+            reconnection=reconnection_config,
+            ping_interval=None,  # use app-level pings
+            ping_timeout=None,
+            max_size=16 * 1024 * 1024,
+            max_queue=None,
+            compression=None,
+            inbox_size=5000,
+            workers=1,
+            app_ping_interval=20.0,  # send {"type":"ping"} every 20s
+            no_rx_reconnect_after=60.0,
         )
 
         self.testnet = testnet
@@ -334,11 +340,14 @@ class LighterWebSocketManager(BaseWebSocketManager):
         Returns:
             Channel identifier or None
         """
-        channel = message.get("channel")
-        if channel:
-            # Convert "order_book:0" back to "order_book/0"
-            channel = channel.replace(":", "/")
-        return channel
+        try:
+            channel = message.get("channel", None)
+            if channel:
+                # Convert "order_book:0" back to "order_book/0"
+                channel = channel.replace(":", "/")
+            return channel
+        except Exception:
+            return None
 
     async def _handle_error_message(self, error: dict) -> None:
         """
@@ -352,13 +361,16 @@ class LighterWebSocketManager(BaseWebSocketManager):
         """
         error_code = error.get("code", -1)
         error_message = error.get("message", "Unknown error")
-        logger.error(f"Lighter WebSocket error [{error_code}] {error_message}")
         match error_code:
             case 20013:
                 # expired token
+                logger.warning("Auth token expired, generating new one")
                 self._update_auth_token()
+            case 30003:
+                # Alread subscribed
+                logger.debug(f"Already subscribed to {error['channel']}")
             case _:
-                pass
+                logger.warning(f"Lighter WebSocket error [{error_code}] {error_message}")
 
     def _update_auth_token(self):
         """
@@ -423,6 +435,9 @@ class LighterWebSocketManager(BaseWebSocketManager):
             if self._on_connected_callback:
                 await self._on_connected_callback()
 
+        elif msg_type == "pong":
+            pass
+
         elif msg_type == "ping":
             # Application-level ping - must respond with pong
             logger.debug("Received application-level ping, sending pong")
@@ -440,4 +455,8 @@ class LighterWebSocketManager(BaseWebSocketManager):
 
         else:
             # Log unknown messages for debugging
-            logger.debug(f"Unhandled Lighter message: {message}")
+            # logger.debug(f"Unhandled Lighter message: {message}")
+            pass
+
+    def _app_ping_payload(self) -> Optional[dict]:
+        return {"type": "ping"}

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/core/basics.py

@@ -295,7 +295,7 @@ class MarketType(StrEnum):
     OPTION = "OPTION"
 
 
-@dataclass
+@dataclass(order=True)
 class Instrument:
     """
     Instrument class.
@@ -326,6 +326,10 @@ class Instrument:
     delist_date: datetime | None = None  # date when instrument is delisted
     inverse: bool = False  # if true, then the future is inverse
 
+    def __post_init__(self):
+        # define how ordering works
+        object.__setattr__(self, "sort_index", f"{self.exchange}:{self.market_type}:{self.symbol}")
+
     @property
     def price_precision(self):
         if not hasattr(self, "_price_precision"):

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/core/metrics.py

@@ -980,7 +980,8 @@ class TradingSessionResult:
             perf = {
                 "cagr": 0.0,
                 "sharpe": 0.0,
-                "max_dd_pct": 0.0,
+                "mdd_pct": 0.0,
+                "daily_turnover": 0.0,
                 "gain": 0.0,
                 "calmar": 0.0,
                 "sortino": 0.0,
@@ -1000,7 +1001,8 @@ class TradingSessionResult:
             "performance": {
                 "cagr": perf.get("cagr", 0.0),
                 "sharpe": perf.get("sharpe", 0.0),
-                "max_dd_pct": perf.get("max_dd_pct", 0.0),
+                "mdd_pct": perf.get("mdd_pct", 0.0),
+                "daily_turnover": perf.get("daily_turnover", 0.0),
                 "total_return": perf.get("gain", 0.0) / self.get_total_capital() * 100
                 if self.get_total_capital() > 0
                 else 0.0,
@@ -1139,7 +1141,7 @@ class TradingSessionResult:
         params = info["parameters"]
 
         # - performance extracting
-        _dd_mtrx = ["max_dd_pct", "mdd_usd", "mdd_start", "mdd_peak", "mdd_recover"]
+        _dd_mtrx = ["mdd_pct", "mdd_usd", "mdd_start", "mdd_peak", "mdd_recover"]
         perf_main = {"".join(list(map(str.capitalize, c.split("_")))): v for c, v in perf.items() if c not in _dd_mtrx}
         perf_dd = {"".join(list(map(str.capitalize, c.split("_")))): v for c, v in perf.items() if c in _dd_mtrx}
         perf_dd["MddStart"] = pd.Timestamp(perf_dd["MddStart"]).strftime("%Y-%m-%d %H:%M:%S")
@@ -1233,7 +1235,11 @@ description: {_desc}
                 executions = pd.DataFrame()
             try:
                 signals = pd.read_csv(
-                    zip_ref.open("signals.csv"), index_col=["timestamp"], parse_dates=["timestamp"], date_format="mixed"
+                    zip_ref.open("signals.csv"),
+                    index_col=["timestamp"],
+                    parse_dates=["timestamp"],
+                    date_format="mixed",
+                    low_memory=False,
                 )
             except:
                 signals = pd.DataFrame()
@@ -1491,7 +1497,7 @@ def portfolio_metrics(
     sheet["drawdown_pct"] = mdd_pct
     sheet["drawdown_usd"] = dd_data
     # 25-May-2019: MDE fixed Max DD pct calculations
-    # sheet["max_dd_pct_on_init"] = 100 * mdd / init_cash
+    # sheet["mdd_pct_on_init"] = 100 * mdd / init_cash
     sheet["mdd_start"] = equity.index[ddstart]
     sheet["mdd_peak"] = equity.index[ddpeak]
     sheet["mdd_recover"] = equity.index[ddrecover]
@@ -1672,7 +1678,7 @@ def tearsheet(
             report = report.set_index("id", append=True).swaplevel()
             if sort_by:
                 report = report.sort_values(by=sort_by, ascending=sort_ascending)
-            return report
+            return report.round(2)
 
     else:
         return _tearsheet_single(

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/utils/misc.py

@@ -447,7 +447,12 @@ class AsyncThreadLoop:
         self.loop = loop
 
     def submit(self, coro: Awaitable) -> concurrent.futures.Future:
-        return asyncio.run_coroutine_threadsafe(coro, self.loop)
+        return asyncio.run_coroutine_threadsafe(coro, self.loop)  # type: ignore
+
+    async def run_in_executor(
+        self, executor: concurrent.futures.Executor, func: Callable, *args, **kwargs
+    ) -> concurrent.futures.Future:
+        return await self.loop.run_in_executor(executor, func, *args, **kwargs)
 
 
 def synchronized(func: Callable):

{qubx-0.6.94 → qubx-0.6.95}/src/qubx/utils/orderbook.py

@@ -13,6 +13,7 @@ import numpy as np
 import pandas as pd
 from numba import njit, types
 from numba.typed import Dict
+from sortedcontainers import SortedDict
 from tqdm.auto import tqdm
 
 from qubx import logger
@@ -573,32 +574,120 @@ class OrderBookState:
 
 
 class OrderBookStateManager:
-    def __init__(self):
+    """
+    Manages orderbook state with efficient updates and lookups.
+
+    Uses SortedDict for maintaining price levels in sorted order without
+    explicit sorting on each access. Pre-allocates numpy buffers to avoid
+    allocation overhead on frequent orderbook generation.
+
+    Args:
+        max_levels: Maximum number of orderbook levels to support (default: 1000)
+                   Used for buffer pre-allocation to avoid repeated allocations
+    """
+
+    def __init__(self, max_levels: int = 1000):
+        self.time = None
+        self.bids: SortedDict = SortedDict()  # Price -> Size (maintained in sorted order)
+        self.asks: SortedDict = SortedDict()  # Price -> Size (maintained in sorted order)
+        self.max_levels = max_levels
+
+        # Pre-allocate buffers to avoid allocation overhead
+        # These are reused across get_orderbook() calls
+        self._bids_buffer = np.zeros(max_levels, dtype=np.float64)
+        self._asks_buffer = np.zeros(max_levels, dtype=np.float64)
+
+    def reset(self):
+        """
+        Reset orderbook state to initial empty state.
+
+        This method clears all bid/ask price levels and resets the timestamp.
+        Should be called when the WebSocket connection is reestablished to ensure
+        clean state before processing new updates.
+        """
+        # Recreate SortedDict instances to ensure clean state
+        self.bids = SortedDict()
+        self.asks = SortedDict()
         self.time = None
-        self.bids = {}
-        self.asks = {}
 
     def get_state(self):
-        bids = sorted(self.bids.items(), key=lambda x: x[0])
-        asks = sorted(self.asks.items(), key=lambda x: x[0])
+        """
+        Get current orderbook state as OrderBookState object.
+
+        Returns sorted price levels from SortedDict (no explicit sorting needed).
+        """
+        # SortedDict maintains sorted order, just convert to list
+        bids = list(self.bids.items())
+        asks = list(self.asks.items())
         return OrderBookState(bids=bids, asks=asks)
 
     def get_orderbook(self, tick_size: float, levels: int) -> OrderBook | None:
+        """
+        Generate OrderBook from current state with aggregation.
+
+        Args:
+            tick_size: Price tick size for aggregation
+            levels: Number of price levels to include
+
+        Returns:
+            OrderBook object, or None if state is empty or invalid
+
+        Note:
+            Detects crossed orderbook (bid >= ask) which indicates corrupted state
+            from missed or out-of-order updates. Automatically resets state on detection.
+        """
         # Check if we have bids and asks
         if not self.bids or not self.asks:
             return None
 
-        # Sort bids descending (highest first) and asks ascending (lowest first)
-        sorted_bids = sorted(self.bids.items(), key=lambda x: x[0], reverse=True)
-        sorted_asks = sorted(self.asks.items(), key=lambda x: x[0])
+        # Validate levels parameter
+        if levels > self.max_levels:
+            logger.warning(
+                f"Requested levels ({levels}) exceeds max_levels ({self.max_levels}). "
+                f"Using max_levels={self.max_levels}"
+            )
+            levels = self.max_levels
+
+        # Get best bid/ask for crossed orderbook detection
+        # SortedDict: keys are in ascending order
+        best_bid = self.bids.keys()[-1]  # Highest bid (last key)
+        best_ask = self.asks.keys()[0]  # Lowest ask (first key)
+
+        # Detect crossed orderbook (invalid state)
+        # This should rarely happen now with proactive cleaning in update_state()
+        if best_bid >= best_ask:
+            logger.error(
+                f"Crossed orderbook detected AFTER proactive cleaning: "
+                f"best_bid={best_bid:.2f} >= best_ask={best_ask:.2f}. "
+                f"This indicates extreme message disorder or logic bug. Resetting state."
+            )
+            self.reset()
+            return None
+
+        # Get bids descending (highest first) and asks ascending (lowest first)
+        # SortedDict maintains ascending order, so reverse bids
+        sorted_bids = list(reversed(self.bids.items()))[:levels]
+        sorted_asks = list(self.asks.items())[:levels]
+
+        if levels == 1:
+            return OrderBook(
+                time=time_as_nsec(self.time),
+                top_bid=best_bid,
+                top_ask=best_ask,
+                tick_size=tick_size,
+                bids=np.array([sorted_bids[0][1]]),
+                asks=np.array([sorted_asks[0][1]]),
+            )
 
-        bids_buffer = np.zeros(levels, dtype=np.float64)
-        asks_buffer = np.zeros(levels, dtype=np.float64)
+        # Clear pre-allocated buffers for requested levels
+        self._bids_buffer[:levels].fill(0.0)
+        self._asks_buffer[:levels].fill(0.0)
 
         # Apply accumulation to aggregate into uniform grid
+        # Use slices of pre-allocated buffers to avoid allocation
         top_bid_agg, bids_accumulated = accumulate_orderbook_levels(
             np.array(sorted_bids, dtype=np.float64),
-            bids_buffer,
+            self._bids_buffer[:levels],
             tick_size,
             True,
             levels,
@@ -606,12 +695,13 @@ class OrderBookStateManager:
         )
         top_ask_agg, asks_accumulated = accumulate_orderbook_levels(
             np.array(sorted_asks, dtype=np.float64),
-            asks_buffer,
+            self._asks_buffer[:levels],
             tick_size,
             False,
             levels,
             False,  # is_bid=False, sizes_in_quoted=False
         )
+
         return OrderBook(
             time=time_as_nsec(self.time),
             top_bid=top_bid_agg,
@@ -622,13 +712,98 @@ class OrderBookStateManager:
         )
 
     def update_state(self, time: dt_64, bids: list[tuple[float, float]], asks: list[tuple[float, float]]):
+        """
+        Update orderbook state and proactively clean crossed levels.
+
+        Uses extreme prices from the UPDATE data (not state) for cross-checking:
+        - Highest bid in updates is the fresh price to check against asks
+        - Lowest ask in updates is the fresh price to check against bids
+
+        This ensures we only clean based on fresh data, not potentially stale state.
+
+        Args:
+            time: Timestamp of update
+            bids: List of (price, size) tuples for bid updates
+            asks: List of (price, size) tuples for ask updates
+        """
         self.time = time
-        self._update_state(self.bids, bids)
-        self._update_state(self.asks, asks)
 
-    def _update_state(self, state: dict[float, float], updates: list[tuple[float, float]]):
-        for price, size in updates:
+        # Find highest bid price in updates (excluding zero sizes)
+        highest_bid_update = None
+        if bids:
+            non_zero_bids = [price for price, size in bids if size > 0]
+            if non_zero_bids:
+                highest_bid_update = max(non_zero_bids)
+
+        # Apply all bid updates
+        for price, size in bids:
             if size == 0:
-                state.pop(price, None)
+                self.bids.pop(price, None)
             else:
-                state[price] = size
+                self.bids[price] = size
+
+        # Clean crossed asks using highest bid from UPDATE (not state)
+        if highest_bid_update is not None:
+            self._clean_crossed_asks(highest_bid_update)
+
+        # Find lowest ask price in updates (excluding zero sizes)
+        lowest_ask_update = None
+        if asks:
+            non_zero_asks = [price for price, size in asks if size > 0]
+            if non_zero_asks:
+                lowest_ask_update = min(non_zero_asks)
+
+        # Apply all ask updates
+        for price, size in asks:
+            if size == 0:
+                self.asks.pop(price, None)
+            else:
+                self.asks[price] = size
+
+        # Clean crossed bids using lowest ask from UPDATE (not state)
+        if lowest_ask_update is not None:
+            self._clean_crossed_bids(lowest_ask_update)
+
+    def _clean_crossed_asks(self, bid_price: float) -> int:
+        """
+        Remove all asks at or below the given bid price.
+
+        Args:
+            bid_price: Bid price from fresh update that may cross asks
+
+        Returns:
+            Number of crossed asks removed
+        """
+        if not self.asks:
+            return 0
+
+        # Use irange() to efficiently find asks <= bid_price
+        crossed_keys = list(self.asks.irange(maximum=bid_price, inclusive=(True, True)))
+
+        if crossed_keys:
+            for key in crossed_keys:
+                del self.asks[key]
+
+        return len(crossed_keys)
+
+    def _clean_crossed_bids(self, ask_price: float) -> int:
+        """
+        Remove all bids at or above the given ask price.
+
+        Args:
+            ask_price: Ask price from fresh update that may cross bids
+
+        Returns:
+            Number of crossed bids removed
+        """
+        if not self.bids:
+            return 0
+
+        # Use irange() to efficiently find bids >= ask_price
+        crossed_keys = list(self.bids.irange(minimum=ask_price, inclusive=(True, True)))
+
+        if crossed_keys:
+            for key in crossed_keys:
+                del self.bids[key]
+
+        return len(crossed_keys)