Qubx 0.6.19__tar.gz → 0.6.20__tar.gz

{qubx-0.6.19 → qubx-0.6.20}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: Qubx
-Version: 0.6.19
+Version: 0.6.20
 Summary: Qubx - Quantitative Trading Framework
 Author: Dmitry Marienko
 Author-email: dmitry.marienko@xlydian.com

{qubx-0.6.19 → qubx-0.6.20}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "Qubx"
-version = "0.6.19"
+version = "0.6.20"
 description = "Qubx - Quantitative Trading Framework"
 authors = [ "Dmitry Marienko <dmitry.marienko@xlydian.com>", "Yuriy Arabskyy <yuriy.arabskyy@xlydian.com>",]
 readme = "README.md"

{qubx-0.6.19 → qubx-0.6.20}/src/qubx/connectors/ccxt/account.py

@@ -81,6 +81,7 @@ class CcxtAccountProcessor(BasicAccountProcessor):
         open_order_backoff: str = "1Min",
         max_position_restore_days: int = 30,
         max_retries: int = 10,
+        read_only: bool = False,
     ):
         super().__init__(
             account_id=account_id,
@@ -106,6 +107,7 @@ class CcxtAccountProcessor(BasicAccountProcessor):
         self._required_instruments = set()
         self._latest_instruments = set()
         self._subscription_manager = None
+        self._read_only = read_only
 
     def set_subscription_manager(self, manager: ISubscriptionManager) -> None:
         self._subscription_manager = manager
@@ -439,9 +441,9 @@ class CcxtAccountProcessor(BasicAccountProcessor):
                 for order in orders:
                     logger.debug(f"    :: [SYNC] {order.side} {order.quantity} @ {order.price} ({order.status})")
         else:
-            # TODO: think if this should actually be here
             # - we need to cancel the unexpected orders
-            await self._cancel_unexpected_orders(_open_orders)
+            if not self._read_only:
+                await self._cancel_unexpected_orders(_open_orders)
 
     async def _cancel_unexpected_orders(self, open_orders: dict[str, Order]) -> None:
         _expected_orders = set(self._active_orders.keys())

{qubx-0.6.19 → qubx-0.6.20}/src/qubx/connectors/ccxt/data.py

@@ -20,8 +20,9 @@ from ccxt.pro import Exchange
 from qubx import logger
 from qubx.core.basics import CtrlChannel, DataType, Instrument, ITimeProvider, dt_64
 from qubx.core.helpers import BasicScheduler
-from qubx.core.interfaces import IDataProvider
+from qubx.core.interfaces import IDataProvider, IHealthMonitor
 from qubx.core.series import Bar, Quote
+from qubx.health import DummyHealthMonitor
 from qubx.utils.misc import AsyncThreadLoop
 
 from .exceptions import CcxtLiquidationParsingError, CcxtSymbolNotRecognized
@@ -64,12 +65,14 @@ class CcxtDataProvider(IDataProvider):
         channel: CtrlChannel,
         max_ws_retries: int = 10,
         warmup_timeout: int = 120,
+        health_monitor: IHealthMonitor | None = None,
     ):
         self._exchange_id = str(exchange.name)
         self.time_provider = time_provider
         self.channel = channel
         self.max_ws_retries = max_ws_retries
         self._warmup_timeout = warmup_timeout
+        self._health_monitor = health_monitor or DummyHealthMonitor()
 
         # - create new even loop
         self._exchange = exchange
@@ -416,11 +419,13 @@ class CcxtDataProvider(IDataProvider):
                 instrument = ccxt_find_instrument(exch_symbol, self._exchange, _symbol_to_instrument)
                 for _, ohlcvs in _data.items():
                     for oh in ohlcvs:
+                        timestamp_ns = oh[0] * 1_000_000
+                        self._health_monitor.record_data_arrival(sub_type, dt_64(timestamp_ns, "ns"))
                         channel.send(
                             (
                                 instrument,
                                 sub_type,
-                                Bar(oh[0] * 1_000_000, oh[1], oh[2], oh[3], oh[4], oh[6], oh[7]),
+                                Bar(timestamp_ns, oh[1], oh[2], oh[3], oh[4], oh[6], oh[7]),
                                 False,  # not historical bar
                             )
                         )
@@ -463,7 +468,9 @@ class CcxtDataProvider(IDataProvider):
             exch_symbol = trades[0]["symbol"]
             instrument = ccxt_find_instrument(exch_symbol, self._exchange, _symbol_to_instrument)
             for trade in trades:
-                channel.send((instrument, sub_type, ccxt_convert_trade(trade), False))
+                converted_trade = ccxt_convert_trade(trade)
+                self._health_monitor.record_data_arrival(sub_type, dt_64(converted_trade.time, "ns"))
+                channel.send((instrument, sub_type, converted_trade, False))
 
         async def un_watch_trades(instruments: list[Instrument]):
             symbols = [_instr_to_ccxt_symbol[i] for i in instruments]
@@ -497,6 +504,7 @@ class CcxtDataProvider(IDataProvider):
             ob = ccxt_convert_orderbook(ccxt_ob, instrument, levels=depth, tick_size_pct=tick_size_pct)
             if ob is None:
                 return
+            self._health_monitor.record_data_arrival(sub_type, dt_64(ob.time, "ns"))
             quote = ob.to_quote()
             self._last_quotes[instrument] = quote
             channel.send((instrument, sub_type, ob, False))
@@ -529,6 +537,7 @@ class CcxtDataProvider(IDataProvider):
             for exch_symbol, ccxt_ticker in ccxt_tickers.items():  # type: ignore
                 instrument = ccxt_find_instrument(exch_symbol, self._exchange, _symbol_to_instrument)
                 quote = ccxt_convert_ticker(ccxt_ticker)
+                self._health_monitor.record_data_arrival(sub_type, dt_64(quote.time, "ns"))
                 self._last_quotes[instrument] = quote
                 channel.send((instrument, sub_type, quote, False))
 
@@ -562,8 +571,11 @@ class CcxtDataProvider(IDataProvider):
             liquidations = await self._exchange.watch_liquidations_for_symbols(symbols)
             for liquidation in liquidations:
                 try:
-                    instrument = ccxt_find_instrument(liquidation["symbol"], self._exchange, _symbol_to_instrument)
-                    channel.send((instrument, sub_type, ccxt_convert_liquidation(liquidation), False))
+                    exch_symbol = liquidation["symbol"]
+                    instrument = ccxt_find_instrument(exch_symbol, self._exchange, _symbol_to_instrument)
+                    liquidation_event = ccxt_convert_liquidation(liquidation)
+                    self._health_monitor.record_data_arrival(sub_type, dt_64(liquidation_event.time, "ns"))
+                    channel.send((instrument, sub_type, liquidation_event, False))
                 except CcxtLiquidationParsingError:
                     logger.debug(f"Could not parse liquidation {liquidation}")
                     continue
@@ -593,12 +605,17 @@ class CcxtDataProvider(IDataProvider):
         async def watch_funding_rates():
             funding_rates = await self._exchange.watch_funding_rates()  # type: ignore
             instrument_to_funding_rate = {}
+            current_time = self.time_provider.time()
+
             for symbol, info in funding_rates.items():
                 try:
                     instrument = ccxt_find_instrument(symbol, self._exchange)
-                    instrument_to_funding_rate[instrument] = ccxt_convert_funding_rate(info)
+                    funding_rate = ccxt_convert_funding_rate(info)
+                    instrument_to_funding_rate[instrument] = funding_rate
+                    self._health_monitor.record_data_arrival(sub_type, dt_64(current_time, "s"))
                 except CcxtSymbolNotRecognized:
                     continue
+
             channel.send((None, sub_type, instrument_to_funding_rate, False))
 
         async def un_watch_funding_rates():

{qubx-0.6.19 → qubx-0.6.20}/src/qubx/core/context.py

@@ -4,7 +4,6 @@ from typing import Any, Callable
 
 from qubx import logger
 from qubx.core.basics import (
-    SW,
     AssetBalance,
     CtrlChannel,
     DataType,
@@ -14,6 +13,7 @@ from qubx.core.basics import (
     Order,
     OrderRequest,
     Position,
+    Timestamped,
     dt_64,
 )
 from qubx.core.exceptions import StrategyExceededMaxNumberOfRuntimeFailuresError
@@ -27,6 +27,7 @@ from qubx.core.interfaces import (
     IAccountProcessor,
     IBroker,
     IDataProvider,
+    IHealthMonitor,
     IMarketManager,
     IMetricEmitter,
     IPositionGathering,
@@ -45,6 +46,7 @@ from qubx.core.interfaces import (
 from qubx.core.loggers import StrategyLogging
 from qubx.data.readers import DataReader
 from qubx.gathering.simplest import SimplePositionGatherer
+from qubx.health import DummyHealthMonitor
 from qubx.trackers.sizers import FixedSizer
 
 from .mixins import (
@@ -102,6 +104,7 @@ class StrategyContext(IStrategyContext):
         initializer: BasicStrategyInitializer | None = None,
         strategy_name: str | None = None,
         strategy_state: StrategyState | None = None,
+        health_monitor: IHealthMonitor | None = None,
     ) -> None:
         self.account = account
         self.strategy = self.__instantiate_strategy(strategy, config)
@@ -127,6 +130,9 @@ class StrategyContext(IStrategyContext):
         self._strategy_state = strategy_state if strategy_state is not None else StrategyState()
         self._strategy_name = strategy_name if strategy_name is not None else strategy.__class__.__name__
 
+        self._health_monitor = health_monitor or DummyHealthMonitor()
+        self.health = self._health_monitor
+
         __position_tracker = self.strategy.tracker(self)
         if __position_tracker is None:
             __position_tracker = StrategyContext.DEFAULT_POSITION_TRACKER()
@@ -180,6 +186,7 @@ class StrategyContext(IStrategyContext):
             scheduler=self._scheduler,
             is_simulation=self._data_provider.is_simulation,
             exporter=self._exporter,
+            health_monitor=self._health_monitor,
         )
         self.__post_init__()
 
@@ -245,6 +252,9 @@ class StrategyContext(IStrategyContext):
         # - start account processing
         self.account.start()
 
+        # - start health metrics monitor
+        self._health_monitor.start()
+
         # - update universe with initial instruments after the strategy is initialized
         self.set_universe(self._initial_instruments, skip_callback=True)
 
@@ -299,6 +309,9 @@ class StrategyContext(IStrategyContext):
         # - stop account processing
         self.account.stop()
 
+        # - stop health metrics monitor
+        self._health_monitor.stop()
+
         # - close logging
         self._logging.close()
 
@@ -513,22 +526,30 @@ class StrategyContext(IStrategyContext):
     def __process_incoming_data_loop(self, channel: CtrlChannel):
         logger.info("[StrategyContext] :: Start processing market data")
         while channel.control.is_set():
-            with SW("StrategyContext._process_incoming_data"):
-                try:
-                    # - waiting for incoming market data
-                    instrument, d_type, data, hist = channel.receive()
-                    if self.process_data(instrument, d_type, data, hist):
-                        channel.stop()
-                        break
-                except StrategyExceededMaxNumberOfRuntimeFailuresError:
+            try:
+                # - waiting for incoming market data
+                instrument, d_type, data, hist = channel.receive()
+
+                _should_record = isinstance(data, Timestamped) and not hist
+                if _should_record:
+                    self._health_monitor.record_start_processing(d_type, dt_64(data.time, "ns"))
+
+                if self.process_data(instrument, d_type, data, hist):
                     channel.stop()
                     break
-                except Exception as e:
-                    logger.error(f"Error processing market data: {e}")
-                    logger.opt(colors=False).error(traceback.format_exc())
-                    if self._lifecycle_notifier:
-                        self._lifecycle_notifier.notify_error(self._strategy_name, e)
-                    # Don't stop the channel here, let it continue processing
+
+                if _should_record:
+                    self._health_monitor.record_end_processing(d_type, dt_64(data.time, "ns"))
+
+            except StrategyExceededMaxNumberOfRuntimeFailuresError:
+                channel.stop()
+                break
+            except Exception as e:
+                logger.error(f"Error processing market data: {e}")
+                logger.opt(colors=False).error(traceback.format_exc())
+                if self._lifecycle_notifier:
+                    self._lifecycle_notifier.notify_error(self._strategy_name, e)
+                # Don't stop the channel here, let it continue processing
         logger.info("[StrategyContext] :: Market data processing stopped")
 
     def __instantiate_strategy(self, strategy: IStrategy, config: dict[str, Any] | None) -> IStrategy:

qubx-0.6.20/src/qubx/core/deque.py

@@ -0,0 +1,182 @@
+"""
+Fast, fixed-size circular buffer implementation using Numba for performance.
+When the deque is full, pushing new elements overwrites the oldest elements.
+"""
+
+import numpy as np
+from numba import from_dtype, int32, types
+from numba.experimental import jitclass
+
+
+def create_deque_class(element_dtype: np.dtype):
+    """
+    Dynamically create and return a Deque jitclass that stores elements of the given NumPy dtype.
+
+    This is a fast, fixed-size circular buffer implementation using Numba for performance.
+    When the deque is full, pushing new elements overwrites the oldest elements.
+
+    Args:
+        element_dtype (np.dtype): The NumPy dtype for elements to store
+
+    Returns:
+        jitclass: A compiled Numba class for the deque
+
+    Raises:
+        ValueError: If the element_dtype is not supported
+    """
+    # Determine whether it's a structured dtype or a scalar dtype
+    if element_dtype.fields is not None:
+        # It's a structured dtype. Convert to a Numba 'record' type
+        element_type = from_dtype(element_dtype)
+        data_type = element_type[:]  # 1D array of that struct
+    else:
+        # It's a scalar dtype (e.g., float32)
+        if element_dtype == np.float32:
+            data_type = types.float32[:]
+        elif element_dtype == np.float64:
+            data_type = types.float64[:]
+        elif element_dtype == np.int32:
+            data_type = types.int32[:]
+        elif element_dtype == np.int64:
+            data_type = types.int64[:]
+        else:
+            raise ValueError(f"Unsupported scalar dtype: {element_dtype}")
+
+    # Build the class spec
+    spec = [
+        ("data", data_type),
+        ("capacity", int32),
+        ("head", int32),
+        ("tail", int32),
+        ("size", int32),
+    ]
+
+    class Deque:
+        def __init__(self, capacity):
+            self.data = np.empty(capacity, dtype=element_dtype)
+            self.capacity = capacity
+            self.head = 0
+            self.tail = 0
+            self.size = 0
+
+        def push_back(self, record):
+            if self.size == self.capacity:
+                # Overwrite oldest from the front
+                self.head = (self.head + 1) % self.capacity
+                self.size -= 1
+
+            self.data[self.tail] = record
+            self.tail = (self.tail + 1) % self.capacity
+            self.size += 1
+
+        def push_front(self, record):
+            if self.size == self.capacity:
+                # Overwrite oldest from the back
+                self.tail = (self.tail - 1) % self.capacity
+                self.size -= 1
+
+            self.head = (self.head - 1) % self.capacity
+            self.data[self.head] = record
+            self.size += 1
+
+        def pop_front(self):
+            if self.size == 0:
+                raise IndexError("Deque is empty")
+            record = self.data[self.head]
+            self.head = (self.head + 1) % self.capacity
+            self.size -= 1
+            return record
+
+        def pop_back(self):
+            if self.size == 0:
+                raise IndexError("Deque is empty")
+            self.tail = (self.tail - 1) % self.capacity
+            record = self.data[self.tail]
+            self.size -= 1
+            return record
+
+        def is_empty(self):
+            return self.size == 0
+
+        def is_full(self):
+            return self.size == self.capacity
+
+        def get_size(self):
+            return self.size
+
+        def __len__(self):
+            return self.size
+
+        def front(self):
+            if self.size == 0:
+                raise IndexError("Deque is empty")
+            return self.data[self.head]
+
+        def back(self):
+            if self.size == 0:
+                raise IndexError("Deque is empty")
+            return self.data[(self.tail - 1) % self.capacity]
+
+        def __getitem__(self, idx):
+            if idx < 0 or idx >= self.size:
+                raise IndexError("Index out of bounds")
+            return self.data[(self.tail - idx - 1) % self.capacity]
+
+        def to_array(self):
+            """Return a NumPy array of the current elements in the Deque, from oldest to newest."""
+            out = np.empty(self.size, dtype=self.data.dtype)
+            for i in range(self.size):
+                idx = (self.head + i) % self.capacity
+                out[i] = self.data[idx]
+            return out
+
+        def push_back_fields(self): ...
+
+    # Generate push_back_fields(...) for structured dtypes
+    if element_dtype.fields is not None:
+        field_names = list(element_dtype.fields.keys())
+        arg_list = ", ".join(field_names)
+        lines = []
+        lines.append(f"def push_back_fields(self, {arg_list}):")
+        lines.append("    if self.size == self.capacity:")
+        lines.append("        self.head = (self.head + 1) % self.capacity")
+        lines.append("        self.size -= 1")
+        lines.append("")
+        for f in field_names:
+            lines.append(f"    self.data[self.tail]['{f}'] = {f}")
+        lines.append("    self.tail = (self.tail + 1) % self.capacity")
+        lines.append("    self.size += 1")
+
+        method_src = "\n".join(lines)
+        tmp_ns = {}
+        exec(method_src, {}, tmp_ns)
+        push_back_fields_func = tmp_ns["push_back_fields"]
+        setattr(Deque, "push_back_fields", push_back_fields_func)
+
+    _Deque = jitclass(spec)(Deque)
+    return _Deque
+
+
+# Pre-compiled deque types for common use cases
+DequeFloat64 = create_deque_class(np.dtype(np.float64))
+DequeFloat32 = create_deque_class(np.dtype(np.float32))
+DequeInt64 = create_deque_class(np.dtype(np.int64))
+DequeInt32 = create_deque_class(np.dtype(np.int32))
+
+# Deque type for storing indicator values with timestamps
+DequeIndicator = create_deque_class(
+    np.dtype(
+        [
+            ("timestamp", np.int64),
+            ("value", np.float64),
+        ],
+        align=True,
+    )
+)
+
+# Instance types for use in other jitclasses
+DequeIndicator_instance = DequeIndicator.class_type.instance_type
+DequeFloat64_instance = DequeFloat64.class_type.instance_type
+DequeFloat32_instance = DequeFloat32.class_type.instance_type
+DequeInt64_instance = DequeInt64.class_type.instance_type
+DequeInt32_instance = DequeInt32.class_type.instance_type