Qubx 0.6.64__cp312-cp312-manylinux_2_39_x86_64.whl → 0.6.66__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/broker.py

@@ -1,3 +1,4 @@
+from qubx import logger
 from qubx.backtester.ome import SimulatedExecutionReport
 from qubx.backtester.simulated_exchange import ISimulatedExchange
 from qubx.core.basics import (
@@ -5,6 +6,7 @@ from qubx.core.basics import (
     Instrument,
     Order,
 )
+from qubx.core.exceptions import OrderNotFound
 from qubx.core.interfaces import IBroker
 
 from .account import SimulatedAccountProcessor
@@ -63,8 +65,13 @@ class SimulatedBroker(IBroker):
         self.send_order(instrument, order_side, order_type, amount, price, client_id, time_in_force, **optional)
 
     def cancel_order(self, order_id: str) -> Order | None:
-        self._send_execution_report(order_update := self._exchange.cancel_order(order_id))
-        return order_update.order if order_update is not None else None
+        try:
+            self._send_execution_report(order_update := self._exchange.cancel_order(order_id))
+            return order_update.order if order_update is not None else None
+        except OrderNotFound:
+            # Order was already cancelled or doesn't exist
+            logger.debug(f"Order {order_id} not found")
+            return None
 
     def cancel_orders(self, instrument: Instrument) -> None:
         raise NotImplementedError("Not implemented yet")

qubx/backtester/data.py

@@ -71,7 +71,7 @@ class SimulatedDataProvider(IDataProvider):
             # Check if the instrument was actually subscribed (not filtered out)
             if not self.has_subscription(i, subscription_type):
                 continue
-                
+
             h_data = self._data_source.peek_historical_data(i, subscription_type)
             if h_data:
                 # _s_type = DataType.from_str(subscription_type)[0]

qubx/backtester/runner.py

@@ -5,7 +5,9 @@ import pandas as pd
 from tqdm.auto import tqdm
 
 from qubx import logger
+from qubx.backtester.sentinels import NoDataContinue
 from qubx.backtester.simulated_data import IterableSimulationData
+from qubx.backtester.utils import SimulationDataConfig, TimeGuardedWrapper
 from qubx.core.account import CompositeAccountProcessor
 from qubx.core.basics import SW, DataType, Instrument, TransactionCostsCalculator
 from qubx.core.context import StrategyContext
@@ -22,8 +24,10 @@ from qubx.core.interfaces import (
 )
 from qubx.core.loggers import StrategyLogging
 from qubx.core.lookups import lookup
+from qubx.data.helpers import CachedPrefetchReader
 from qubx.loggers.inmemory import InMemoryLogsWriter
 from qubx.pandaz.utils import _frame_to_str
+from qubx.utils.time import now_ns
 
 from .account import SimulatedAccountProcessor
 from .broker import SimulatedBroker
@@ -80,6 +84,7 @@ class SimulationRunner:
         emitter: IMetricEmitter | None = None,
         strategy_state: StrategyState | None = None,
         initializer: BasicStrategyInitializer | None = None,
+        warmup_mode: bool = False,
     ):
         """
         Initialize the BacktestContextRunner with a strategy context.
@@ -102,8 +107,10 @@ class SimulationRunner:
         self.emitter = emitter
         self.strategy_state = strategy_state if strategy_state is not None else StrategyState()
         self.initializer = initializer
+        self.warmup_mode = warmup_mode
         self._pregenerated_signals = dict()
         self._to_process = {}
+        self._aux_data_reader = None
 
         # - get strategy parameters BEFORE simulation start
         #   potentially strategy may change it's parameters during simulation
@@ -126,6 +133,8 @@ class SimulationRunner:
         """
         logger.debug(f"[<y>SimulationRunner</y>] :: Running simulation from {self.start} to {self.stop}")
 
+        self._prefetch_aux_data()
+
         # Start the context
         self.ctx.start()
 
@@ -164,6 +173,8 @@ class SimulationRunner:
             logger.error("Simulated trading interrupted by user!")
             if not catch_keyboard_interrupt:
                 raise
+        except Exception as e:
+            raise e
         finally:
             # Stop the context
             self.ctx.stop()
@@ -196,7 +207,7 @@ class SimulationRunner:
             for i in self._data_providers[0].get_subscribed_instruments():
                 # - we can process series with variable id's if we can find some similar instrument
                 if s == i.symbol or s == str(i) or s == f"{i.exchange}:{i.symbol}" or str(s) == str(i):
-                    _start, _end = pd.Timestamp(start), pd.Timestamp(end)
+                    _start, _end = np.datetime64(start), np.datetime64(end)
                     _start_idx, _end_idx = v.index.get_indexer([_start, _end], method="ffill")
                     sel = v.iloc[max(_start_idx, 0) : _end_idx + 1]
 
@@ -268,21 +279,34 @@ class SimulationRunner:
         start, stop = pd.Timestamp(start), pd.Timestamp(stop)
         total_duration = stop - start
         update_delta = total_duration / 100
-        prev_dt = pd.Timestamp(start)
+        prev_dt = np.datetime64(start)
 
         # - date iteration
         qiter = self._data_source.create_iterable(start, stop)
+
         if silent:
             for instrument, data_type, event, is_hist in qiter:
-                if not _run(instrument, data_type, event, is_hist):
+                # Handle NoDataContinue sentinel
+                if isinstance(event, NoDataContinue):
+                    if not self._handle_no_data_scenario(stop):
+                        break
+                    continue
+
+                if not self._process_event(instrument, data_type, event, is_hist, _run, stop):
                     break
         else:
             _p = 0
             with tqdm(total=100, desc="Simulating", unit="%", leave=False) as pbar:
                 for instrument, data_type, event, is_hist in qiter:
-                    if not _run(instrument, data_type, event, is_hist):
+                    # Handle NoDataContinue sentinel
+                    if isinstance(event, NoDataContinue):
+                        if not self._handle_no_data_scenario(stop):
+                            break
+                        continue
+
+                    if not self._process_event(instrument, data_type, event, is_hist, _run, stop):
                         break
-                    dt = pd.Timestamp(event.time)
+                    dt = np.datetime64(event.time, "ns")
                     # update only if date has changed
                     if dt - prev_dt > update_delta:
                         _p += 1
@@ -294,6 +318,43 @@ class SimulationRunner:
 
         logger.info(f"{self.__class__.__name__} ::: Simulation finished at {stop} :::")
 
+    def _process_event(self, instrument, data_type, event, is_hist, _run, stop_time):
+        """Process a single simulation event with proper time advancement and scheduler checks."""
+        # During warmup, clamp future timestamps to current time
+        if self.warmup_mode and hasattr(event, "time"):
+            current_real_time = now_ns()
+            if event.time > current_real_time:
+                event.time = current_real_time
+
+        if not _run(instrument, data_type, event, is_hist):
+            return False
+        return True
+
+    def _handle_no_data_scenario(self, stop_time):
+        """Handle scenario when no data is available but scheduler might have events."""
+        # Check if we have pending scheduled events
+        if hasattr(self.scheduler, "_next_nearest_time"):
+            next_scheduled_time = self.scheduler._next_nearest_time
+            current_time = self.time_provider.time()
+
+            # Convert to int64 for numerical comparisons (avoid type issues)
+            next_time_ns = next_scheduled_time.astype("int64")
+            current_time_ns = current_time.astype("int64")
+            stop_time_ns = stop_time.value  # Already int64
+
+            # Check if we've reached the stop time
+            if current_time_ns >= stop_time_ns:
+                return False  # Stop simulation
+
+            # If there's a scheduled event before stop time, advance to it
+            if next_time_ns < np.iinfo(np.int64).max and next_time_ns < stop_time_ns:
+                # Use the original datetime64 object for set_time (not the int64 conversion)
+                self.time_provider.set_time(next_scheduled_time)
+                self.scheduler.check_and_run_tasks()
+                return True  # Continue simulation
+
+        return False  # No scheduled events, stop simulation
+
     def print_latency_report(self) -> None:
         _l_r = SW.latency_report()
         if _l_r is not None:
@@ -311,11 +372,6 @@ class SimulationRunner:
             f"for {self.setup.capital} {self.setup.base_currency}..."
         )
 
-        data_source = IterableSimulationData(
-            self.data_config.data_providers,
-            open_close_time_indent_secs=self.data_config.adjusted_open_close_time_indent_secs,
-        )
-
         channel = SimulatedCtrlChannel("databus", sentinel=(None, None, None, None))
         simulated_clock = SimulatedTimeProvider(np.datetime64(self.start, "ns"))
 
@@ -325,6 +381,11 @@ class SimulationRunner:
 
         scheduler = SimulatedScheduler(channel, lambda: simulated_clock.time().item())
 
+        data_source = IterableSimulationData(
+            self.data_config.data_providers,
+            open_close_time_indent_secs=self.data_config.adjusted_open_close_time_indent_secs,
+        )
+
         brokers = []
         for exchange in self.setup.exchanges:
             _exchange_account = account.get_account_processor(exchange)
@@ -349,7 +410,7 @@ class SimulationRunner:
             )
 
         # - get aux data provider
-        _aux_data = self.data_config.get_timeguarded_aux_reader(simulated_clock)
+        self._aux_data_reader = self.data_config.get_timeguarded_aux_reader(simulated_clock)
 
         # - it will store simulation results into memory
         logs_writer = InMemoryLogsWriter(self.account_id, self.setup.name, "0")
@@ -401,7 +462,7 @@ class SimulationRunner:
             time_provider=simulated_clock,
             instruments=self.setup.instruments,
             logging=StrategyLogging(logs_writer, portfolio_log_freq=self.portfolio_log_freq),
-            aux_data_provider=_aux_data,
+            aux_data_provider=self._aux_data_reader,
             emitter=self.emitter,
             strategy_state=self.strategy_state,
             initializer=self.initializer,
@@ -422,6 +483,10 @@ class SimulationRunner:
             logger.debug(f"[<y>simulator</y>] :: Setting default schedule: {self.data_config.default_trigger_schedule}")
             ctx.set_event_schedule(self.data_config.default_trigger_schedule)
 
+        if self.setup.enable_funding:
+            logger.debug("[<y>simulator</y>] :: Enabling funding rate simulation")
+            ctx.subscribe(DataType.FUNDING_PAYMENT)
+
         self.logs_writer = logs_writer
         self.channel = channel
         self.time_provider = simulated_clock
@@ -483,3 +548,42 @@ class SimulationRunner:
             time_provider=time_provider,
             account_processors=_account_processors,
         )
+
+    def _prefetch_aux_data(self):
+        # Perform prefetch of aux data if enabled
+        if self._aux_data_reader is None:
+            return
+
+        aux_reader = self._aux_data_reader
+        if isinstance(aux_reader, TimeGuardedWrapper) and isinstance(aux_reader._reader, CachedPrefetchReader):
+            aux_reader = aux_reader._reader
+        elif isinstance(aux_reader, CachedPrefetchReader):
+            aux_reader = aux_reader
+        else:
+            return
+
+        if self.data_config.prefetch_config and self.data_config.prefetch_config.enabled:
+            # Prepare prefetch arguments
+            prefetch_args = self.data_config.prefetch_config.args.copy()
+
+            # Add exchange info if available from instruments
+            if self.setup.instruments and "exchange" not in prefetch_args:
+                # Get exchange from first instrument
+                first_exchange = self.setup.instruments[0].exchange
+                if first_exchange:
+                    prefetch_args["exchange"] = first_exchange
+
+            logger.info(
+                f"Prefetching aux data: {self.data_config.prefetch_config.aux_data_names} for period {self.start} to {self.stop}"
+            )
+
+            try:
+                # Perform the prefetch
+                aux_reader.prefetch_aux_data(
+                    self.data_config.prefetch_config.aux_data_names,
+                    start=str(self.start),
+                    stop=str(self.stop),
+                    **prefetch_args,
+                )
+            except Exception as e:
+                logger.warning(f"Prefetch failed: {e}")

qubx/backtester/sentinels.py

@@ -0,0 +1,23 @@
+from __future__ import annotations
+
+from typing import Optional
+
+
+class NoDataContinue:
+    """Sentinel indicating no data streams available but simulation should continue.
+    
+    This is used when all instruments are unsubscribed but there may still be
+    scheduled events to process before the simulation stop time.
+    """
+    
+    def __init__(self, next_scheduled_time: Optional[int] = None):
+        """Initialize the sentinel.
+        
+        Args:
+            next_scheduled_time: The next scheduled event time in nanoseconds,
+                                or None if no scheduled events exist.
+        """
+        self.next_scheduled_time = next_scheduled_time
+    
+    def __repr__(self) -> str:
+        return f"NoDataContinue(next_scheduled_time={self.next_scheduled_time})"

qubx/backtester/simulated_data.py

@@ -3,6 +3,7 @@ from typing import Any, Iterator
 import pandas as pd
 
 from qubx import logger
+from qubx.backtester.sentinels import NoDataContinue
 from qubx.core.basics import DataType, Instrument, MarketType, Timestamped
 from qubx.core.exceptions import SimulationError
 from qubx.data.composite import IteratedDataStreamsSlicer
@@ -259,14 +260,14 @@ class IterableSimulationData(Iterator):
     def _filter_instruments_for_subscription(self, data_type: str, instruments: list[Instrument]) -> list[Instrument]:
         """
         Filter instruments based on subscription type requirements.
-        
+
         For funding payment subscriptions, only SWAP instruments are supported since
         funding payments are specific to perpetual swap contracts.
-        
+
         Args:
             data_type: The data type being subscribed to
             instruments: List of instruments to filter
-            
+
         Returns:
             Filtered list of instruments appropriate for the subscription type
         """
@@ -275,23 +276,25 @@ class IterableSimulationData(Iterator):
             original_count = len(instruments)
             filtered_instruments = [i for i in instruments if i.market_type == MarketType.SWAP]
             filtered_count = len(filtered_instruments)
-            
+
             # Log if instruments were filtered out (debug info)
             if filtered_count < original_count:
-                logger.debug(f"Filtered {original_count - filtered_count} non-SWAP instruments from funding payment subscription")
-            
+                logger.debug(
+                    f"Filtered {original_count - filtered_count} non-SWAP instruments from funding payment subscription"
+                )
+
             return filtered_instruments
-        
+
         # For all other subscription types, return instruments unchanged
         return instruments
 
     def add_instruments_for_subscription(self, subscription: str, instruments: list[Instrument] | Instrument):
         instruments = instruments if isinstance(instruments, list) else [instruments]
         _subt_key, _data_type, _params = self._parse_subscription_spec(subscription)
-        
+
         # Filter instruments based on subscription type requirements
         instruments = self._filter_instruments_for_subscription(_data_type, instruments)
-        
+
         # If no instruments remain after filtering, skip subscription
         if not instruments:
             return
@@ -445,6 +448,17 @@ class IterableSimulationData(Iterator):
         try:
             while data := next(self._slicing_iterator):  # type: ignore
                 k, t, v = data
+
+                # Check if we've reached or exceeded the stop time
+                # It's commented out because we expect data readers to stop on their own
+                # if self._stop is not None and t > self._stop.value:
+                #     raise StopIteration
+                
+                # Handle NoDataContinue sentinel
+                if isinstance(v, NoDataContinue):
+                    # Return the sentinel as the event - the runner will detect it with isinstance
+                    return None, "", v, False
+
                 instr, fetcher, subt = self._instruments[k]
                 data_type = fetcher._producing_data_type
                 _is_historical = False

qubx/backtester/simulated_exchange.py

@@ -11,6 +11,7 @@ from qubx.core.basics import (
     TransactionCostsCalculator,
     dt_64,
 )
+from qubx.core.exceptions import OrderNotFound
 from qubx.core.series import Bar, OrderBook, Quote, Trade, TradeArray
 
 
@@ -155,8 +156,7 @@ class BasicSimulatedExchange(ISimulatedExchange):
                     if order.id == order_id:
                         return self._process_ome_response(o.cancel_order(order_id))
 
-            logger.warning(f"[<y>{self.__class__.__name__}</y>] :: cancel_order :: can't find order '{order_id}'!")
-            return None
+            raise OrderNotFound(f"Order '{order_id}' not found")
 
         ome = self._ome.get(instrument)
         if ome is None:
@@ -165,7 +165,10 @@ class BasicSimulatedExchange(ISimulatedExchange):
             )
 
         # - cancel order in OME and remove from the map to free memory
-        return self._process_ome_response(ome.cancel_order(order_id))
+        result = self._process_ome_response(ome.cancel_order(order_id))
+        if result is None:
+            raise OrderNotFound(f"Order '{order_id}' not found")
+        return result
 
     def _process_ome_response(self, report: SimulatedExecutionReport | None) -> SimulatedExecutionReport | None:
         if report is not None:

qubx/backtester/simulator.py

@@ -4,13 +4,15 @@ import pandas as pd
 from joblib import delayed
 
 from qubx import QubxLogConfig, logger
+from qubx.backtester.utils import SetupTypes
 from qubx.core.basics import Instrument
 from qubx.core.exceptions import SimulationError
 from qubx.core.metrics import TradingSessionResult
 from qubx.data.readers import DataReader
 from qubx.emitters.inmemory import InMemoryMetricEmitter
 from qubx.utils.misc import ProgressParallel, Stopwatch, get_current_user
-from qubx.utils.time import handle_start_stop
+from qubx.utils.runner.configs import PrefetchConfig
+from qubx.utils.time import handle_start_stop, to_utc_naive
 
 from .runner import SimulationRunner
 from .utils import (
@@ -31,10 +33,10 @@ def simulate(
     strategies: StrategiesDecls_t,
     data: DataDecls_t,
     capital: float | dict[str, float],
-    instruments: list[str] | list[Instrument] | dict[ExchangeName_t, list[SymbolOrInstrument_t]],
-    commissions: str | dict[str, str | None] | None,
     start: str | pd.Timestamp,
     stop: str | pd.Timestamp | None = None,
+    instruments: list[str] | list[Instrument] | dict[ExchangeName_t, list[SymbolOrInstrument_t]] | None = None,
+    commissions: str | dict[str, str | None] | None = None,
     exchange: ExchangeName_t | list[ExchangeName_t] | None = None,
     base_currency: str = "USDT",
     n_jobs: int = 1,
@@ -43,6 +45,7 @@ def simulate(
     accurate_stop_orders_execution: bool = False,
     signal_timeframe: str = "1Min",
     open_close_time_indent_secs=1,
+    enable_funding: bool = False,
     debug: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None = "WARNING",
     show_latency_report: bool = False,
     portfolio_log_freq: str = "5Min",
@@ -50,6 +53,7 @@ def simulate(
     enable_inmemory_emitter: bool = False,
     emitter_stats_interval: str = "1h",
     run_separate_instruments: bool = False,
+    prefetch_config: PrefetchConfig | None = None,
 ) -> list[TradingSessionResult]:
     """
     Backtest utility for trading strategies or signals using historical data.
@@ -70,6 +74,7 @@ def simulate(
         - accurate_stop_orders_execution (bool): If True, enables more accurate stop order execution simulation.
         - signal_timeframe (str): Timeframe for signals, default is "1Min".
         - open_close_time_indent_secs (int): Time indent in seconds for open/close times, default is 1.
+        - enable_funding (bool): If True, enables funding rate simulation, default is False.
         - debug (Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] | None): Logging level for debugging.
         - show_latency_report: If True, shows simulator's latency report.
         - portfolio_log_freq (str): Frequency for portfolio logging, default is "5Min".
@@ -77,6 +82,7 @@ def simulate(
         - enable_inmemory_emitter (bool): If True, attaches an in-memory metric emitter and returns its dataframe in TradingSessionResult.emitter_data.
         - emitter_stats_interval (str): Interval for emitting stats in the in-memory emitter (default: "1h").
         - run_separate_instruments (bool): If True, creates separate simulation setups for each instrument, default is False.
+        - prefetch_config (dict[str, Any] | None): Configuration for prefetching auxiliary data, default is None.
 
     Returns:
         - list[TradingSessionResult]: A list of TradingSessionResult objects containing the results of each simulation setup.
@@ -88,6 +94,9 @@ def simulate(
     # - we need to reset stopwatch
     Stopwatch().reset()
 
+    if instruments is None:
+        instruments = []
+
     # - process instruments:
     _instruments, _exchanges = find_instruments_and_exchanges(instruments, exchange)
 
@@ -100,7 +109,9 @@ def simulate(
         raise SimulationError(_msg)
 
     # - recognize provided data
-    data_setup = recognize_simulation_data_config(data, _instruments, open_close_time_indent_secs, aux_data)
+    data_setup = recognize_simulation_data_config(
+        data, _instruments, open_close_time_indent_secs, aux_data, prefetch_config
+    )
 
     # - recognize setup: it can be either a strategy or set of signals
     simulation_setups = recognize_simulation_configuration(
@@ -114,6 +125,7 @@ def simulate(
         signal_timeframe=signal_timeframe,
         accurate_stop_orders_execution=accurate_stop_orders_execution,
         run_separate_instruments=run_separate_instruments,
+        enable_funding=enable_funding,
     )
     if not simulation_setups:
         logger.error(
@@ -129,7 +141,7 @@ def simulate(
     # - preprocess start and stop and convert to datetime if necessary
     if stop is None:
         # - check stop time : here we try to backtest till now (may be we need to get max available time from data reader ?)
-        stop = pd.Timestamp.now(tz="UTC").astimezone(None)
+        stop = to_utc_naive(pd.Timestamp.now(tz="UTC"))
 
     _start, _stop = handle_start_stop(start, stop, convert=pd.Timestamp)
     assert isinstance(_start, pd.Timestamp) and isinstance(_stop, pd.Timestamp), "Invalid start and stop times"
@@ -204,6 +216,7 @@ def _run_setups(
                 portfolio_log_freq,
                 enable_inmemory_emitter,
                 emitter_stats_interval,
+                close_data_readers=True,
             )
             for id, setup in enumerate(strategies_setups)
         )
@@ -219,6 +232,20 @@ def _run_setups(
     return successful_reports
 
 
+def _adjust_start_date_for_min_instrument_onboard(setup: SimulationSetup, start: pd.Timestamp) -> pd.Timestamp:
+    """
+    Adjust the start date for the simulation to the onboard date of the instrument with the minimum onboard date.
+    """
+    onboard_dates = [
+        to_utc_naive(pd.Timestamp(instrument.onboard_date))
+        for instrument in setup.instruments
+        if instrument.onboard_date is not None
+    ]
+    if not onboard_dates:
+        return start
+    return max(start, min(onboard_dates))
+
+
 def _run_setup(
     setup_id: int,
     account_id: str,
@@ -231,12 +258,18 @@ def _run_setup(
     portfolio_log_freq: str,
     enable_inmemory_emitter: bool = False,
     emitter_stats_interval: str = "1h",
+    close_data_readers: bool = False,
 ) -> TradingSessionResult | None:
     try:
         emitter = None
         emitter_data = None
         if enable_inmemory_emitter:
             emitter = InMemoryMetricEmitter(stats_interval=emitter_stats_interval)
+
+        # TODO: this can be removed once we add some artificial data stream to move the simulation
+        if setup.setup_type in [SetupTypes.SIGNAL, SetupTypes.SIGNAL_AND_TRACKER]:
+            start = _adjust_start_date_for_min_instrument_onboard(setup, start)
+
         runner = SimulationRunner(
             setup=setup,
             data_config=data_setup,
@@ -252,7 +285,7 @@ def _run_setup(
             level=QubxLogConfig.get_log_level(), custom_formatter=SimulatedLogFormatter(runner.ctx).formatter
         )
 
-        runner.run(silent=silent)
+        runner.run(silent=silent, close_data_readers=close_data_readers)
 
         # - service latency report
         if show_latency_report: