Qubx 0.6.48__cp312-cp312-manylinux_2_39_x86_64.whl → 0.6.50__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/data.py

@@ -139,26 +139,31 @@ class SimulatedDataProvider(IDataProvider):
         bars = []
 
         # - if no records, return empty list to avoid exception from infer_series_frequency
-        if not records:
+        if not records or records is None:
             return bars
 
-        _data_tf = infer_series_frequency([r.time for r in records[:50]])
-        timeframe_ns = _data_tf.item()
-
-        if records is not None:
-            for r in records:
-                # _b_ts_0 = np.datetime64(r.time, "ns").item()
-                _b_ts_0 = r.time
-                _b_ts_1 = _b_ts_0 + timeframe_ns - self._open_close_time_indent_ns
-
-                if _b_ts_0 <= cut_time_ns and cut_time_ns < _b_ts_1:
-                    break
-
-                bars.append(
-                    Bar(
-                        _b_ts_0, r.data["open"], r.data["high"], r.data["low"], r.data["close"], r.data.get("volume", 0)
-                    )
+        if len(records) > 1:
+            _data_tf = infer_series_frequency([r.time for r in records[:50]])
+            timeframe_ns = _data_tf.item()
+
+        for r in records:
+            _b_ts_0 = r.time
+            _b_ts_1 = _b_ts_0 + timeframe_ns - self._open_close_time_indent_ns
+
+            if _b_ts_0 <= cut_time_ns and cut_time_ns < _b_ts_1:
+                break
+
+            bars.append(
+                Bar(
+                    _b_ts_0,
+                    r.data["open"],
+                    r.data["high"],
+                    r.data["low"],
+                    r.data["close"],
+                    r.data.get("volume", 0),
+                    r.data.get("bought_volume", 0),
                 )
+            )
 
         return bars
 

qubx/backtester/simulator.py

@@ -48,6 +48,7 @@ def simulate(
     portfolio_log_freq: str = "5Min",
     parallel_backend: Literal["loky", "multiprocessing"] = "multiprocessing",
     emission: EmissionConfig | None = None,
+    run_separate_instruments: bool = False,
 ) -> list[TradingSessionResult]:
     """
     Backtest utility for trading strategies or signals using historical data.
@@ -73,6 +74,7 @@ def simulate(
         - portfolio_log_freq (str): Frequency for portfolio logging, default is "5Min".
         - parallel_backend (Literal["loky", "multiprocessing"]): Backend for parallel processing, default is "multiprocessing".
         - emission (EmissionConfig | None): Configuration for metric emitters, default is None.
+        - run_separate_instruments (bool): If True, creates separate simulation setups for each instrument, default is False.
 
     Returns:
         - list[TradingSessionResult]: A list of TradingSessionResult objects containing the results of each simulation setup.
@@ -109,6 +111,7 @@ def simulate(
         commissions=commissions,
         signal_timeframe=signal_timeframe,
         accurate_stop_orders_execution=accurate_stop_orders_execution,
+        run_separate_instruments=run_separate_instruments,
     )
     if not simulation_setups:
         logger.error(
@@ -117,6 +120,10 @@ def simulate(
         )
         raise SimulationError(_msg)
 
+    # - inform about separate instruments mode
+    if run_separate_instruments and len(simulation_setups) > 1:
+        logger.info(f"Running separate simulations for each instrument. Total simulations: {len(simulation_setups)}")
+
     # - preprocess start and stop and convert to datetime if necessary
     if stop is None:
         # - check stop time : here we try to backtest till now (may be we need to get max available time from data reader ?)
@@ -160,24 +167,50 @@ def _run_setups(
     _main_loop_silent = len(strategies_setups) == 1
     n_jobs = 1 if _main_loop_silent else n_jobs
 
-    reports = ProgressParallel(
-        n_jobs=n_jobs, total=len(strategies_setups), silent=_main_loop_silent, backend=parallel_backend
-    )(
-        delayed(_run_setup)(
-            id,
-            f"Simulated-{id}",
-            setup,
-            data_setup,
-            start,
-            stop,
-            silent,
-            show_latency_report,
-            portfolio_log_freq,
-            emission,
+    if n_jobs == 1:
+        reports = [
+            _run_setup(
+                id,
+                f"Simulated-{id}",
+                setup,
+                data_setup,
+                start,
+                stop,
+                silent,
+                show_latency_report,
+                portfolio_log_freq,
+                emission,
+            )
+            for id, setup in enumerate(strategies_setups)
+        ]
+    else:
+        reports = ProgressParallel(
+            n_jobs=n_jobs, total=len(strategies_setups), silent=_main_loop_silent, backend=parallel_backend
+        )(
+            delayed(_run_setup)(
+                id,
+                f"Simulated-{id}",
+                setup,
+                data_setup,
+                start,
+                stop,
+                silent,
+                show_latency_report,
+                portfolio_log_freq,
+                emission,
+            )
+            for id, setup in enumerate(strategies_setups)
         )
-        for id, setup in enumerate(strategies_setups)
-    )
-    return reports  # type: ignore
+
+    # Filter out None results and log warnings for failed simulations
+    successful_reports = []
+    for i, report in enumerate(reports):
+        if report is None:
+            logger.warning(f"Simulation setup {i} failed - skipping from results")
+        else:
+            successful_reports.append(report)
+
+    return successful_reports
 
 
 def _run_setup(
@@ -191,48 +224,58 @@ def _run_setup(
     show_latency_report: bool,
     portfolio_log_freq: str,
     emission: EmissionConfig | None = None,
-) -> TradingSessionResult:
-    # Create metric emitter if configured
-    emitter = None
-    if emission is not None:
-        emitter = create_metric_emitters(emission, setup.name)
-
-    runner = SimulationRunner(
-        setup=setup,
-        data_config=data_setup,
-        start=start,
-        stop=stop,
-        account_id=account_id,
-        portfolio_log_freq=portfolio_log_freq,
-        emitter=emitter,
-    )
+) -> TradingSessionResult | None:
+    try:
+        # Create metric emitter if configured
+        emitter = None
+        if emission is not None:
+            emitter = create_metric_emitters(emission, setup.name)
 
-    # - we want to see simulate time in log messages
-    QubxLogConfig.setup_logger(
-        level=QubxLogConfig.get_log_level(), custom_formatter=SimulatedLogFormatter(runner.ctx).formatter
-    )
+        runner = SimulationRunner(
+            setup=setup,
+            data_config=data_setup,
+            start=start,
+            stop=stop,
+            account_id=account_id,
+            portfolio_log_freq=portfolio_log_freq,
+            emitter=emitter,
+        )
 
-    runner.run(silent=silent)
-
-    # - service latency report
-    if show_latency_report:
-        runner.print_latency_report()
-
-    return TradingSessionResult(
-        setup_id,
-        setup.name,
-        start,
-        stop,
-        setup.exchanges,
-        setup.instruments,
-        setup.capital,
-        setup.base_currency,
-        setup.commissions,
-        runner.logs_writer.get_portfolio(as_plain_dataframe=True),
-        runner.logs_writer.get_executions(),
-        runner.logs_writer.get_signals(),
-        strategy_class=runner.strategy_class,
-        parameters=runner.strategy_params,
-        is_simulation=True,
-        author=get_current_user(),
-    )
+        # - we want to see simulate time in log messages
+        QubxLogConfig.setup_logger(
+            level=QubxLogConfig.get_log_level(), custom_formatter=SimulatedLogFormatter(runner.ctx).formatter
+        )
+
+        runner.run(silent=silent)
+
+        # - service latency report
+        if show_latency_report:
+            runner.print_latency_report()
+
+        # Convert commissions to the expected type for TradingSessionResult
+        commissions_for_result = setup.commissions
+        if isinstance(commissions_for_result, dict):
+            # Filter out None values to match TradingSessionResult expected type
+            commissions_for_result = {k: v for k, v in commissions_for_result.items() if v is not None}
+
+        return TradingSessionResult(
+            setup_id,
+            setup.name,
+            start,
+            stop,
+            setup.exchanges,
+            setup.instruments,
+            setup.capital,
+            setup.base_currency,
+            commissions_for_result,
+            runner.logs_writer.get_portfolio(as_plain_dataframe=True),
+            runner.logs_writer.get_executions(),
+            runner.logs_writer.get_signals(),
+            strategy_class=runner.strategy_class,
+            parameters=runner.strategy_params,
+            is_simulation=True,
+            author=get_current_user(),
+        )
+    except Exception as e:
+        logger.error(f"Simulation setup {setup_id} failed with error: {e}")
+        return None

qubx/backtester/utils.py

@@ -419,6 +419,7 @@ def recognize_simulation_configuration(
     commissions: str | dict[str, str | None] | None,
     signal_timeframe: str,
     accurate_stop_orders_execution: bool,
+    run_separate_instruments: bool = False,
 ) -> list[SimulationSetup]:
     """
     Recognize and create setups based on the provided simulation configuration.
@@ -438,6 +439,7 @@ def recognize_simulation_configuration(
     - commissions (str): The commission structure to be applied.
     - signal_timeframe (str): Timeframe for generated signals.
     - accurate_stop_orders_execution (bool): If True, enables more accurate stop order execution simulation.
+    - run_separate_instruments (bool): If True, creates separate setups for each instrument.
 
     Returns:
     - list[SimulationSetup]: A list of SimulationSetup objects, each representing a
@@ -458,7 +460,7 @@ def recognize_simulation_configuration(
             r.extend(
                 recognize_simulation_configuration(
                     _n + n, v, instruments, exchanges, capital, basic_currency, commissions, 
-                    signal_timeframe, accurate_stop_orders_execution
+                    signal_timeframe, accurate_stop_orders_execution, run_separate_instruments
                 )
             )
 
@@ -474,45 +476,85 @@ def recognize_simulation_configuration(
                 _t = SetupTypes.STRATEGY_AND_TRACKER
 
             # - extract actual symbols that have signals
-            r.append(
-                SimulationSetup(
-                    _t, name, _s, c1,   # type: ignore
-                    _sniffer._pick_instruments(instruments, _s) if _sniffer._is_signal(c0) else instruments,
-                    exchanges, capital, basic_currency, commissions, 
-                    signal_timeframe, accurate_stop_orders_execution
+            setup_instruments = _sniffer._pick_instruments(instruments, _s) if _sniffer._is_signal(c0) else instruments
+            
+            if run_separate_instruments:
+                # Create separate setups for each instrument
+                for instrument in setup_instruments:
+                    r.append(
+                        SimulationSetup(
+                            _t, f"{name}/{instrument.symbol}", _s, c1,   # type: ignore
+                            [instrument],
+                            exchanges, capital, basic_currency, commissions, 
+                            signal_timeframe, accurate_stop_orders_execution
+                        )
+                    )
+            else:
+                r.append(
+                    SimulationSetup(
+                        _t, name, _s, c1,   # type: ignore
+                        setup_instruments,
+                        exchanges, capital, basic_currency, commissions, 
+                        signal_timeframe, accurate_stop_orders_execution
+                    )
                 )
-            )
         else:
             for j, s in enumerate(configs):
                 r.extend(
                     recognize_simulation_configuration(
                         # name + "/" + str(j), s, instruments, exchange, capital, basic_currency, commissions
                         name, s, instruments, exchanges, capital, basic_currency, commissions,  # type: ignore
-                        signal_timeframe, accurate_stop_orders_execution
+                        signal_timeframe, accurate_stop_orders_execution, run_separate_instruments
                     )
                 )
 
     elif _sniffer._is_strategy(configs):
-        r.append(
-            SimulationSetup(
-                SetupTypes.STRATEGY,
-                name, configs, None, instruments,
-                exchanges, capital, basic_currency, commissions, 
-                signal_timeframe, accurate_stop_orders_execution
+        if run_separate_instruments:
+            # Create separate setups for each instrument
+            for instrument in instruments:
+                r.append(
+                    SimulationSetup(
+                        SetupTypes.STRATEGY,
+                        f"{name}/{instrument.symbol}", configs, None, [instrument],
+                        exchanges, capital, basic_currency, commissions, 
+                        signal_timeframe, accurate_stop_orders_execution
+                    )
+                )
+        else:
+            r.append(
+                SimulationSetup(
+                    SetupTypes.STRATEGY,
+                    name, configs, None, instruments,
+                    exchanges, capital, basic_currency, commissions, 
+                    signal_timeframe, accurate_stop_orders_execution
+                )
             )
-        )
 
     elif _sniffer._is_signal(configs):
         # - check structure of signals
         c1 = _sniffer._check_signals_structure(instruments, configs)  # type: ignore
-        r.append(
-            SimulationSetup(
-                SetupTypes.SIGNAL,
-                name, c1, None, _sniffer._pick_instruments(instruments, c1),
-                exchanges, capital, basic_currency, commissions, 
-                signal_timeframe, accurate_stop_orders_execution
+        setup_instruments = _sniffer._pick_instruments(instruments, c1)
+        
+        if run_separate_instruments:
+            # Create separate setups for each instrument
+            for instrument in setup_instruments:
+                r.append(
+                    SimulationSetup(
+                        SetupTypes.SIGNAL,
+                        f"{name}/{instrument.symbol}", c1, None, [instrument],
+                        exchanges, capital, basic_currency, commissions, 
+                        signal_timeframe, accurate_stop_orders_execution
+                    )
+                )
+        else:
+            r.append(
+                SimulationSetup(
+                    SetupTypes.SIGNAL,
+                    name, c1, None, setup_instruments,
+                    exchanges, capital, basic_currency, commissions, 
+                    signal_timeframe, accurate_stop_orders_execution
+                )
             )
-        )
 
     # fmt: on
     return r

qubx/core/loggers.py

@@ -10,10 +10,8 @@ from qubx.core.basics import (
     Position,
     TargetPosition,
 )
-
 from qubx.core.series import time_as_nsec
 from qubx.core.utils import recognize_timeframe
-
 from qubx.utils.misc import Stopwatch
 from qubx.utils.time import convert_tf_str_td64, floor_t64
 
@@ -21,14 +19,14 @@ _SW = Stopwatch()
 
 
 class LogsWriter:
-    account_id: str
-    strategy_id: str
-    run_id: str
-
     """
     Log writer interface with default implementation
     """
 
+    account_id: str
+    strategy_id: str
+    run_id: str
+
     def __init__(self, account_id: str, strategy_id: str, run_id: str) -> None:
         self.account_id = account_id
         self.strategy_id = strategy_id

qubx/core/mixins/processing.py

@@ -39,7 +39,8 @@ from qubx.core.series import Bar, OrderBook, Quote, Trade
 
 
 class ProcessingManager(IProcessingManager):
-    MAX_NUMBER_OF_STRATEGY_FAILURES = 10
+    MAX_NUMBER_OF_STRATEGY_FAILURES: int = 10
+    DATA_READY_TIMEOUT_SECONDS: int = 60
 
     _context: IStrategyContext
     _strategy: IStrategy
@@ -67,6 +68,7 @@ class ProcessingManager(IProcessingManager):
     _trig_bar_freq_nsec: int | None = None
     _cur_sim_step: int | None = None
     _updated_instruments: set[Instrument] = set()
+    _data_ready_start_time: dt_64 | None = None
 
     def __init__(
         self,
@@ -111,6 +113,7 @@ class ProcessingManager(IProcessingManager):
         self._strategy_name = strategy.__class__.__name__
         self._trig_bar_freq_nsec = None
         self._updated_instruments = set()
+        self._data_ready_start_time = None
 
     def set_fit_schedule(self, schedule: str) -> None:
         rule = process_schedule_spec(schedule)
@@ -344,9 +347,56 @@ class ProcessingManager(IProcessingManager):
 
     def _is_data_ready(self) -> bool:
         """
-        Check if at least one update was received for all instruments in the context.
+        Check if strategy can start based on data availability with timeout logic.
+
+        Two-phase approach:
+        - Phase 1 (0-DATA_READY_TIMEOUT_SECONDS): Wait for ALL instruments to have data
+        - Phase 2 (after timeout): Wait for at least 1 instrument to have data
+
+        Returns:
+            bool: True if strategy can start, False if still waiting
         """
-        return all(instrument in self._updated_instruments for instrument in self._context.instruments)
+        total_instruments = len(self._context.instruments)
+
+        # Handle edge case: no instruments
+        if total_instruments == 0:
+            return True
+
+        ready_instruments = len(self._updated_instruments)
+
+        # Record start time on first call
+        if self._data_ready_start_time is None:
+            self._data_ready_start_time = self._time_provider.time()
+
+        # Phase 1: Try to get all instruments ready within timeout
+        elapsed_time_seconds = (self._time_provider.time() - self._data_ready_start_time) / 1e9
+
+        if elapsed_time_seconds <= self.DATA_READY_TIMEOUT_SECONDS:
+            # Within timeout period - wait for ALL instruments
+            if ready_instruments == total_instruments:
+                logger.info(f"All {total_instruments} instruments have data - strategy ready to start")
+                return True
+            else:
+                # Log periodic status during Phase 1
+                if int(elapsed_time_seconds) % 10 == 0 and elapsed_time_seconds > 0:  # Log every 10 seconds
+                    missing_instruments = set(self._context.instruments) - self._updated_instruments
+                    missing_symbols = [inst.symbol for inst in missing_instruments]
+                    logger.info(
+                        f"Phase 1: Waiting for all instruments ({ready_instruments}/{total_instruments} ready). "
+                        f"Missing: {missing_symbols}. Timeout in {self.DATA_READY_TIMEOUT_SECONDS - elapsed_time_seconds:.1f}s"
+                    )
+                return False
+        else:
+            # Phase 2: After timeout - need at least 1 instrument
+            if ready_instruments >= 1:
+                missing_instruments = set(self._context.instruments) - self._updated_instruments
+                missing_symbols = [inst.symbol for inst in missing_instruments]
+                logger.info(
+                    f"Starting strategy with {ready_instruments}/{total_instruments} instruments ready. Missing: {missing_symbols}"
+                )
+                return True
+            else:
+                return False
 
     def __update_base_data(
         self, instrument: Instrument, event_type: str, data: Timestamped, is_historical: bool = False

qubx/core/mixins/universe.py

@@ -2,8 +2,6 @@ from qubx.core.basics import DataType, Instrument, TargetPosition
 from qubx.core.helpers import CachedMarketDataHolder
 from qubx.core.interfaces import (
     IAccountProcessor,
-    IBroker,
-    IDataProvider,
     IPositionGathering,
     IStrategy,
     IStrategyContext,
@@ -14,7 +12,6 @@ from qubx.core.interfaces import (
     RemovalPolicy,
 )
 from qubx.core.loggers import StrategyLogging
-from qubx.core.lookups import lookup
 
 
 class UniverseManager(IUniverseManager):
@@ -50,7 +47,7 @@ class UniverseManager(IUniverseManager):
         self._time_provider = time_provider
         self._account = account
         self._position_gathering = position_gathering
-        self._instruments = []
+        self._instruments = set()
         self._removal_queue = {}
 
     def _has_position(self, instrument: Instrument) -> bool:
@@ -72,7 +69,7 @@ class UniverseManager(IUniverseManager):
         ), "Invalid if_has_position_then policy"
 
         new_set = set(instruments)
-        prev_set = set(self._instruments)
+        prev_set = self._instruments.copy()
 
         # - determine instruments to remove depending on if_has_position_then policy
         may_be_removed = list(prev_set - new_set)
@@ -93,9 +90,7 @@ class UniverseManager(IUniverseManager):
         self._subscription_manager.commit()  # apply pending changes
 
         # set new instruments
-        self._instruments.clear()
-        self._instruments.extend(instruments)
-        self._instruments.extend(to_keep)
+        self._instruments = new_set | set(to_keep)
 
     def _get_what_can_be_removed_or_kept(
         self, may_be_removed: list[Instrument], skip_callback: bool, if_has_position_then: RemovalPolicy
@@ -105,12 +100,11 @@ class UniverseManager(IUniverseManager):
         for instr in may_be_removed:
             if immediately_close:
                 to_remove.append(instr)
+            elif self._has_position(instr):
+                self._removal_queue[instr] = (if_has_position_then, skip_callback)
+                to_keep.append(instr)
             else:
-                if self._has_position(instr):
-                    self._removal_queue[instr] = (if_has_position_then, skip_callback)
-                    to_keep.append(instr)
-                else: 
-                    to_remove.append(instr)
+                to_remove.append(instr)
         return to_remove, to_keep
 
     def __cleanup_removal_queue(self, instruments: list[Instrument]):
@@ -124,7 +118,7 @@ class UniverseManager(IUniverseManager):
         self.__cleanup_removal_queue(instruments)
         self._strategy.on_universe_change(self._context, instruments, [])
         self._subscription_manager.commit()
-        self._instruments.extend(instruments)
+        self._instruments.update(instruments)
 
     def remove_instruments(
         self,
@@ -146,12 +140,11 @@ class UniverseManager(IUniverseManager):
         self._subscription_manager.commit()
 
         # - update instruments list
-        self._instruments = list(set(self._instruments) - set(to_remove))
-        self._instruments.extend(to_keep)
+        self._instruments = (self._instruments - set(to_remove)) | set(to_keep)
 
     @property
     def instruments(self) -> list[Instrument]:
-        return self._instruments
+        return list(self._instruments)
 
     def __do_remove_instruments(self, instruments: list[Instrument]):
         """
@@ -227,7 +220,7 @@ class UniverseManager(IUniverseManager):
             # if aux is not None:
             #     instrument._aux_instrument = aux
             #     instruments.append(aux)
-            #     _ = self._trading_service.get_position(aux)
+            #     _ = self._account.get_position(aux)
 
     def on_alter_position(self, instrument: Instrument) -> None:
         """
@@ -247,7 +240,7 @@ class UniverseManager(IUniverseManager):
 
                 # - commit changes and remove instrument from the universe
                 self._subscription_manager.commit()
-                self._instruments.remove(instrument)
+                self._instruments.discard(instrument)
 
     def is_trading_allowed(self, instrument: Instrument) -> bool:
         if instrument in self._removal_queue:
@@ -261,7 +254,7 @@ class UniverseManager(IUniverseManager):
 
                 # - commit changes and remove instrument from the universe
                 self._subscription_manager.commit()
-                self._instruments.remove(instrument)
+                self._instruments.discard(instrument)
                 return False
 
         return True

qubx/data/composite.py

@@ -71,12 +71,15 @@ class IteratedDataStreamsSlicer(Iterator[SlicerOutData]):
         return self
 
     def _build_initial_iteration_seq(self):
-        _init_seq = {k: self._time_func(self._buffers[k][-1]) for k in self._keys}
+        _init_seq = {k: self._time_func(self._buffers[k][-1]) for k in self._keys if self._buffers[k]}
         _init_seq = dict(sorted(_init_seq.items(), key=lambda item: item[1]))
         self._keys = deque(_init_seq.keys())
 
     def _load_next_chunk_to_buffer(self, index: str) -> list[Timestamped]:
-        return list(reversed(next(self._iterators[index])))
+        try:
+            return list(reversed(next(self._iterators[index])))
+        except StopIteration:
+            return []
 
     def _remove_iterator(self, key: str):
         self._buffers.pop(key)
@@ -95,6 +98,9 @@ class IteratedDataStreamsSlicer(Iterator[SlicerOutData]):
         Returns:
             Timestamped: The most recent timestamped data element from the buffer.
         """
+        if not self._buffers[k]:
+            raise StopIteration
+
         v = (data := self._buffers[k]).pop()
         if not data:
             try:
@@ -154,6 +160,9 @@ class IteratedDataStreamsSlicer(Iterator[SlicerOutData]):
         _min_t = math.inf
         _min_k = self._keys[0]
         for i in self._keys:
+            if not self._buffers[i]:
+                continue
+
             _x = self._buffers[i][-1]
             if self._time_func(_x) < _min_t:
                 _min_t = self._time_func(_x)