Qubx 0.6.1__cp312-cp312-manylinux_2_39_x86_64.whl → 0.6.3__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/account.py

@@ -12,7 +12,8 @@ from qubx.core.basics import (
     dt_64,
 )
 from qubx.core.interfaces import ITimeProvider
-from qubx.core.series import Bar, OrderBook, Quote, Trade
+from qubx.core.series import Bar, OrderBook, Quote, Trade, TradeArray
+from qubx.restorers import RestoredState
 
 
 class SimulatedAccountProcessor(BasicAccountProcessor):
@@ -32,6 +33,7 @@ class SimulatedAccountProcessor(BasicAccountProcessor):
         time_provider: ITimeProvider,
         tcc: TransactionCostsCalculator = ZERO_COSTS,
         accurate_stop_orders_execution: bool = False,
+        restored_state: RestoredState | None = None,
     ) -> None:
         super().__init__(
             account_id=account_id,
@@ -48,6 +50,12 @@ class SimulatedAccountProcessor(BasicAccountProcessor):
         if self._fill_stop_order_at_price:
             logger.info(f"[<y>{self.__class__.__name__}</y>] :: emulates stop orders executions at exact price")
 
+        if restored_state is not None:
+            self._balances.update(restored_state.balances)
+            for instrument, position in restored_state.positions.items():
+                _pos = self.get_position(instrument)
+                _pos.reset_by_position(position)
+
     def get_orders(self, instrument: Instrument | None = None) -> dict[str, Order]:
         if instrument is not None:
             ome = self.ome.get(instrument)
@@ -76,20 +84,27 @@ class SimulatedAccountProcessor(BasicAccountProcessor):
         self.attach_positions(position)
         return self.positions[instrument]
 
-    def update_position_price(self, time: dt_64, instrument: Instrument, price: float) -> None:
-        super().update_position_price(time, instrument, price)
+    def update_position_price(self, time: dt_64, instrument: Instrument, update: float | Timestamped) -> None:
+        super().update_position_price(time, instrument, update)
 
         # - first we need to update OME with new quote.
         # - if update is not a quote we need 'emulate' it.
         # - actually if SimulatedExchangeService is used in backtesting mode it will recieve only quotes
         # - case when we need that - SimulatedExchangeService is used for paper trading and data provider configured to listen to OHLC or TAS.
         # - probably we need to subscribe to quotes in real data provider in any case and then this emulation won't be needed.
-        quote = price if isinstance(price, Quote) else self.emulate_quote_from_data(instrument, time, price)
+        quote = update if isinstance(update, Quote) else self.emulate_quote_from_data(instrument, time, update)
         if quote is None:
             return
 
-        # - process new quote
-        self._process_new_quote(instrument, quote)
+        # - process new data
+        self._process_new_data(instrument, quote)
+
+    def process_market_data(self, time: dt_64, instrument: Instrument, update: Timestamped) -> None:
+        if isinstance(update, (TradeArray, Quote, Trade, OrderBook)):
+            # - process new data
+            self._process_new_data(instrument, update)
+
+        super().process_market_data(time, instrument, update)
 
     def process_order(self, order: Order, update_locked_value: bool = True) -> None:
         _new = order.status == "NEW"
@@ -113,7 +128,7 @@ class SimulatedAccountProcessor(BasicAccountProcessor):
 
         elif isinstance(data, Trade):
             _ts2 = self._half_tick_size[instrument]
-            if data.taker:  # type: ignore
+            if data.side == 1:  # type: ignore
                 return Quote(timestamp, data.price - _ts2 * 2, data.price, 0, 0)  # type: ignore
             else:
                 return Quote(timestamp, data.price, data.price + _ts2 * 2, 0, 0)  # type: ignore
@@ -132,12 +147,12 @@ class SimulatedAccountProcessor(BasicAccountProcessor):
         else:
             return None
 
-    def _process_new_quote(self, instrument: Instrument, data: Quote) -> None:
+    def _process_new_data(self, instrument: Instrument, data: Quote | OrderBook | Trade | TradeArray) -> None:
         ome = self.ome.get(instrument)
         if ome is None:
-            logger.warning("ExchangeService:update :: No OME configured for '{symbol}' yet !")
+            logger.warning(f"ExchangeService:update :: No OME configured for '{instrument}' yet !")
             return
-        for r in ome.update_bbo(data):
+        for r in ome.process_market_data(data):
             if r.exec is not None:
                 self.order_to_instrument.pop(r.order.id)
                 # - process methods will be called from stg context

qubx/backtester/data.py

@@ -151,7 +151,7 @@ class SimulatedDataProvider(IDataProvider):
                     self._last_quotes[i] = last_quote
 
                     # - also need to pass this quote to OME !
-                    self._account._process_new_quote(i, last_quote)
+                    self._account._process_new_data(i, last_quote)
 
                     logger.debug(f" | subscribed {subscription_type} {i} -> {last_quote}")
 

qubx/backtester/ome.py

@@ -19,8 +19,9 @@ from qubx.core.basics import (
 from qubx.core.exceptions import (
     ExchangeError,
     InvalidOrder,
+    SimulationError,
 )
-from qubx.core.series import Quote
+from qubx.core.series import OrderBook, Quote, Trade, TradeArray
 
 
 @dataclass
@@ -31,13 +32,18 @@ class OmeReport:
 
 
 class OrdersManagementEngine:
+    """
+    Orders Management Engine (OME) is a simple implementation of a management of orders for simulation of a limit order book.
+    """
+
     instrument: Instrument
     time_service: ITimeProvider
     active_orders: dict[str, Order]
     stop_orders: dict[str, Order]
     asks: SortedDict[float, list[str]]
     bids: SortedDict[float, list[str]]
-    bbo: Quote | None  # current best bid/ask order book (simplest impl)
+    bbo: Quote | None  # - current best bid/ask order book
+    __prev_bbo: Quote | None  # - previous best bid/ask order book
     __order_id: int
     __trade_id: int
     _fill_stops_at_price: bool
@@ -73,46 +79,80 @@ class OrdersManagementEngine:
         return "SIM-EXEC-" + self.instrument.symbol + "-" + str(self.__trade_id)
 
     def get_quote(self) -> Quote:
-        return self.bbo
+        return self.bbo  # type: ignore
 
     def get_open_orders(self) -> list[Order]:
         return list(self.active_orders.values()) + list(self.stop_orders.values())
 
-    def update_bbo(self, quote: Quote) -> list[OmeReport]:
+    def process_market_data(self, mdata: Quote | OrderBook | Trade | TradeArray) -> list[OmeReport]:
+        """
+        Processes the new market data (quote, trade or trades array) and simulates the execution of pending orders.
+        """
         timestamp = self.time_service.time()
-        rep = []
-
-        if self.bbo is not None:
-            if quote.bid >= self.bbo.ask:
-                for level in self.asks.irange(0, quote.bid):
-                    for order_id in self.asks[level]:
-                        order = self.active_orders.pop(order_id)
-                        rep.append(self._execute_order(timestamp, order.price, order, False))
-                    self.asks.pop(level)
-
-            if quote.ask <= self.bbo.bid:
-                for level in self.bids.irange(np.inf, quote.ask):
-                    for order_id in self.bids[level]:
-                        order = self.active_orders.pop(order_id)
-                        rep.append(self._execute_order(timestamp, order.price, order, False))
-                    self.bids.pop(level)
-
-            # - processing stop orders
-            for soid in list(self.stop_orders.keys()):
-                so = self.stop_orders[soid]
-                _emulate_price_exec = self._fill_stops_at_price or so.options.get(OPTION_FILL_AT_SIGNAL_PRICE, False)
-
-                if so.side == "BUY" and quote.ask >= so.price:
-                    _exec_price = quote.ask if not _emulate_price_exec else so.price
-                    self.stop_orders.pop(soid)
-                    rep.append(self._execute_order(timestamp, _exec_price, so, True))
-                elif so.side == "SELL" and quote.bid <= so.price:
-                    _exec_price = quote.bid if not _emulate_price_exec else so.price
-                    self.stop_orders.pop(soid)
-                    rep.append(self._execute_order(timestamp, _exec_price, so, True))
-
-        self.bbo = quote
-        return rep
+        _exec_report = []
+
+        # - new quote
+        if isinstance(mdata, Quote):
+            _b, _a = mdata.bid, mdata.ask
+            _bs, _as = _b, _a
+
+            # - update BBO by new quote
+            self.__prev_bbo = self.bbo
+            self.bbo = mdata
+
+        # - bunch of trades
+        elif isinstance(mdata, TradeArray):
+            _b = mdata.max_buy_price
+            _a = mdata.min_sell_price
+            _bs, _as = _a, _b
+
+        # - single trade
+        elif isinstance(mdata, Trade):
+            _b, _a = mdata.price, mdata.price
+            _bs, _as = _b, _a
+
+        # - order book
+        elif isinstance(mdata, OrderBook):
+            _b, _a = mdata.top_bid, mdata.top_ask
+            _bs, _as = _b, _a
+
+        else:
+            raise SimulationError(f"Invalid market data type: {type(mdata)} for update OME({self.instrument.symbol})")
+
+        # - when new quote bid is higher than the lowest ask order execute all affected orders
+        if self.asks and _b >= self.asks.keys()[0]:
+            _asks_to_execute = list(self.asks.irange(0, _b))
+            for level in _asks_to_execute:
+                for order_id in self.asks[level]:
+                    order = self.active_orders.pop(order_id)
+                    _exec_report.append(self._execute_order(timestamp, order.price, order, False))
+                self.asks.pop(level)
+
+        # - when new quote ask is lower than the highest bid order execute all affected orders
+        if self.bids and _a <= self.bids.keys()[0]:
+            _bids_to_execute = list(self.bids.irange(np.inf, _a))
+            for level in _bids_to_execute:
+                for order_id in self.bids[level]:
+                    order = self.active_orders.pop(order_id)
+                    _exec_report.append(self._execute_order(timestamp, order.price, order, False))
+                self.bids.pop(level)
+
+        # - processing stop orders
+        for soid in list(self.stop_orders.keys()):
+            so = self.stop_orders[soid]
+            _emulate_price_exec = self._fill_stops_at_price or so.options.get(OPTION_FILL_AT_SIGNAL_PRICE, False)
+
+            if so.side == "BUY" and _as >= so.price:
+                _exec_price = _as if not _emulate_price_exec else so.price
+                self.stop_orders.pop(soid)
+                _exec_report.append(self._execute_order(timestamp, _exec_price, so, True))
+
+            elif so.side == "SELL" and _bs <= so.price:
+                _exec_price = _bs if not _emulate_price_exec else so.price
+                self.stop_orders.pop(soid)
+                _exec_report.append(self._execute_order(timestamp, _exec_price, so, True))
+
+        return _exec_report
 
     def place_order(
         self,
@@ -163,7 +203,23 @@ class OrdersManagementEngine:
         _need_update_book = False
 
         if order.type == "MARKET":
-            exec_price = c_ask if buy_side else c_bid
+            if exec_price is None:
+                exec_price = c_ask if buy_side else c_bid
+
+            # - special case - fill at signal price for market order
+            # - only for simulation
+            # - only if this is valid price: market crossed this desired price on last update
+            if order.options.get(OPTION_FILL_AT_SIGNAL_PRICE, False) and order.price and self.__prev_bbo:
+                _desired_fill_price = order.price
+
+                if (buy_side and self.__prev_bbo.ask < _desired_fill_price <= c_ask) or (
+                    not buy_side and self.__prev_bbo.bid > _desired_fill_price >= c_bid
+                ):
+                    exec_price = _desired_fill_price
+                else:
+                    raise SimulationError(
+                        f"Special execution price at {_desired_fill_price} for market order {order.id} cannot be filled because market didn't cross this price on last update !"
+                    )
 
         elif order.type == "LIMIT":
             _need_update_book = True

qubx/backtester/runner.py

@@ -0,0 +1,248 @@
+from typing import Any
+
+import numpy as np
+import pandas as pd
+
+from qubx import logger
+from qubx.core.basics import SW, DataType
+from qubx.core.context import StrategyContext
+from qubx.core.exceptions import SimulationConfigError, SimulationError
+from qubx.core.helpers import extract_parameters_from_object, full_qualified_class_name
+from qubx.core.interfaces import IStrategy, IStrategyContext
+from qubx.core.loggers import InMemoryLogsWriter, StrategyLogging
+from qubx.core.lookups import lookup
+from qubx.pandaz.utils import _frame_to_str
+
+from .account import SimulatedAccountProcessor
+from .broker import SimulatedBroker
+from .data import SimulatedDataProvider
+from .utils import (
+    SetupTypes,
+    SignalsProxy,
+    SimulatedCtrlChannel,
+    SimulatedScheduler,
+    SimulatedTimeProvider,
+    SimulationDataConfig,
+    SimulationSetup,
+)
+
+
+class SimulationRunner:
+    """
+    A wrapper around the StrategyContext that encapsulates the simulation logic.
+    This class is responsible for running a backtest context from a start time to an end time.
+    """
+
+    setup: SimulationSetup
+    data_config: SimulationDataConfig
+    start: pd.Timestamp
+    stop: pd.Timestamp
+    account_id: str
+    portfolio_log_freq: str
+    ctx: IStrategyContext
+    data_provider: SimulatedDataProvider
+    logs_writer: InMemoryLogsWriter
+
+    strategy_params: dict[str, Any]
+    strategy_class: str
+
+    # adjusted times
+    _stop: pd.Timestamp | None = None
+
+    def __init__(
+        self,
+        setup: SimulationSetup,
+        data_config: SimulationDataConfig,
+        start: pd.Timestamp | str,
+        stop: pd.Timestamp | str,
+        account_id: str = "SimulatedAccount",
+        portfolio_log_freq: str = "5Min",
+    ):
+        """
+        Initialize the BacktestContextRunner with a strategy context.
+
+        Args:
+            setup (SimulationSetup): The setup to run.
+            data_config (SimulationDataConfig): The data setup to use.
+            start (pd.Timestamp): The start time of the simulation.
+            stop (pd.Timestamp): The end time of the simulation.
+            account_id (str): The account id to use.
+            portfolio_log_freq (str): The portfolio log frequency to use.
+        """
+        self.setup = setup
+        self.data_config = data_config
+        self.start = pd.Timestamp(start)
+        self.stop = pd.Timestamp(stop)
+        self.account_id = account_id
+        self.portfolio_log_freq = portfolio_log_freq
+        self.ctx = self._create_backtest_context()
+
+        # - get strategy parameters BEFORE simulation start
+        #   potentially strategy may change it's parameters during simulation
+        self.strategy_params = {}
+        self.strategy_class = ""
+        if self.setup.setup_type in [SetupTypes.STRATEGY, SetupTypes.STRATEGY_AND_TRACKER]:
+            self.strategy_params = extract_parameters_from_object(self.setup.generator)
+            self.strategy_class = full_qualified_class_name(self.setup.generator)
+
+    def run(self, silent: bool = False):
+        """
+        Run the backtest from start to stop.
+
+        Args:
+            start (pd.Timestamp | str): The start time of the simulation.
+            stop (pd.Timestamp | str): The end time of the simulation.
+            silent (bool, optional): Whether to suppress progress output. Defaults to False.
+        """
+        logger.debug(f"[<y>BacktestContextRunner</y>] :: Running simulation from {self.start} to {self.stop}")
+
+        # Start the context
+        self.ctx.start()
+
+        # Apply default warmup periods if strategy didn't set them
+        for s in self.ctx.get_subscriptions():
+            if not self.ctx.get_warmup(s) and (_d_wt := self.data_config.default_warmups.get(s)):
+                logger.debug(
+                    f"[<y>BacktestContextRunner</y>] :: Strategy didn't set warmup period for <c>{s}</c> so default <c>{_d_wt}</c> will be used"
+                )
+                self.ctx.set_warmup({s: _d_wt})
+
+        # Subscribe to any custom data types if needed
+        def _is_known_type(t: str):
+            try:
+                DataType(t)
+                return True
+            except:  # noqa: E722
+                return False
+
+        for t, r in self.data_config.data_providers.items():
+            if not _is_known_type(t) or t in [
+                DataType.TRADE,
+                DataType.OHLC_TRADES,
+                DataType.OHLC_QUOTES,
+                DataType.QUOTE,
+                DataType.ORDERBOOK,
+            ]:
+                logger.debug(f"[<y>BacktestContextRunner</y>] :: Subscribing to: {t}")
+                self.ctx.subscribe(t, self.ctx.instruments)
+
+        stop = self._stop or self.stop
+
+        try:
+            # Run the data provider
+            self.data_provider.run(self.start, stop, silent=silent)
+        except KeyboardInterrupt:
+            logger.error("Simulated trading interrupted by user!")
+        finally:
+            # Stop the context
+            self.ctx.stop()
+
+    def print_latency_report(self) -> None:
+        _l_r = SW.latency_report()
+        if _l_r is not None:
+            logger.info(
+                "<BLUE>   Time spent in simulation report   </BLUE>\n<r>"
+                + _frame_to_str(
+                    _l_r.sort_values("latency", ascending=False).reset_index(drop=True), "simulation", -1, -1, False
+                )
+                + "</r>"
+            )
+
+    def _create_backtest_context(self) -> IStrategyContext:
+        tcc = lookup.fees.find(self.setup.exchange.lower(), self.setup.commissions)
+        if tcc is None:
+            raise SimulationConfigError(
+                f"Can't find transaction costs calculator for '{self.setup.exchange}' for specification '{self.setup.commissions}' !"
+            )
+
+        channel = SimulatedCtrlChannel("databus", sentinel=(None, None, None, None))
+        simulated_clock = SimulatedTimeProvider(np.datetime64(self.start, "ns"))
+
+        logger.debug(
+            f"[<y>simulator</y>] :: Preparing simulated trading on <g>{self.setup.exchange.upper()}</g> for {self.setup.capital} {self.setup.base_currency}..."
+        )
+
+        account = SimulatedAccountProcessor(
+            account_id=self.account_id,
+            channel=channel,
+            base_currency=self.setup.base_currency,
+            initial_capital=self.setup.capital,
+            time_provider=simulated_clock,
+            tcc=tcc,
+            accurate_stop_orders_execution=self.setup.accurate_stop_orders_execution,
+        )
+        scheduler = SimulatedScheduler(channel, lambda: simulated_clock.time().item())
+        broker = SimulatedBroker(channel, account, self.setup.exchange)
+        data_provider = SimulatedDataProvider(
+            exchange_id=self.setup.exchange,
+            channel=channel,
+            scheduler=scheduler,
+            time_provider=simulated_clock,
+            account=account,
+            readers=self.data_config.data_providers,
+            open_close_time_indent_secs=self.data_config.adjusted_open_close_time_indent_secs,
+        )
+        # - get aux data provider
+        _aux_data = self.data_config.get_timeguarded_aux_reader(simulated_clock)
+        # - it will store simulation results into memory
+        logs_writer = InMemoryLogsWriter(self.account_id, self.setup.name, "0")
+
+        # - it will store simulation results into memory
+        strat: IStrategy | None = None
+
+        match self.setup.setup_type:
+            case SetupTypes.STRATEGY:
+                strat = self.setup.generator  # type: ignore
+
+            case SetupTypes.STRATEGY_AND_TRACKER:
+                strat = self.setup.generator  # type: ignore
+                strat.tracker = lambda ctx: self.setup.tracker  # type: ignore
+
+            case SetupTypes.SIGNAL:
+                strat = SignalsProxy(timeframe=self.setup.signal_timeframe)
+                data_provider.set_generated_signals(self.setup.generator)  # type: ignore
+
+                # - we don't need any unexpected triggerings
+                self._stop = min(self.setup.generator.index[-1], self.stop)  # type: ignore
+
+            case SetupTypes.SIGNAL_AND_TRACKER:
+                strat = SignalsProxy(timeframe=self.setup.signal_timeframe)
+                strat.tracker = lambda ctx: self.setup.tracker
+                data_provider.set_generated_signals(self.setup.generator)  # type: ignore
+
+                # - we don't need any unexpected triggerings
+                self._stop = min(self.setup.generator.index[-1], self.stop)  # type: ignore
+
+            case _:
+                raise SimulationError(f"Unsupported setup type: {self.setup.setup_type} !")
+
+        if not isinstance(strat, IStrategy):
+            raise SimulationConfigError(f"Strategy should be an instance of IStrategy, but got {strat} !")
+
+        ctx = StrategyContext(
+            strategy=strat,
+            broker=broker,
+            data_provider=data_provider,
+            account=account,
+            scheduler=scheduler,
+            time_provider=simulated_clock,
+            instruments=self.setup.instruments,
+            logging=StrategyLogging(logs_writer, portfolio_log_freq=self.portfolio_log_freq),
+            aux_data_provider=_aux_data,
+        )
+
+        # - setup base subscription from spec
+        if ctx.get_base_subscription() == DataType.NONE:
+            logger.debug(
+                f"[<y>simulator</y>] :: Setting up default base subscription: {self.data_config.default_base_subscription}"
+            )
+            ctx.set_base_subscription(self.data_config.default_base_subscription)
+
+        # - set default on_event schedule if detected and strategy didn't set it's own schedule
+        if not ctx.get_event_schedule("time") and self.data_config.default_trigger_schedule:
+            logger.debug(f"[<y>simulator</y>] :: Setting default schedule: {self.data_config.default_trigger_schedule}")
+            ctx.set_event_schedule(self.data_config.default_trigger_schedule)
+
+        self.data_provider = data_provider
+        self.logs_writer = logs_writer
+        return ctx

qubx/backtester/simulated_data.py

@@ -9,6 +9,7 @@ from qubx.core.basics import DataType, Instrument, Timestamped
 from qubx.core.exceptions import SimulationError
 from qubx.data.readers import (
     AsDict,
+    AsOrderBook,
     AsQuotes,
     AsTrades,
     DataReader,
@@ -230,10 +231,16 @@ class DataFetcher:
                 self._transformer = AsTrades()
 
             case DataType.QUOTE:
-                self._requested_data_type = "orderbook"
+                # self._requested_data_type = "orderbook"
+                self._requested_data_type = "quote"
                 self._producing_data_type = "quote"  # ???
                 self._transformer = AsQuotes()
 
+            case DataType.ORDERBOOK:
+                self._requested_data_type = "orderbook"
+                self._producing_data_type = "orderbook"
+                self._transformer = AsOrderBook()
+
             case _:
                 self._requested_data_type = subtype
                 self._producing_data_type = subtype
@@ -311,6 +318,12 @@ class DataFetcher:
                 if self._timeframe:
                     _args["timeframe"] = self._timeframe
 
+                # get arguments from self._reader.read
+                _reader_args = set(self._reader.read.__code__.co_varnames[: self._reader.read.__code__.co_argcount])
+                # match _args with self._params
+                _params = {k: v for k, v in self._params.items() if k in _reader_args}
+                _args = {**_args, **_params}
+
                 try:
                     _r_iters[self._fetcher_id + "." + _r] = self._reader.read(**_args)  # type: ignore
                 except Exception as e:
@@ -382,7 +395,7 @@ class IterableSimulationData(Iterator):
             case DataType.OHLC | DataType.OHLC_QUOTES:
                 _timeframe = _params.get("timeframe", "1Min")
                 _access_key = f"{_subtype}.{_timeframe}"
-            case DataType.TRADE | DataType.QUOTE:
+            case DataType.TRADE | DataType.QUOTE | DataType.ORDERBOOK:
                 _access_key = f"{_subtype}"
             case _:
                 # - any arbitrary data type is passed as is