Qubx 0.6.3__cp312-cp312-manylinux_2_39_x86_64.whl → 0.6.6__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/account.py

@@ -154,7 +154,9 @@ class SimulatedAccountProcessor(BasicAccountProcessor):
             return
         for r in ome.process_market_data(data):
             if r.exec is not None:
-                self.order_to_instrument.pop(r.order.id)
+                # TODO: why do we pop here if it is popped later after send
+                if r.order.id in self.order_to_instrument:
+                    self.order_to_instrument.pop(r.order.id)
                 # - process methods will be called from stg context
                 self._channel.send((instrument, "order", r.order, False))
                 self._channel.send((instrument, "deals", [r.exec], False))

qubx/backtester/ome.py

@@ -7,6 +7,8 @@ from sortedcontainers import SortedDict
 from qubx import logger
 from qubx.core.basics import (
     OPTION_FILL_AT_SIGNAL_PRICE,
+    OPTION_SIGNAL_PRICE,
+    OPTION_SKIP_PRICE_CROSS_CONTROL,
     Deal,
     Instrument,
     ITimeProvider,
@@ -194,54 +196,67 @@ class OrdersManagementEngine:
         if order.status in ["CLOSED", "CANCELED"]:
             raise InvalidOrder(f"Order {order.id} is already closed or canceled.")
 
-        buy_side = order.side == "BUY"
-        c_ask = self.bbo.ask
-        c_bid = self.bbo.bid
+        _buy_side = order.side == "BUY"
+        _c_ask = self.bbo.ask  # type: ignore
+        _c_bid = self.bbo.bid  # type: ignore
 
         # - check if order can be "executed" immediately
-        exec_price = None
+        _exec_price = None
         _need_update_book = False
 
-        if order.type == "MARKET":
-            if exec_price is None:
-                exec_price = c_ask if buy_side else c_bid
-
-            # - special case - fill at signal price for market order
-            # - only for simulation
-            # - only if this is valid price: market crossed this desired price on last update
-            if order.options.get(OPTION_FILL_AT_SIGNAL_PRICE, False) and order.price and self.__prev_bbo:
-                _desired_fill_price = order.price
-
-                if (buy_side and self.__prev_bbo.ask < _desired_fill_price <= c_ask) or (
-                    not buy_side and self.__prev_bbo.bid > _desired_fill_price >= c_bid
-                ):
-                    exec_price = _desired_fill_price
-                else:
-                    raise SimulationError(
-                        f"Special execution price at {_desired_fill_price} for market order {order.id} cannot be filled because market didn't cross this price on last update !"
-                    )
-
-        elif order.type == "LIMIT":
-            _need_update_book = True
-            if (buy_side and order.price >= c_ask) or (not buy_side and order.price <= c_bid):
-                exec_price = c_ask if buy_side else c_bid
-
-        elif order.type == "STOP_MARKET":
-            # - it processes stop orders separately without adding to orderbook (as on real exchanges)
-            order.status = "OPEN"
-            self.stop_orders[order.id] = order
-
-        elif order.type == "STOP_LIMIT":
-            # TODO: (OME) check trigger conditions in options etc
-            raise NotImplementedError("'STOP_LIMIT' order is not supported in Qubx simulator yet !")
+        match order.type:
+            case "MARKET":
+                if _exec_price is None:
+                    _exec_price = _c_ask if _buy_side else _c_bid
+
+                # - special case only for simulation: exact fill at signal price for market orders
+                _fill_at_signal_price = order.options.get(OPTION_FILL_AT_SIGNAL_PRICE, False)
+                _signal_price = order.options.get(OPTION_SIGNAL_PRICE, None)
+
+                # - some cases require to skip price cross control
+                _skip_price_cross_control = order.options.get(OPTION_SKIP_PRICE_CROSS_CONTROL, False)
+
+                # - it's passed only if signal price is valid: market crossed this desired price on last update
+                if _fill_at_signal_price and _signal_price and self.__prev_bbo:
+                    _desired_fill_price = _signal_price
+                    _prev_mp = self.__prev_bbo.mid_price()
+                    _c_mid_price = self.bbo.mid_price()  # type: ignore
+
+                    if (
+                        _skip_price_cross_control
+                        or (_prev_mp < _desired_fill_price <= _c_mid_price)
+                        or (_prev_mp > _desired_fill_price >= _c_mid_price)
+                    ):
+                        _exec_price = _desired_fill_price
+                    else:
+                        raise SimulationError(
+                            f"Special execution price at {_desired_fill_price} for market order {order.id} cannot be filled because market didn't cross this price on last update !"
+                        )
+
+            case "LIMIT":
+                _need_update_book = True
+                if (_buy_side and order.price >= _c_ask) or (not _buy_side and order.price <= _c_bid):
+                    _exec_price = _c_ask if _buy_side else _c_bid
+
+            case "STOP_MARKET":
+                # - it processes stop orders separately without adding to orderbook (as on real exchanges)
+                order.status = "OPEN"
+                self.stop_orders[order.id] = order
+
+            case "STOP_LIMIT":
+                # TODO: (OME) check trigger conditions in options etc
+                raise NotImplementedError("'STOP_LIMIT' order is not supported in Qubx simulator yet !")
+
+            case _:
+                raise SimulationError(f"Invalid order type: {order.type} for {self.instrument.symbol}")
 
         # - if order must be "executed" immediately
-        if exec_price is not None:
-            return self._execute_order(timestamp, exec_price, order, True)
+        if _exec_price is not None:
+            return self._execute_order(timestamp, _exec_price, order, True)
 
         # - processing limit orders
         if _need_update_book:
-            if buy_side:
+            if _buy_side:
                 self.bids.setdefault(order.price, list()).append(order.id)
             else:
                 self.asks.setdefault(order.price, list()).append(order.id)
@@ -288,8 +303,8 @@ class OrdersManagementEngine:
         if (_ot == "LIMIT" or _ot.startswith("STOP")) and (price is None or price <= 0):
             raise InvalidOrder("Invalid order price. Price must be positively defined for LIMIT or STOP orders.")
 
-        if time_in_force.upper() not in ["GTC", "IOC"]:
-            raise InvalidOrder("Invalid time in force. Only GTC or IOC is supported for now.")
+        if time_in_force.upper() not in ["GTC", "IOC", "GTX"]:
+            raise InvalidOrder("Invalid time in force. Only GTC, IOC, GTX are supported for now.")
 
         if _ot.startswith("STOP"):
             assert price is not None

qubx/backtester/runner.py

@@ -8,7 +8,7 @@ from qubx.core.basics import SW, DataType
 from qubx.core.context import StrategyContext
 from qubx.core.exceptions import SimulationConfigError, SimulationError
 from qubx.core.helpers import extract_parameters_from_object, full_qualified_class_name
-from qubx.core.interfaces import IStrategy, IStrategyContext
+from qubx.core.interfaces import IMetricEmitter, IStrategy, IStrategyContext
 from qubx.core.loggers import InMemoryLogsWriter, StrategyLogging
 from qubx.core.lookups import lookup
 from qubx.pandaz.utils import _frame_to_str
@@ -57,6 +57,7 @@ class SimulationRunner:
         stop: pd.Timestamp | str,
         account_id: str = "SimulatedAccount",
         portfolio_log_freq: str = "5Min",
+        emitter: IMetricEmitter | None = None,
     ):
         """
         Initialize the BacktestContextRunner with a strategy context.
@@ -68,6 +69,7 @@ class SimulationRunner:
             stop (pd.Timestamp): The end time of the simulation.
             account_id (str): The account id to use.
             portfolio_log_freq (str): The portfolio log frequency to use.
+            emitter (IMetricEmitter): The emitter to use.
         """
         self.setup = setup
         self.data_config = data_config
@@ -75,6 +77,7 @@ class SimulationRunner:
         self.stop = pd.Timestamp(stop)
         self.account_id = account_id
         self.portfolio_log_freq = portfolio_log_freq
+        self.emitter = emitter
         self.ctx = self._create_backtest_context()
 
         # - get strategy parameters BEFORE simulation start
@@ -85,7 +88,7 @@ class SimulationRunner:
             self.strategy_params = extract_parameters_from_object(self.setup.generator)
             self.strategy_class = full_qualified_class_name(self.setup.generator)
 
-    def run(self, silent: bool = False):
+    def run(self, silent: bool = False, catch_keyboard_interrupt: bool = True, close_data_readers: bool = False):
         """
         Run the backtest from start to stop.
 
@@ -94,7 +97,7 @@ class SimulationRunner:
             stop (pd.Timestamp | str): The end time of the simulation.
             silent (bool, optional): Whether to suppress progress output. Defaults to False.
         """
-        logger.debug(f"[<y>BacktestContextRunner</y>] :: Running simulation from {self.start} to {self.stop}")
+        logger.debug(f"[<y>SimulationRunner</y>] :: Running simulation from {self.start} to {self.stop}")
 
         # Start the context
         self.ctx.start()
@@ -103,7 +106,7 @@ class SimulationRunner:
         for s in self.ctx.get_subscriptions():
             if not self.ctx.get_warmup(s) and (_d_wt := self.data_config.default_warmups.get(s)):
                 logger.debug(
-                    f"[<y>BacktestContextRunner</y>] :: Strategy didn't set warmup period for <c>{s}</c> so default <c>{_d_wt}</c> will be used"
+                    f"[<y>SimulationRunner</y>] :: Strategy didn't set warmup period for <c>{s}</c> so default <c>{_d_wt}</c> will be used"
                 )
                 self.ctx.set_warmup({s: _d_wt})
 
@@ -129,13 +132,19 @@ class SimulationRunner:
         stop = self._stop or self.stop
 
         try:
-            # Run the data provider
             self.data_provider.run(self.start, stop, silent=silent)
         except KeyboardInterrupt:
             logger.error("Simulated trading interrupted by user!")
+            if not catch_keyboard_interrupt:
+                raise
         finally:
             # Stop the context
             self.ctx.stop()
+            if close_data_readers:
+                assert isinstance(self.data_provider, SimulatedDataProvider)
+                for reader in self.data_provider._readers.values():
+                    if hasattr(reader, "close"):
+                        reader.close()  # type: ignore
 
     def print_latency_report(self) -> None:
         _l_r = SW.latency_report()
@@ -229,6 +238,7 @@ class SimulationRunner:
             instruments=self.setup.instruments,
             logging=StrategyLogging(logs_writer, portfolio_log_freq=self.portfolio_log_freq),
             aux_data_provider=_aux_data,
+            emitter=self.emitter,
         )
 
         # - setup base subscription from spec

qubx/backtester/simulated_data.py

@@ -1,12 +1,11 @@
-import math
-from collections import defaultdict, deque
-from typing import Any, Iterator, TypeAlias
+from typing import Any, Iterator
 
 import pandas as pd
 
 from qubx import logger
 from qubx.core.basics import DataType, Instrument, Timestamped
 from qubx.core.exceptions import SimulationError
+from qubx.data.composite import IteratedDataStreamsSlicer
 from qubx.data.readers import (
     AsDict,
     AsOrderBook,
@@ -19,159 +18,6 @@ from qubx.data.readers import (
     RestoreTradesFromOHLC,
 )
 
-SlicerOutData: TypeAlias = tuple[str, int, Timestamped] | tuple
-
-
-class IteratedDataStreamsSlicer(Iterator[SlicerOutData]):
-    """
-    This class manages seamless iteration over multiple time-series data streams,
-    ensuring that events are processed in the correct chronological order regardless of their source.
-    It supports adding / removing new data streams to the slicer on the fly (during the itration).
-    """
-
-    _iterators: dict[str, Iterator[list[Timestamped]]]
-    _buffers: dict[str, list[Timestamped]]
-    _keys: deque[str]
-    _iterating: bool
-
-    def __init__(self):
-        self._buffers = defaultdict(list)
-        self._iterators = {}
-        self._keys = deque()
-        self._iterating = False
-
-    def put(self, data: dict[str, Iterator[list[Timestamped]]]):
-        _rebuild = False
-        for k, vi in data.items():
-            if k not in self._keys:
-                self._iterators[k] = vi
-                self._buffers[k] = self._load_next_chunk_to_buffer(k)  # do initial chunk fetching
-                self._keys.append(k)
-                _rebuild = True
-
-        # - rebuild strategy
-        if _rebuild and self._iterating:
-            self._build_initial_iteration_seq()
-
-    def __add__(self, data: dict[str, Iterator]) -> "IteratedDataStreamsSlicer":
-        self.put(data)
-        return self
-
-    def remove(self, keys: list[str] | str):
-        """
-        Remove data iterator and associated keys from the queue.
-        If the key is not found, it does nothing.
-        """
-        _keys = keys if isinstance(keys, list) else [keys]
-        _rebuild = False
-        for i in _keys:
-            if i in self._buffers:
-                self._buffers.pop(i)
-                self._iterators.pop(i)
-                self._keys.remove(i)
-                _rebuild = True
-
-        # - rebuild strategy
-        if _rebuild and self._iterating:
-            self._build_initial_iteration_seq()
-
-    def __iter__(self) -> Iterator:
-        self._build_initial_iteration_seq()
-        self._iterating = True
-        return self
-
-    def _build_initial_iteration_seq(self):
-        _init_seq = {k: self._buffers[k][-1].time for k in self._keys}
-        _init_seq = dict(sorted(_init_seq.items(), key=lambda item: item[1]))
-        self._keys = deque(_init_seq.keys())
-
-    def _load_next_chunk_to_buffer(self, index: str) -> list[Timestamped]:
-        return list(reversed(next(self._iterators[index])))
-
-    def _remove_iterator(self, key: str):
-        self._buffers.pop(key)
-        self._iterators.pop(key)
-        self._keys.remove(key)
-
-    def _pop_top(self, k: str) -> Timestamped:
-        """
-        Removes and returns the most recent timestamped data element from the buffer associated with the given key.
-        If the buffer is empty after popping, it attempts to load the next chunk of data into the buffer.
-        If no more data is available, the iterator associated with the key is removed.
-
-        Parameters:
-            k (str): The key identifying the data stream buffer to pop from.
-
-        Returns:
-            Timestamped: The most recent timestamped data element from the buffer.
-        """
-        v = (data := self._buffers[k]).pop()
-        if not data:
-            try:
-                data.extend(self._load_next_chunk_to_buffer(k))  # - get next chunk of data
-            except StopIteration:
-                self._remove_iterator(k)  # - remove iterable data
-        return v
-
-    def fetch_before_time(self, key: str, time_ns: int) -> list[Timestamped]:
-        """
-        Fetches and returns all timestamped data elements from the buffer associated with the given key
-        that have a timestamp earlier than the specified time.
-
-        Parameters:
-            - key (str): The key identifying the data stream buffer to fetch from.
-            - time_ns (int): The timestamp in nanoseconds. All returned elements will have a timestamp less than this value.
-
-        Returns:
-            - list[Timestamped]: A list of timestamped data elements that occur before the specified time.
-        """
-        values = []
-        data = self._buffers[key]
-        if not data:
-            try:
-                data.extend(self._load_next_chunk_to_buffer(key))  # - get next chunk of data
-            except StopIteration:
-                self._remove_iterator(key)
-
-        # pull most past elements
-        v = data[-1]
-        while v.time < time_ns:
-            values.append(data.pop())
-            if not data:
-                try:
-                    data.extend(self._load_next_chunk_to_buffer(key))  # - get next chunk of data
-                except StopIteration:
-                    self._remove_iterator(key)
-                    break
-            v = data[-1]
-
-        return values
-
-    def __next__(self) -> SlicerOutData:
-        """
-        Advances the iterator to the next available timestamped data element across all data streams.
-
-        Returns:
-            - SlicerOutData: A tuple containing the key of the data stream, the timestamp of the data element, and the data element itself.
-
-        Raises:
-            - StopIteration: If there are no more data elements to iterate over.
-        """
-        if not self._keys:
-            self._iterating = False
-            raise StopIteration
-
-        _min_t = math.inf
-        _min_k = self._keys[0]
-        for i in self._keys:
-            _x = self._buffers[i][-1]
-            if _x.time < _min_t:
-                _min_t = _x.time
-                _min_k = i
-
-        _v = self._pop_top(_min_k)
-        return (_min_k, _v.time, _v)
-
 
 class DataFetcher:
     _fetcher_id: str

qubx/backtester/utils.py

@@ -23,6 +23,7 @@ from qubx.core.interfaces import IStrategy, IStrategyContext, PositionsTracker
 from qubx.core.lookups import lookup
 from qubx.core.series import OHLCV, Bar, Quote, Trade
 from qubx.core.utils import time_delta_to_str
+from qubx.data import TardisMachineReader
 from qubx.data.helpers import InMemoryCachedReader, TimeGuardedWrapper
 from qubx.data.hft import HftDataReader
 from qubx.data.readers import AsDict, DataReader, InMemoryDataFrameReader
@@ -86,9 +87,9 @@ class SimulationSetup:
     exchange: str
     capital: float
     base_currency: str
-    commissions: str | None
-    signal_timeframe: str
-    accurate_stop_orders_execution: bool
+    commissions: str | None = None
+    signal_timeframe: str = "1Min"
+    accurate_stop_orders_execution: bool = False
 
     def __str__(self) -> str:
         return f"{self.name} {self.setup_type} capital {self.capital} {self.base_currency} for [{','.join(map(lambda x: x.symbol, self.instruments))}] @ {self.exchange}[{self.commissions}]"
@@ -729,6 +730,12 @@ def recognize_simulation_data_config(
 
             _available_symbols = list(set.intersection(*_sets_of_symbols.values()))
 
+        case TardisMachineReader():
+            _supported_types = [DataType.ORDERBOOK, DataType.TRADE]
+            _available_symbols = decls.get_symbols(exchange, _supported_types[0])
+            for _type in _supported_types:
+                _requests[_type] = (_type, decls)
+
         case DataReader():
             _supported_data_type = sniffer._sniff_reader(f"{exchange}:{instruments[0].symbol}", decls, None)
             _available_symbols = decls.get_symbols(exchange, DataType.from_str(_supported_data_type)[0])
@@ -766,6 +773,11 @@ def recognize_simulation_data_config(
                         _available_symbols = _provider.get_symbols(exchange, _supported_data_type)
                         _requests[_requested_type] = (_supported_data_type, _provider)
 
+                    case TardisMachineReader():
+                        _supported_data_type = _requested_type
+                        _available_symbols = _provider.get_symbols(exchange, _supported_data_type)
+                        _requests[_requested_type] = (_supported_data_type, _provider)
+
                     case DataReader():
                         _supported_data_type = sniffer._sniff_reader(
                             f"{exchange}:{instruments[0].symbol}", _provider, _requested_type

qubx/cli/commands.py

@@ -1,8 +1,8 @@
 import os
-import sys
 from pathlib import Path
 
 import click
+from dotenv import load_dotenv
 
 from qubx import QubxLogConfig, logger
 
@@ -32,13 +32,18 @@ def main(debug: bool, debug_port: int, log_level: str):
     """
     Qubx CLI.
     """
+    os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1"
     log_level = log_level.upper() if not debug else "DEBUG"
 
+    env_file = Path.cwd().joinpath(".env")
+    if env_file.exists():
+        logger.info(f"Loading environment variables from {env_file}")
+        load_dotenv(env_file)
+        log_level = os.getenv("QUBX_LOG_LEVEL", log_level)
+
     QubxLogConfig.set_log_level(log_level)
 
     if debug:
-        os.environ["PYDEVD_DISABLE_FILE_VALIDATION"] = "1"
-
         import debugpy
 
         logger.info(f"Waiting for debugger to attach (port {debug_port})")
@@ -146,7 +151,7 @@ def ls(directory: str):
     "-o",
     type=click.STRING,
     help="Output directory to put zip file.",
-    default="releases",
+    default=".releases",
     show_default=True,
 )
 @click.option(
@@ -173,27 +178,6 @@ def ls(directory: str):
     help="Commit changes and create tag in repo (default: False)",
     show_default=True,
 )
-@click.option(
-    "--default-exchange",
-    type=click.STRING,
-    help="Default exchange to use in the generated config.",
-    default="BINANCE.UM",
-    show_default=True,
-)
-@click.option(
-    "--default-connector",
-    type=click.STRING,
-    help="Default connector to use in the generated config.",
-    default="ccxt",
-    show_default=True,
-)
-@click.option(
-    "--default-instruments",
-    type=click.STRING,
-    help="Default instruments to use in the generated config (comma-separated).",
-    default="BTCUSDT",
-    show_default=True,
-)
 def release(
     directory: str,
     strategy: str,
@@ -201,15 +185,12 @@ def release(
     message: str | None,
     commit: bool,
     output_dir: str,
-    default_exchange: str,
-    default_connector: str,
-    default_instruments: str,
 ) -> None:
     """
     Releases the strategy to a zip file.
 
     The strategy can be specified in two ways:
-    1. As a strategy name (class name) - strategies are scanned in the given directory
+    1. As a strategy name (class name) - strategies are scanned in the given directory (NOT SUPPORTED ANYMORE !)
     2. As a path to a config YAML file containing the strategy configuration in StrategyConfig format
 
     If a strategy name is provided, a default configuration will be generated with:
@@ -228,9 +209,6 @@ def release(
     """
     from .release import release_strategy
 
-    # Parse default instruments
-    instruments = [instr.strip() for instr in default_instruments.split(",")]
-
     release_strategy(
         directory=directory,
         strategy_name=strategy,
@@ -238,9 +216,6 @@ def release(
         message=message,
         commit=commit,
         output_dir=output_dir,
-        default_exchange=default_exchange,
-        default_connector=default_connector,
-        default_instruments=instruments,
     )
 
 

qubx/cli/deploy.py

@@ -109,9 +109,7 @@ def ensure_poetry_lock_exists(output_dir: str) -> bool:
     if not os.path.exists(poetry_lock_path):
         logger.warning("poetry.lock not found in the zip file. Attempting to generate it.")
         try:
-            subprocess.run(
-                ["poetry", "lock", "--no-update"], cwd=output_dir, check=True, capture_output=True, text=True
-            )
+            subprocess.run(["poetry", "lock"], cwd=output_dir, check=True, capture_output=True, text=True)
             return True
         except subprocess.CalledProcessError as e:
             logger.error(f"Failed to generate poetry.lock: {e.stderr}")
@@ -163,10 +161,10 @@ def setup_poetry_environment(output_dir: str) -> bool:
                 if var in env:
                     del env[var]
 
-            subprocess.run(install_cmd, cwd=output_dir, check=True, capture_output=True, text=True, env=env)
+            subprocess.run(install_cmd, cwd=output_dir, check=True, capture_output=False, text=True, env=env)
         else:
             # Normal case - not in a Poetry shell
-            subprocess.run(install_cmd, cwd=output_dir, check=True, capture_output=True, text=True)
+            subprocess.run(install_cmd, cwd=output_dir, check=True, capture_output=False, text=True)
 
         # Verify that the virtual environment was created
         venv_path = os.path.join(output_dir, ".venv")