Qubx 0.2.2__cp311-cp311-manylinux_2_35_x86_64.whl → 0.2.5__cp311-cp311-manylinux_2_35_x86_64.whl

qubx/backtester/ome.py

@@ -90,6 +90,7 @@ class OrdersManagementEngine:
         price: float | None = None,
         client_id: str | None = None,
         time_in_force: str = "gtc",
+        fill_at_price: bool = False,
     ) -> OmeReport:
 
         if self.bbo is None:
@@ -114,12 +115,12 @@ class OrdersManagementEngine:
             client_id,
         )
 
-        return self._process_order(timestamp, order)
+        return self._process_order(timestamp, order, fill_at_price=fill_at_price)
 
     def _dbg(self, message, **kwargs) -> None:
         logger.debug(f"[OMS] {self.instrument.symbol} - {message}", **kwargs)
 
-    def _process_order(self, timestamp: dt_64, order: Order) -> OmeReport:
+    def _process_order(self, timestamp: dt_64, order: Order, fill_at_price: bool = False) -> OmeReport:
         if order.status in ["CLOSED", "CANCELED"]:
             raise InvalidOrder(f"Order {order.id} is already closed or canceled.")
 
@@ -129,7 +130,10 @@ class OrdersManagementEngine:
 
         # - check if order can be "executed" immediately
         exec_price = None
-        if order.type == "MARKET":
+        if fill_at_price and order.price:
+            exec_price = order.price
+
+        elif order.type == "MARKET":
             exec_price = c_ask if buy_side else c_bid
 
         elif order.type == "LIMIT":

qubx/backtester/optimization.py

@@ -1,4 +1,4 @@
-from typing import Any, Dict, List, Sequence, Tuple
+from typing import Any, Dict, List, Sequence, Tuple, Type
 import numpy as np
 import re
 
@@ -71,7 +71,7 @@ def permutate_params(
             case str():
                 vals.append([v])
             case _:
-                vals.append(list(v))
+                vals.append([v])
         # vals.append(v if isinstance(v, (List, Tuple)) else list(v) if isinstance(v, range) else [v])
     d = [dict(zip(args, p)) for p in product(*vals)]
     result = []
@@ -90,7 +90,7 @@ def permutate_params(
     return _wrap_single_list(result) if wrap_as_list else result
 
 
-def variate(clz, *args, conditions=None, **kwargs) -> Dict[str, Any]:
+def variate(clz: Type[Any] | List[Type[Any]], *args, conditions=None, **kwargs) -> Dict[str, Any]:
     """
     Make variations of parameters for simulations (micro optimizer)
 
@@ -127,15 +127,29 @@ def variate(clz, *args, conditions=None, **kwargs) -> Dict[str, Any]:
     >>>             variate(MomentumStrategy_Ex1_test, 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False]),
     >>>             data, capital, ["BINANCE.UM:BTCUSDT"], dict(type="ohlc", timeframe="5Min", nback=0), "5Min -1Sec", "vip0_usdt", "2024-01-01", "2024-01-02"
     >>>    )
+
+    Also it's possible to pass a class with tracker:
+    >>>    variate([MomentumStrategy_Ex1_test, AtrTracker(2, 1)], 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False])
     """
 
     def _cmprss(xs: str):
         return "".join([x[0] for x in re.split("((?<!-)(?=[A-Z]))|_|(\d)", xs) if x])
 
-    sfx = _cmprss(clz.__name__)
+    if isinstance(clz, type):
+        sfx = _cmprss(clz.__name__)
+        _mk = lambda k, *args, **kwargs: k(*args, **kwargs)
+    elif isinstance(clz, (list, tuple)) and clz and isinstance(clz[0], type):
+        sfx = _cmprss(clz[0].__name__)
+        _mk = lambda k, *args, **kwargs: [k[0](*args, **kwargs), *k[1:]]
+    else:
+        raise ValueError(
+            "Can't recognize data for variating: must be either a class type or a list where first element is class type"
+        )
+
     to_excl = [s for s, v in kwargs.items() if not isinstance(v, (list, set, tuple, range))]
     dic2str = lambda ds: [_cmprss(k) + "=" + str(v) for k, v in ds.items() if k not in to_excl]
 
     return {
-        f"{sfx}_({ ','.join(dic2str(z)) })": clz(*args, **z) for z in permutate_params(kwargs, conditions=conditions)
+        f"{sfx}_({ ','.join(dic2str(z)) })": _mk(clz, *args, **z)
+        for z in permutate_params(kwargs, conditions=conditions)
     }

qubx/backtester/queue.py

@@ -4,11 +4,13 @@ import heapq
 from dataclasses import dataclass
 from collections import defaultdict
 from typing import Any, Iterator, Iterable
+from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, Future
 
 from qubx import logger
 from qubx.core.basics import Instrument, dt_64, BatchEvent
 from qubx.data.readers import DataReader, DataTransformer
 from qubx.utils.misc import Stopwatch
+from qubx.core.exceptions import SimulatorError
 
 
 _SW = Stopwatch()
@@ -133,17 +135,141 @@ class SimulatedDataQueue:
         return self
 
     def __iter__(self) -> Iterator:
-        logger.info("Initializing chunks for each loader")
+        logger.debug("Initializing chunks for each loader")
+        assert self._start is not None
+        self._current_time = int(pd.Timestamp(self._start).timestamp() * 1e9)
+        self._index_to_chunk_size = {}
+        self._index_to_iterator = {}
+        self._event_heap = []
+        for loader_index in self._index_to_loader.keys():
+            self._add_chunk_to_heap(loader_index)
+        return self
+
+    @_SW.watch("DataQueue")
+    def __next__(self) -> tuple[str, str, Any]:
+        if not self._event_heap:
+            raise StopIteration
+
+        loader_index = None
+
+        # get the next event from the heap
+        # if the loader_index is in the removed_loader_indices, skip it (optimization to avoid unnecessary heap operations)
+        while self._event_heap and (loader_index is None or loader_index in self._removed_loader_indices):
+            dt, loader_index, chunk_index, event = heapq.heappop(self._event_heap)
+
+        if loader_index is None or loader_index in self._removed_loader_indices:
+            raise StopIteration
+
+        loader = self._index_to_loader[loader_index]
+        data_type = loader.data_type
+        if dt < self._current_time:  # type: ignore
+            data_type = f"hist_{data_type}"
+        else:
+            # only update the current time if the event is not historical
+            self._current_time = dt
+
+        chunk_size = self._index_to_chunk_size[loader_index]
+        if chunk_index + 1 == chunk_size:
+            self._add_chunk_to_heap(loader_index)
+
+        return loader.symbol, data_type, event
+
+    @_SW.watch("DataQueue")
+    def _add_chunk_to_heap(self, loader_index: int):
+        chunk = self._next_chunk(loader_index)
+        self._index_to_chunk_size[loader_index] = len(chunk)
+        for chunk_index, event in enumerate(chunk):
+            dt = event.time  # type: ignore
+            heapq.heappush(self._event_heap, (dt, loader_index, chunk_index, event))
+
+    @_SW.watch("DataQueue")
+    def _next_chunk(self, index: int) -> list[Any]:
+        if index not in self._index_to_iterator:
+            self._index_to_iterator[index] = self._index_to_loader[index].load(pd.Timestamp(self._current_time, unit="ns"), self._stop)  # type: ignore
+        iterator = self._index_to_iterator[index]
+        try:
+            return next(iterator)
+        except StopIteration:
+            return []
+
+
+class SimulatedDataQueueWithThreads(SimulatedDataQueue):
+    _loaders: dict[str, list[DataLoader]]
+
+    def __init__(self, workers: int = 4, prefetch_chunk_count: int = 1):
+        self._loaders = defaultdict(list)
+        self._start = None
+        self._stop = None
+        self._current_time = None
+        self._index_to_loader: dict[int, DataLoader] = {}
+        self._index_to_prefetch: dict[int, list[Future]] = defaultdict(list)
+        self._index_to_done: dict[int, bool] = defaultdict(bool)
+        self._loader_to_index = {}
+        self._index_to_chunk_size = {}
+        self._index_to_iterator = {}
+        self._latest_loader_index = -1
+        self._removed_loader_indices = set()
+        # TODO: potentially use ProcessPoolExecutor for better performance
+        self._pool = ThreadPoolExecutor(max_workers=workers)
+        self._prefetch_chunk_count = prefetch_chunk_count
+
+    @property
+    def is_running(self) -> bool:
+        return self._current_time is not None
+
+    def __add__(self, loader: DataLoader) -> "SimulatedDataQueueWithThreads":
+        self._latest_loader_index += 1
+        new_loader_index = self._latest_loader_index
+        self._loaders[loader.symbol].append(loader)
+        self._index_to_loader[new_loader_index] = loader
+        self._loader_to_index[loader] = new_loader_index
+        if self.is_running:
+            self._submit_chunk(new_loader_index)
+            self._add_chunk_to_heap(new_loader_index)
+        return self
+
+    def __sub__(self, loader: DataLoader) -> "SimulatedDataQueueWithThreads":
+        loader_index = self._loader_to_index[loader]
+        self._loaders[loader.symbol].remove(loader)
+        del self._index_to_loader[loader_index]
+        del self._loader_to_index[loader]
+        del self._index_to_chunk_size[loader_index]
+        del self._index_to_iterator[loader_index]
+        del self._index_to_done[loader_index]
+        for future in self._index_to_prefetch[loader_index]:
+            future.cancel()
+        del self._index_to_prefetch[loader_index]
+        self._removed_loader_indices.add(loader_index)
+        return self
+
+    def get_loader(self, symbol: str, data_type: str) -> DataLoader:
+        loaders = self._loaders[symbol]
+        for loader in loaders:
+            if loader.data_type == data_type:
+                return loader
+        raise ValueError(f"Loader for {symbol} and {data_type} not found")
+
+    def create_iterable(self, start: str | pd.Timestamp, stop: str | pd.Timestamp) -> Iterator:
+        self._start = start
+        self._stop = stop
+        self._current_time = None
+        return self
+
+    def __iter__(self) -> Iterator:
+        logger.debug("Initializing chunks for each loader")
         self._current_time = self._start
         self._index_to_chunk_size = {}
         self._index_to_iterator = {}
         self._event_heap = []
+        self._submit_chunk_prefetchers()
         for loader_index in self._index_to_loader.keys():
             self._add_chunk_to_heap(loader_index)
         return self
 
     @_SW.watch("DataQueue")
     def __next__(self) -> tuple[str, str, Any]:
+        self._submit_chunk_prefetchers()
+
         if not self._event_heap:
             raise StopIteration
 
@@ -167,13 +293,21 @@ class SimulatedDataQueue:
 
     @_SW.watch("DataQueue")
     def _add_chunk_to_heap(self, loader_index: int):
-        chunk = self._next_chunk(loader_index)
+        futures = self._index_to_prefetch[loader_index]
+        if not futures and not self._index_to_done[loader_index]:
+            loader = self._index_to_loader[loader_index]
+            logger.error(f"Error state: No submitted tasks for loader {loader.symbol} {loader.data_type}")
+            raise SimulatorError("No submitted tasks for loader")
+        elif self._index_to_done[loader_index]:
+            return
+
+        # wait for future to finish if needed
+        chunk = futures.pop(0).result()
         self._index_to_chunk_size[loader_index] = len(chunk)
         for chunk_index, event in enumerate(chunk):
             dt = event.time  # type: ignore
             heapq.heappush(self._event_heap, (dt, loader_index, chunk_index, event))
 
-    @_SW.watch("DataQueue")
     def _next_chunk(self, index: int) -> list[Any]:
         if index not in self._index_to_iterator:
             self._index_to_iterator[index] = self._index_to_loader[index].load(self._current_time, self._stop)  # type: ignore
@@ -183,6 +317,18 @@ class SimulatedDataQueue:
         except StopIteration:
             return []
 
+    def _submit_chunk_prefetchers(self):
+        for index in self._index_to_loader.keys():
+            if len(self._index_to_prefetch[index]) < self._prefetch_chunk_count:
+                self._submit_chunk(index)
+
+    def _submit_chunk(self, loader_index: int) -> None:
+        future = self._pool.submit(self._next_chunk, loader_index)
+        self._index_to_prefetch[loader_index].append(future)
+
+    def __del__(self):
+        self._pool.shutdown()
+
 
 class EventBatcher:
     _BATCH_SETTINGS = {
@@ -202,7 +348,8 @@ class EventBatcher:
             yield from _iter
             return
 
-        last_symbol, last_data_type = None, None
+        last_symbol: str = None  # type: ignore
+        last_data_type: str = None  # type: ignore
         buffer = []
         for symbol, data_type, event in self.source_iterator:
             time: dt_64 = event.time  # type: ignore
@@ -233,7 +380,7 @@ class EventBatcher:
             if pd.Timedelta(time - buffer[0].time) >= self._batch_settings[data_type]:
                 yield symbol, data_type, self._batch_event(buffer)
                 buffer = []
-                last_symbol, last_data_type = None, None
+                last_symbol, last_data_type = None, None  # type: ignore
 
         if buffer:
             yield last_symbol, last_data_type, self._batch_event(buffer)

qubx/backtester/simulator.py

@@ -187,13 +187,22 @@ class SimulatedTrading(ITradingServiceProvider):
         price: float | None = None,
         client_id: str | None = None,
         time_in_force: str = "gtc",
+        **optional,
     ) -> Order:
         ome = self._ome.get(instrument.symbol)
         if ome is None:
             raise ValueError(f"ExchangeService:send_order :: No OME configured for '{instrument.symbol}'!")
 
         # - try to place order in OME
-        report = ome.place_order(order_side.upper(), order_type.upper(), amount, price, client_id, time_in_force)
+        report = ome.place_order(
+            order_side.upper(),
+            order_type.upper(),
+            amount,
+            price,
+            client_id,
+            time_in_force,
+            fill_at_price=optional.get("fill_at_price", False),
+        )
         order = report.order
         self._order_to_symbol[order.id] = instrument.symbol
 
@@ -476,6 +485,9 @@ class SimulatedExchange(IBrokerServiceProvider):
         logger.info(f"SimulatedExchangeService :: run :: Simulation finished at {end}")
 
     def _run_generated_signals(self, symbol: str, data_type: str, data: Any) -> None:
+        is_hist = data_type.startswith("hist")
+        if is_hist:
+            raise ValueError("Historical data is not supported for pre-generated signals !")
         cc = self.get_communication_channel()
         t = data.time  # type: ignore
         self._current_time = max(np.datetime64(t, "ns"), self._current_time)
@@ -496,18 +508,20 @@ class SimulatedExchange(IBrokerServiceProvider):
         t = data.time  # type: ignore
         self._current_time = max(np.datetime64(t, "ns"), self._current_time)
         q = self.trading_service.emulate_quote_from_data(symbol, np.datetime64(t, "ns"), data)
-        if q is not None:
+        is_hist = data_type.startswith("hist")
+        if not is_hist and q is not None:
             self._last_quotes[symbol] = q
             self.trading_service.update_position_price(symbol, self._current_time, q)
 
         cc.send((symbol, data_type, data))
 
-        if q is not None:
-            cc.send((symbol, "quote", q))
+        if not is_hist:
+            if q is not None and data_type != "quote":
+                cc.send((symbol, "quote", q))
 
-        if self._scheduler.check_and_run_tasks():
-            # - push nothing - it will force to process last event
-            cc.send((None, "time", None))
+            if self._scheduler.check_and_run_tasks():
+                # - push nothing - it will force to process last event
+                cc.send((None, "time", None))
 
     def get_quote(self, symbol: str) -> Optional[Quote]:
         return self._last_quotes[symbol]
@@ -664,13 +678,14 @@ def simulate(
     commissions: str,
     start: str | pd.Timestamp,
     stop: str | pd.Timestamp | None = None,
+    fit: str | None = None,
     exchange: str | None = None,  # in case if exchange is not specified in symbols list
     base_currency: str = "USDT",
     leverage: float = 1.0,  # TODO: we need to add support for leverage
     n_jobs: int = 1,
     silent: bool = False,
     enable_event_batching: bool = True,
-) -> TradingSessionResult | List[TradingSessionResult]:
+) -> list[TradingSessionResult]:
     # - recognize provided data
     if isinstance(data, dict):
         data_reader = InMemoryDataFrameReader(data)
@@ -723,6 +738,7 @@ def simulate(
         data_reader,
         subscription,
         trigger,
+        fit=fit,
         n_jobs=n_jobs,
         silent=silent,
         enable_event_batching=enable_event_batching,
@@ -785,6 +801,7 @@ def _run_setups(
     data_reader: DataReader,
     subscription: Dict[str, Any],
     trigger: str | list[str],
+    fit: str | None,
     n_jobs: int = -1,
     silent: bool = False,
     enable_event_batching: bool = True,
@@ -799,27 +816,31 @@ def _run_setups(
 
     reports = ProgressParallel(n_jobs=n_jobs, total=len(setups), silent=_main_loop_silent, backend="multiprocessing")(
         delayed(_run_setup)(
+            id,
             s,
             start,
             stop,
             data_reader,
             subscription,
             trigger,
+            fit=fit,
             silent=silent,
             enable_event_batching=enable_event_batching,
         )
-        for s in setups
+        for id, s in enumerate(setups)
     )
     return reports  # type: ignore
 
 
 def _run_setup(
+    setup_id: int,
     setup: SimulationSetup,
     start: str | pd.Timestamp,
     stop: str | pd.Timestamp,
     data_reader: DataReader,
     subscription: Dict[str, Any],
     trigger: str | list[str],
+    fit: str | None,
     silent: bool = False,
     enable_event_batching: bool = True,
 ) -> TradingSessionResult:
@@ -870,6 +891,7 @@ def _run_setup(
         instruments=setup.instruments,
         md_subscription=subscription,
         trigger_spec=_trigger,
+        fit_spec=fit,
         logs_writer=logs_writer,
     )
     ctx.start()
@@ -880,6 +902,7 @@ def _run_setup(
         logger.error("Simulated trading interrupted by user !")
 
     return TradingSessionResult(
+        setup_id,
         setup.name,
         start,
         stop,

qubx/core/basics.py

@@ -19,6 +19,13 @@ td_64 = np.timedelta64
 class Signal:
     """
     Class for presenting signals generated by strategy
+
+    Attributes:
+        reference_price: float - exact price when signal was generated
+
+        Options:
+        - fill_at_signal_price: bool - if True, then fill order at signal price (only used in backtesting)
+        - allow_override: bool - if True, and there is another signal for the same instrument, then override current.
     """
 
     instrument: "Instrument"
@@ -26,15 +33,20 @@ class Signal:
     price: float | None = None
     stop: float | None = None
     take: float | None = None
+    reference_price: float | None = None
     group: str = ""
     comment: str = ""
+    options: dict[str, Any] = field(default_factory=dict)
 
     def __str__(self) -> str:
         _p = f" @ { self.price }" if self.price is not None else ""
         _s = f" stop: { self.stop }" if self.stop is not None else ""
         _t = f" take: { self.take }" if self.take is not None else ""
+        _r = f" {self.reference_price:.2f}" if self.reference_price is not None else ""
         _c = f" [{self.comment}]" if self.take is not None else ""
-        return f"{self.group} {self.signal:+f} {self.instrument.symbol}{_p}{_s}{_t} on {self.instrument.exchange}{_c}"
+        return (
+            f"{self.group}{_r} {self.signal:+f} {self.instrument.symbol}{_p}{_s}{_t} on {self.instrument.exchange}{_c}"
+        )
 
 
 @dataclass
@@ -43,9 +55,18 @@ class TargetPosition:
     Class for presenting target position calculated from signal
     """
 
+    time: dt_64  # time when position was set
     signal: Signal  # original signal
     target_position_size: float  # actual position size after processing in sizer
 
+    @staticmethod
+    def create(ctx: "ITimeProvider", signal: Signal, target_size: float) -> "TargetPosition":
+        return TargetPosition(ctx.time(), signal, target_size)
+
+    @staticmethod
+    def zero(ctx: "ITimeProvider", signal: Signal) -> "TargetPosition":
+        return TargetPosition(ctx.time(), signal, 0.0)
+
     @property
     def instrument(self) -> "Instrument":
         return self.signal.instrument
@@ -63,7 +84,7 @@ class TargetPosition:
         return self.signal.take
 
     def __str__(self) -> str:
-        return f"Target for {self.signal} -> {self.target_position_size}"
+        return f"Target for {self.signal} -> {self.target_position_size} at {self.time}"
 
 
 @dataclass
@@ -125,14 +146,26 @@ class Instrument:
         take: float | None = None,
         group: str = "",
         comment: str = "",
+        options: dict[str, Any] = None,
     ) -> Signal:
-        return Signal(self, signal, price, stop, take, group, comment)
+        return Signal(
+            self,
+            signal=signal,
+            price=price,
+            stop=stop,
+            take=take,
+            group=group,
+            comment=comment,
+            options=options or {},
+        )
 
     def __hash__(self) -> int:
         return hash((self.symbol, self.exchange, self.market_type))
 
     def __eq__(self, other: Any) -> bool:
-        if not isinstance(other, "Instrument"):
+        if other is None:
+            return False
+        if type(other) != type(self):
             return False
         return self.symbol == other.symbol and self.exchange == other.exchange and self.market_type == other.market_type
 
@@ -502,7 +535,8 @@ class ITimeProvider:
 
 
 class TradingSessionResult:
-    trading_id: str
+    id: int
+    name: str
     start: str | pd.Timestamp
     stop: str | pd.Timestamp
     exchange: str
@@ -518,7 +552,8 @@ class TradingSessionResult:
 
     def __init__(
         self,
-        trading_id: str,
+        id: int,
+        name: str,
         start: str | pd.Timestamp,
         stop: str | pd.Timestamp,
         exchange: str,
@@ -532,7 +567,8 @@ class TradingSessionResult:
         signals_log: pd.DataFrame,
         is_simulation=True,
     ):
-        self.trading_id = trading_id
+        self.id = id
+        self.name = name
         self.start = start
         self.stop = stop
         self.exchange = exchange