Qubx 0.3.0__cp311-cp311-manylinux_2_35_x86_64.whl → 0.4.2__cp311-cp311-manylinux_2_35_x86_64.whl

qubx/backtester/ome.py

@@ -140,7 +140,7 @@ class OrdersManagementEngine:
         order = Order(
             self._generate_order_id(),
             order_type,
-            self.instrument.symbol,
+            self.instrument,
             timestamp,
             amount,
             price if price is not None else 0,

qubx/backtester/queue.py

@@ -17,7 +17,7 @@ _SW = Stopwatch()
 
 
 class DataLoader:
-    _TYPE_MAPPERS = {"agg_trade": "trade", "ohlc": "bar", "ohlcv": "bar"}
+    _TYPE_MAPPERS = {"agg_trade": "trade", "bar": "ohlc", "ohlcv": "ohlc"}
 
     def __init__(
         self,
@@ -25,7 +25,7 @@ class DataLoader:
         reader: DataReader,
         instrument: Instrument,
         timeframe: str | None,
-        preload_bars: int = 0,
+        warmup_period: str | None = None,
         data_type: str = "ohlc",
         output_type: str | None = None,  # transfomer can somtimes map to a different output type
         chunksize: int = 5_000,
@@ -34,7 +34,7 @@ class DataLoader:
         self._spec = f"{instrument.exchange}:{instrument.symbol}"
         self._reader = reader
         self._transformer = transformer
-        self._init_bars_required = preload_bars
+        self._warmup_period = warmup_period
         self._timeframe = timeframe
         self._data_type = data_type
         self._output_type = output_type
@@ -43,8 +43,8 @@ class DataLoader:
 
     def load(self, start: str | pd.Timestamp, end: str | pd.Timestamp) -> Iterator:
         if self._first_load:
-            if self._init_bars_required > 0 and self._timeframe:
-                start = pd.Timestamp(start) - self._init_bars_required * pd.Timedelta(self._timeframe)
+            if self._warmup_period:
+                start = pd.Timestamp(start) - pd.Timedelta(self._warmup_period)
             self._first_load = False
 
         args = dict(
@@ -65,10 +65,6 @@ class DataLoader:
     def instrument(self) -> Instrument:
         return self._instrument
 
-    @property
-    def symbol(self) -> str:
-        return self._instrument.symbol
-
     @property
     def data_type(self) -> str:
         if self._output_type:
@@ -76,16 +72,16 @@ class DataLoader:
         return self._TYPE_MAPPERS.get(self._data_type, self._data_type)
 
     def __hash__(self) -> int:
-        return hash((self._instrument.symbol, self._data_type))
+        return hash((self._instrument, self._data_type))
 
     def __eq__(self, other: Any) -> bool:
         if not isinstance(other, DataLoader):
             return False
-        return self._instrument.symbol == other._instrument.symbol and self._data_type == other._data_type
+        return self._instrument == other._instrument and self._data_type == other._data_type
 
 
 class SimulatedDataQueue:
-    _loaders: dict[str, list[DataLoader]]
+    _loaders: dict[Instrument, list[DataLoader]]
 
     def __init__(self):
         self._loaders = defaultdict(list)
@@ -104,7 +100,7 @@ class SimulatedDataQueue:
     def __add__(self, loader: DataLoader) -> "SimulatedDataQueue":
         self._latest_loader_index += 1
         new_loader_index = self._latest_loader_index
-        self._loaders[loader.symbol].append(loader)
+        self._loaders[loader.instrument].append(loader)
         self._index_to_loader[new_loader_index] = loader
         self._loader_to_index[loader] = new_loader_index
         if self.is_running:
@@ -113,7 +109,7 @@ class SimulatedDataQueue:
 
     def __sub__(self, loader: DataLoader) -> "SimulatedDataQueue":
         loader_index = self._loader_to_index[loader]
-        self._loaders[loader.symbol].remove(loader)
+        self._loaders[loader.instrument].remove(loader)
         del self._index_to_loader[loader_index]
         del self._loader_to_index[loader]
         del self._index_to_chunk_size[loader_index]
@@ -121,12 +117,12 @@ class SimulatedDataQueue:
         self._removed_loader_indices.add(loader_index)
         return self
 
-    def get_loader(self, symbol: str, data_type: str) -> DataLoader:
-        loaders = self._loaders[symbol]
+    def get_loader(self, instrument: Instrument, data_type: str) -> DataLoader:
+        loaders = self._loaders[instrument]
         for loader in loaders:
             if loader.data_type == data_type:
                 return loader
-        raise ValueError(f"Loader for {symbol} and {data_type} not found")
+        raise ValueError(f"Loader for {instrument} and {data_type} not found")
 
     def create_iterable(self, start: str | pd.Timestamp, stop: str | pd.Timestamp) -> Iterator:
         self._start = start
@@ -146,7 +142,7 @@ class SimulatedDataQueue:
         return self
 
     @_SW.watch("DataQueue")
-    def __next__(self) -> tuple[str, str, Any]:
+    def __next__(self) -> tuple[Instrument, str, Any]:
         if not self._event_heap:
             raise StopIteration
 
@@ -162,6 +158,7 @@ class SimulatedDataQueue:
 
         loader = self._index_to_loader[loader_index]
         data_type = loader.data_type
+        # TODO: return an additional flag to indicate if the event is historical
         if dt < self._current_time:  # type: ignore
             data_type = f"hist_{data_type}"
         else:
@@ -172,7 +169,7 @@ class SimulatedDataQueue:
         if chunk_index + 1 == chunk_size:
             self._add_chunk_to_heap(loader_index)
 
-        return loader.symbol, data_type, event
+        return loader.instrument, data_type, event
 
     @_SW.watch("DataQueue")
     def _add_chunk_to_heap(self, loader_index: int):
@@ -193,143 +190,6 @@ class SimulatedDataQueue:
             return []
 
 
-class SimulatedDataQueueWithThreads(SimulatedDataQueue):
-    _loaders: dict[str, list[DataLoader]]
-
-    def __init__(self, workers: int = 4, prefetch_chunk_count: int = 1):
-        self._loaders = defaultdict(list)
-        self._start = None
-        self._stop = None
-        self._current_time = None
-        self._index_to_loader: dict[int, DataLoader] = {}
-        self._index_to_prefetch: dict[int, list[Future]] = defaultdict(list)
-        self._index_to_done: dict[int, bool] = defaultdict(bool)
-        self._loader_to_index = {}
-        self._index_to_chunk_size = {}
-        self._index_to_iterator = {}
-        self._latest_loader_index = -1
-        self._removed_loader_indices = set()
-        # TODO: potentially use ProcessPoolExecutor for better performance
-        self._pool = ThreadPoolExecutor(max_workers=workers)
-        self._prefetch_chunk_count = prefetch_chunk_count
-
-    @property
-    def is_running(self) -> bool:
-        return self._current_time is not None
-
-    def __add__(self, loader: DataLoader) -> "SimulatedDataQueueWithThreads":
-        self._latest_loader_index += 1
-        new_loader_index = self._latest_loader_index
-        self._loaders[loader.symbol].append(loader)
-        self._index_to_loader[new_loader_index] = loader
-        self._loader_to_index[loader] = new_loader_index
-        if self.is_running:
-            self._submit_chunk(new_loader_index)
-            self._add_chunk_to_heap(new_loader_index)
-        return self
-
-    def __sub__(self, loader: DataLoader) -> "SimulatedDataQueueWithThreads":
-        loader_index = self._loader_to_index[loader]
-        self._loaders[loader.symbol].remove(loader)
-        del self._index_to_loader[loader_index]
-        del self._loader_to_index[loader]
-        del self._index_to_chunk_size[loader_index]
-        del self._index_to_iterator[loader_index]
-        del self._index_to_done[loader_index]
-        for future in self._index_to_prefetch[loader_index]:
-            future.cancel()
-        del self._index_to_prefetch[loader_index]
-        self._removed_loader_indices.add(loader_index)
-        return self
-
-    def get_loader(self, symbol: str, data_type: str) -> DataLoader:
-        loaders = self._loaders[symbol]
-        for loader in loaders:
-            if loader.data_type == data_type:
-                return loader
-        raise ValueError(f"Loader for {symbol} and {data_type} not found")
-
-    def create_iterable(self, start: str | pd.Timestamp, stop: str | pd.Timestamp) -> Iterator:
-        self._start = start
-        self._stop = stop
-        self._current_time = None
-        return self
-
-    def __iter__(self) -> Iterator:
-        logger.debug("Initializing chunks for each loader")
-        self._current_time = self._start
-        self._index_to_chunk_size = {}
-        self._index_to_iterator = {}
-        self._event_heap = []
-        self._submit_chunk_prefetchers()
-        for loader_index in self._index_to_loader.keys():
-            self._add_chunk_to_heap(loader_index)
-        return self
-
-    @_SW.watch("DataQueue")
-    def __next__(self) -> tuple[str, str, Any]:
-        self._submit_chunk_prefetchers()
-
-        if not self._event_heap:
-            raise StopIteration
-
-        loader_index = None
-
-        # get the next event from the heap
-        # if the loader_index is in the removed_loader_indices, skip it (optimization to avoid unnecessary heap operations)
-        while self._event_heap and (loader_index is None or loader_index in self._removed_loader_indices):
-            dt, loader_index, chunk_index, event = heapq.heappop(self._event_heap)
-
-        if loader_index is None or loader_index in self._removed_loader_indices:
-            raise StopIteration
-
-        self._current_time = dt
-        chunk_size = self._index_to_chunk_size[loader_index]
-        if chunk_index + 1 == chunk_size:
-            self._add_chunk_to_heap(loader_index)
-
-        loader = self._index_to_loader[loader_index]
-        return loader.symbol, loader.data_type, event
-
-    @_SW.watch("DataQueue")
-    def _add_chunk_to_heap(self, loader_index: int):
-        futures = self._index_to_prefetch[loader_index]
-        if not futures and not self._index_to_done[loader_index]:
-            loader = self._index_to_loader[loader_index]
-            logger.error(f"Error state: No submitted tasks for loader {loader.symbol} {loader.data_type}")
-            raise SimulatorError("No submitted tasks for loader")
-        elif self._index_to_done[loader_index]:
-            return
-
-        # wait for future to finish if needed
-        chunk = futures.pop(0).result()
-        self._index_to_chunk_size[loader_index] = len(chunk)
-        for chunk_index, event in enumerate(chunk):
-            dt = event.time  # type: ignore
-            heapq.heappush(self._event_heap, (dt, loader_index, chunk_index, event))
-
-    def _next_chunk(self, index: int) -> list[Any]:
-        if index not in self._index_to_iterator:
-            self._index_to_iterator[index] = self._index_to_loader[index].load(self._current_time, self._stop)  # type: ignore
-        iterator = self._index_to_iterator[index]
-        try:
-            return next(iterator)
-        except StopIteration:
-            return []
-
-    def _submit_chunk_prefetchers(self):
-        for index in self._index_to_loader.keys():
-            if len(self._index_to_prefetch[index]) < self._prefetch_chunk_count:
-                self._submit_chunk(index)
-
-    def _submit_chunk(self, loader_index: int) -> None:
-        future = self._pool.submit(self._next_chunk, loader_index)
-        self._index_to_prefetch[loader_index].append(future)
-
-    def __del__(self):
-        self._pool.shutdown()
-
-
 class EventBatcher:
     _BATCH_SETTINGS = {
         "trade": "1Sec",
@@ -342,48 +202,48 @@ class EventBatcher:
         self._batch_settings = {**self._BATCH_SETTINGS, **kwargs}
         self._batch_settings = {k: pd.Timedelta(v) for k, v in self._batch_settings.items()}
 
-    def __iter__(self) -> Iterator[tuple[str, str, Any]]:
+    def __iter__(self) -> Iterator[tuple[Instrument, str, Any]]:
         if self._passthrough:
             _iter = iter(self.source_iterator) if isinstance(self.source_iterator, Iterable) else self.source_iterator
             yield from _iter
             return
 
-        last_symbol: str = None  # type: ignore
+        last_instrument: Instrument = None  # type: ignore
         last_data_type: str = None  # type: ignore
         buffer = []
-        for symbol, data_type, event in self.source_iterator:
+        for instrument, data_type, event in self.source_iterator:
             time: dt_64 = event.time  # type: ignore
 
             if data_type not in self._batch_settings:
                 if buffer:
-                    yield last_symbol, last_data_type, self._batch_event(buffer)
+                    yield last_instrument, last_data_type, self._batch_event(buffer)
                     buffer = []
-                yield symbol, data_type, event
-                last_symbol, last_data_type = symbol, data_type
+                yield instrument, data_type, event
+                last_instrument, last_data_type = instrument, data_type
                 continue
 
-            if symbol != last_symbol:
+            if instrument != last_instrument:
                 if buffer:
-                    yield last_symbol, last_data_type, self._batch_event(buffer)
-                last_symbol, last_data_type = symbol, data_type
+                    yield last_instrument, last_data_type, self._batch_event(buffer)
+                last_instrument, last_data_type = instrument, data_type
                 buffer = [event]
                 continue
 
             if buffer and data_type != last_data_type:
-                yield symbol, last_data_type, buffer
+                yield instrument, last_data_type, buffer
                 buffer = [event]
-                last_symbol, last_data_type = symbol, data_type
+                last_instrument, last_data_type = instrument, data_type
                 continue
 
-            last_symbol, last_data_type = symbol, data_type
+            last_instrument, last_data_type = instrument, data_type
             buffer.append(event)
             if pd.Timedelta(time - buffer[0].time) >= self._batch_settings[data_type]:
-                yield symbol, data_type, self._batch_event(buffer)
+                yield instrument, data_type, self._batch_event(buffer)
                 buffer = []
-                last_symbol, last_data_type = None, None  # type: ignore
+                last_instrument, last_data_type = None, None  # type: ignore
 
         if buffer:
-            yield last_symbol, last_data_type, self._batch_event(buffer)
+            yield last_instrument, last_data_type, self._batch_event(buffer)
 
     @staticmethod
     def _batch_event(buffer: list[Any]) -> Any: