Qubx 0.7.40.dev12__tar.gz → 1.0.0.dev2__tar.gz

{qubx-0.7.40.dev12 → qubx-1.0.0.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: Qubx
-Version: 0.7.40.dev12
+Version: 1.0.0.dev2
 Summary: Qubx - Quantitative Trading Framework
 Project-URL: homepage, https://xlydian.com
 Project-URL: repository, https://github.com/xLydianSoftware/Qubx
@@ -55,8 +55,6 @@ Requires-Dist: toml<1,>=0.10.2
 Requires-Dist: tqdm
 Requires-Dist: uvloop<1,>=0.22.1; sys_platform != 'win32'
 Requires-Dist: websockets==15.0.1
-Provides-Extra: hft
-Requires-Dist: hftbacktest<3,>=2.2.0; extra == 'hft'
 Provides-Extra: k8
 Requires-Dist: prometheus-client<1,>=0.21.1; extra == 'k8'
 Description-Content-Type: text/markdown

{qubx-0.7.40.dev12 → qubx-1.0.0.dev2}/pyproject.toml

@@ -66,7 +66,6 @@ docs = "https://xlydiansoftware.github.io/Qubx"
 [project.optional-dependencies]
 # Runtime optional features only (shipped with package)
 k8 = ["prometheus-client>=0.21.1,<1"]
-hft = ["hftbacktest>=2.2.0,<3"]
 
 [project.scripts]
 qubx = "qubx.cli.commands:main"
@@ -131,7 +130,7 @@ extend-select = ["I"]
 ignore = ["E731", "E722", "E741"]
 
 [tool.ruff.lint.extend-per-file-ignores]
-"*.ipynb" = ["F405", "F401", "E701", "E402", "F403", "E401", "E702", "I001"]
+"*.ipynb" = [ "F405", "F401", "E701", "E402", "F403", "E401", "E702", "F821", "E731", ]
 
 [tool.pytest.ini_options]
 asyncio_mode = "auto"

qubx-1.0.0.dev2/src/qubx/_nb_magic.py

@@ -0,0 +1,99 @@
+""" "
+Here stuff we want to have in every Jupyter notebook after calling %qubx magic
+"""
+
+import qubx
+from qubx import runtime_env
+from qubx.utils.misc import add_project_to_system_path, logo
+
+
+def np_fmt_short():
+    # default np output is 75 columns so extend it a bit and suppress scientific fmt for small floats
+    np.set_printoptions(linewidth=240, suppress=True)
+
+
+def np_fmt_reset():
+    # reset default np printing options
+    np.set_printoptions(
+        edgeitems=3,
+        infstr="inf",
+        linewidth=75,
+        nanstr="nan",
+        precision=8,
+        suppress=False,
+        threshold=1000,
+        formatter=None,
+    )
+
+
+if runtime_env() in ["notebook", "shell"]:
+    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    # -- all imports below will appear in notebook after calling %%qubx magic ---
+    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+    # - - - - Common stuff - - - -
+    import numpy as np  # type: ignore # noqa: F401
+    import pandas as pd  # type: ignore # noqa: F401
+
+    # - - - - Charting stuff - - - -
+    from matplotlib import pyplot as plt  # type: ignore # noqa: F401
+    from tqdm.auto import tqdm  # type: ignore # noqa: F401
+
+    # - - - - TA stuff and indicators - - - -
+    import qubx.pandaz.ta as pta  # type: ignore # noqa: F401
+    import qubx.ta.indicators as ta  # type: ignore # noqa: F401
+    from qubx.backtester.optimization import variate  # type: ignore # noqa: F401
+
+    # - - - - Simulator stuff - - - -
+    from qubx.backtester.simulator import simulate  # type: ignore # noqa: F401
+
+    # - - - - Portfolio analysis - - - -
+    from qubx.core.metrics import (  # type: ignore # noqa: F401
+        calculate_leverage_per_symbol,  # type: ignore
+        calculate_pnl_per_symbol,  # type: ignore
+        chart_signals,  # type: ignore
+        combine_sessions,  # type: ignore
+        drop_symbols,  # type: ignore
+        extend_trading_results,  # type: ignore
+        find_session,  # type: ignore
+        find_sessions,  # type: ignore
+        get_symbol_pnls,  # type: ignore
+        pick_symbols,  # type: ignore
+        pnl,  # type: ignore
+        portfolio_metrics,  # type: ignore
+        tearsheet,  # type: ignore
+    )
+
+    # - - - - Data storages - - - -
+    from qubx.data import (  # noqa: F401
+        CachedStorage,  # type: ignore
+        CsvStorage,  # type: ignore
+        QuestDBStorage,  # type: ignore
+    )
+    from qubx.data.registry import StorageRegistry  # noqa: F401  # type: ignore
+
+    # - - - - Utils - - - -
+    from qubx.pandaz.utils import (  # noqa: F401
+        continuous_periods,  # type: ignore
+        drop_duplicated_indexes,  # type: ignore
+        generate_equal_date_ranges,  # type: ignore
+        ohlc_resample,  # type: ignore
+        retain_columns_and_join,  # type: ignore
+        rolling_forward_test_split,  # type: ignore
+        scols,  # type: ignore
+        srows,  # type: ignore
+    )
+    from qubx.utils.charting.lookinglass import LookingGlass  # type: ignore # noqa: F401
+    from qubx.utils.charting.mpl_helpers import fig, ohlc_plot, plot_trends, sbp, subplot  # type: ignore # noqa: F401
+    from qubx.utils.misc import this_project_root  # type: ignore # noqa: F401
+
+    # - setup short numpy output format
+    np_fmt_short()
+
+    # - add project home to system path
+    add_project_to_system_path()
+
+    # show logo first time
+    if not hasattr(qubx.QubxMagics, "__already_initialized__"):
+        setattr(qubx.QubxMagics, "__already_initialized__", True)
+        logo()

{qubx-0.7.40.dev12 → qubx-1.0.0.dev2}/src/qubx/_version.py

@@ -28,7 +28,7 @@ version_tuple: VERSION_TUPLE
 commit_id: COMMIT_ID
 __commit_id__: COMMIT_ID
 
-__version__ = version = '0.7.40.dev12'
-__version_tuple__ = version_tuple = (0, 7, 40, 'dev12')
+__version__ = version = '1.0.0.dev2'
+__version_tuple__ = version_tuple = (1, 0, 0, 'dev2')
 
 __commit_id__ = commit_id = None

{qubx-0.7.40.dev12 → qubx-1.0.0.dev2}/src/qubx/backtester/data.py

@@ -4,18 +4,14 @@ from typing import TypeVar
 import pandas as pd
 
 from qubx import logger
-from qubx.backtester.simulated_data import IterableSimulationData
+from qubx.backtester.simulated_data import SimulatedDataIterator
 from qubx.core.basics import (
     CtrlChannel,
     DataType,
     Instrument,
-    TimestampedDict,
 )
-from qubx.core.helpers import BasicScheduler
 from qubx.core.interfaces import IDataProvider
-from qubx.core.series import Bar, Quote, time_as_nsec
-from qubx.data.readers import AsDict, DataReader
-from qubx.utils.time import infer_series_frequency
+from qubx.core.series import Bar, Quote
 
 from .account import SimulatedAccountProcessor
 from .utils import SimulatedTimeProvider
@@ -23,49 +19,50 @@ from .utils import SimulatedTimeProvider
 T = TypeVar("T")
 
 
-def _get_first_existing(data: dict, keys: list, default: T = None) -> T:
-    data_get = data.get  # Cache method lookup
-    sentinel = object()
-    for key in keys:
-        if (value := data_get(key, sentinel)) is not sentinel and value is not None:
-            return value
-    return default
+class SimulatedDataProvider(IDataProvider):
+    """
+    Per-exchange data provider for backtesting simulation.
 
+    Thin wrapper around SimulatedDataIterator (shared across all exchanges in a simulation).
+    Handles exchange-specific concerns: subscribe/unsubscribe lifecycle, last-quote tracking,
+    and account notifications (OME). One instance per exchange in the simulation.
+
+    Data flow for market data subscriptions:
+        subscribe() → SimulatedDataIterator.add_instruments_for_subscription()
+                     → peek historical data → emulate last quote → notify account OME
+
+    Data flow for historical OHLC lookback (ctx.ohlc()):
+        get_ohlc(instrument, timeframe, nbarsback)
+          → uses time_provider.time() as current simulated time
+          → delegates to SimulatedDataIterator.get_ohlc() which reads from IStorage,
+            transforms via TypedRecords, and applies open_close_time_indent cut
+            to exclude bars that haven't "closed" yet at the simulated time
+    """
 
-class SimulatedDataProvider(IDataProvider):
     time_provider: SimulatedTimeProvider
     channel: CtrlChannel
 
-    _scheduler: BasicScheduler
     _account: SimulatedAccountProcessor
     _last_quotes: dict[Instrument, Quote | None]
-    _readers: dict[str, DataReader]
-    _data_source: IterableSimulationData
-    _open_close_time_indent_ns: int
+    _data_source: SimulatedDataIterator
 
     def __init__(
         self,
         exchange_id: str,
         channel: CtrlChannel,
-        scheduler: BasicScheduler,
         time_provider: SimulatedTimeProvider,
         account: SimulatedAccountProcessor,
-        readers: dict[str, DataReader],
-        data_source: IterableSimulationData,
-        open_close_time_indent_secs=1,
+        data_source: SimulatedDataIterator,
     ):
         self.channel = channel
         self.time_provider = time_provider
         self._exchange_id = exchange_id
-        self._scheduler = scheduler
         self._account = account
-        self._readers = readers
 
         # - simulation data source
         self._data_source = data_source
-        self._open_close_time_indent_ns = open_close_time_indent_secs * 1_000_000_000  # convert seconds to nanoseconds
 
-        # - create exchange's instance
+        # - create last quote holder
         self._last_quotes = defaultdict(lambda: None)
 
         logger.info(f"{self.__class__.__name__}.{exchange_id} is initialized")
@@ -152,19 +149,9 @@ class SimulatedDataProvider(IDataProvider):
             self._data_source.set_warmup_period(si[0], warm_period)
 
     def get_ohlc(self, instrument: Instrument, timeframe: str, nbarsback: int) -> list[Bar]:
-        _reader = self._readers.get(DataType.OHLC)
-        if _reader is None:
-            logger.error(f"Reader for {DataType.OHLC} data not configured")
-            return []
-
         start = pd.Timestamp(self.time_provider.time())
-        end = start - nbarsback * (_timeframe := pd.Timedelta(timeframe))
-        _spec = f"{instrument.exchange}:{instrument.symbol}"
-        return self._convert_records_to_bars(
-            _reader.read(data_id=_spec, start=start, stop=end, timeframe=timeframe, transform=AsDict()),  # type: ignore
-            time_as_nsec(self.time_provider.time()),
-            _timeframe.asm8.item(),
-        )
+        end = start - nbarsback * pd.Timedelta(timeframe)
+        return self._data_source.get_ohlc(instrument, timeframe, start, end)
 
     def get_quote(self, instrument: Instrument) -> Quote | None:
         return self._last_quotes[instrument]
@@ -172,57 +159,5 @@ class SimulatedDataProvider(IDataProvider):
     def close(self):
         pass
 
-    def _convert_records_to_bars(
-        self, records: list[TimestampedDict], cut_time_ns: int, timeframe_ns: int
-    ) -> list[Bar]:
-        """
-        Convert records to bars and we need to cut last bar up to the cut_time_ns
-        """
-        bars = []
-
-        # - if no records, return empty list to avoid exception from infer_series_frequency
-        if not records or records is None:
-            return bars
-
-        if len(records) > 1:
-            _data_tf = infer_series_frequency([r.time for r in records[:50]])
-            timeframe_ns = _data_tf.item()
-
-        for r in records:
-            _b_ts_0 = r.time
-            _b_ts_1 = _b_ts_0 + timeframe_ns - self._open_close_time_indent_ns
-
-            if _b_ts_0 <= cut_time_ns and cut_time_ns < _b_ts_1:
-                break
-
-            # Handle None values in OHLC data
-            open_price = r.data["open"]
-            high_price = r.data["high"]
-            low_price = r.data["low"]
-            close_price = r.data["close"]
-
-            # Skip this record if any OHLC value is None
-            if open_price is None or high_price is None or low_price is None or close_price is None:
-                continue
-
-            bars.append(
-                Bar(
-                    _b_ts_0,
-                    open_price,
-                    high_price,
-                    low_price,
-                    close_price,
-                    volume=r.data.get("volume", 0) or 0,  # Handle None volume
-                    bought_volume=_get_first_existing(r.data, ["taker_buy_volume", "bought_volume"], 0),
-                    volume_quote=_get_first_existing(r.data, ["quote_volume", "volume_quote"], 0),
-                    bought_volume_quote=_get_first_existing(
-                        r.data, ["taker_buy_quote_volume", "bought_volume_quote"], 0
-                    ),
-                    trade_count=_get_first_existing(r.data, ["count", "trade_count"], 0),
-                )
-            )
-
-        return bars
-
     def exchange(self) -> str:
         return self._exchange_id.upper()

qubx-1.0.0.dev2/src/qubx/backtester/iteratedstream.py

@@ -0,0 +1,194 @@
+import math
+from collections import defaultdict, deque
+from collections.abc import Callable, Iterator, Mapping
+from typing import Any, TypeAlias
+
+from qubx.core.basics import Timestamped
+
+SlicerOutData: TypeAlias = tuple[str, int, Timestamped] | tuple
+
+
+class IteratedDataStreamsSlicer(Iterator[SlicerOutData]):
+    """
+    This class manages seamless iteration over multiple time-series data streams,
+    ensuring that events are processed in the correct chronological order regardless of their source.
+    It supports adding / removing new data streams to the slicer on the fly (during the itration).
+    """
+
+    _iterators: dict[str, Iterator[list[Timestamped]]]
+    _buffers: dict[str, list[Timestamped]]
+    _keys: deque[str]
+    _iterating: bool
+
+    def __init__(self, time_func: Callable[[Timestamped], Any] = lambda x: x.time):
+        self._buffers = defaultdict(list)
+        self._iterators = {}
+        self._keys = deque()
+        self._iterating = False
+        self._time_func = time_func
+
+    def put(self, data: Mapping[str, Iterator[list[Timestamped]]]):
+        _rebuild = False
+        for k, vi in data.items():
+            if k not in self._keys:
+                self._iterators[k] = vi
+                self._buffers[k] = self._load_next_chunk_to_buffer(k)  # do initial chunk fetching
+                self._keys.append(k)
+                _rebuild = True
+
+        # - rebuild strategy
+        if _rebuild and self._iterating:
+            self._build_initial_iteration_seq()
+
+    def __add__(self, data: Mapping[str, Iterator]) -> "IteratedDataStreamsSlicer":
+        self.put(data)
+        return self
+
+    def remove(self, keys: list[str] | str):
+        """
+        Remove data iterator and associated keys from the queue.
+        If the key is not found, it does nothing.
+        """
+        _keys = keys if isinstance(keys, list) else [keys]
+        _rebuild = False
+        for i in _keys:
+            # Check and remove from each data structure independently
+            removed_any = False
+
+            if i in self._buffers:
+                self._buffers.pop(i)
+                removed_any = True
+
+            if i in self._iterators:
+                self._iterators.pop(i)
+                removed_any = True
+
+            if i in self._keys:
+                self._keys.remove(i)
+                removed_any = True
+
+            if removed_any:
+                _rebuild = True
+
+        # - rebuild strategy
+        if _rebuild and self._iterating:
+            self._build_initial_iteration_seq()
+
+    def __iter__(self) -> Iterator:
+        self._build_initial_iteration_seq()
+        self._iterating = True
+        return self
+
+    def _build_initial_iteration_seq(self):
+        _init_seq = {k: self._time_func(self._buffers[k][-1]) for k in self._keys if self._buffers[k]}
+        _init_seq = dict(sorted(_init_seq.items(), key=lambda item: item[1]))
+        self._keys = deque(_init_seq.keys())
+
+    def _load_next_chunk_to_buffer(self, index: str) -> list[Timestamped]:
+        try:
+            return list(reversed(next(self._iterators[index])))
+        except (StopIteration, IndexError, RuntimeError):
+            return []
+
+    def _remove_iterator(self, key: str):
+        self._buffers.pop(key)
+        self._iterators.pop(key)
+        self._keys.remove(key)
+
+    def _pop_top(self, k: str) -> Timestamped:
+        """
+        Removes and returns the most recent timestamped data element from the buffer associated with the given key.
+        If the buffer is empty after popping, it attempts to load the next chunk of data into the buffer.
+        If no more data is available, the iterator associated with the key is removed.
+
+        Parameters:
+            k (str): The key identifying the data stream buffer to pop from.
+
+        Returns:
+            Timestamped: The most recent timestamped data element from the buffer.
+        """
+        if not self._buffers[k]:
+            raise StopIteration
+
+        v = (data := self._buffers[k]).pop()
+        if not data:
+            try:
+                data.extend(self._load_next_chunk_to_buffer(k))  # - get next chunk of data
+            except StopIteration:
+                self._remove_iterator(k)  # - remove iterable data
+        return v
+
+    def fetch_before_time(self, key: str, time_ns: int) -> list[Timestamped]:
+        """
+        Fetches and returns all timestamped data elements from the buffer associated with the given key
+        that have a timestamp earlier than the specified time.
+
+        Parameters:
+            - key (str): The key identifying the data stream buffer to fetch from.
+            - time_ns (int): The timestamp in nanoseconds. All returned elements will have a timestamp less than this value.
+
+        Returns:
+            - list[Timestamped]: A list of timestamped data elements that occur before the specified time.
+        """
+        values = []
+        data = self._buffers[key]
+        if not data:
+            try:
+                data.extend(self._load_next_chunk_to_buffer(key))  # - get next chunk of data
+            except StopIteration:
+                self._remove_iterator(key)
+                # Return empty list if no data is available
+                return values
+
+        # Check if data is still empty after attempting to load
+        if not data:
+            return values
+
+        # pull most past elements
+        v = data[-1]
+        while self._time_func(v) < time_ns:
+            values.append(data.pop())
+            if not data:
+                try:
+                    data.extend(self._load_next_chunk_to_buffer(key))  # - get next chunk of data
+                except StopIteration:
+                    self._remove_iterator(key)
+                    break
+                # Check if data is still empty after loading attempt
+                if not data:
+                    break
+            v = data[-1]
+
+        return values
+
+    def __next__(self) -> SlicerOutData:
+        """
+        Advances the iterator to the next available timestamped data element across all data streams.
+
+        Returns:
+            - SlicerOutData: A tuple containing the key of the data stream, the timestamp of the data element, and the data element itself.
+            - NoDataContinue: If there are no data streams but scheduler has pending events.
+
+        Raises:
+            - StopIteration: If there are no more data elements to iterate over and no scheduled events.
+        """
+        if not self._keys:
+            # DON'T set _iterating = False here! We're still iterating, just temporarily out of data
+            # Return sentinel indicating no data streams but iteration could continue
+            from qubx.backtester.sentinels import NoDataContinue
+
+            return ("", 0, NoDataContinue())
+
+        _min_t = math.inf
+        _min_k = self._keys[0]
+        for i in self._keys:
+            if not self._buffers[i]:
+                continue
+
+            _x = self._buffers[i][-1]
+            if self._time_func(_x) < _min_t:
+                _min_t = self._time_func(_x)
+                _min_k = i
+
+        _v = self._pop_top(_min_k)
+        return (_min_k, self._time_func(_v), _v)