Qubx 0.4.3__cp311-cp311-manylinux_2_35_x86_64.whl → 0.5.0__cp311-cp311-manylinux_2_35_x86_64.whl

qubx/__init__.py

@@ -15,7 +15,10 @@ def formatter(record):
     if record["level"].name in {"WARNING", "SNAKY"}:
         fmt = "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - %s" % fmt
 
-    prefix = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> [ <level>%s</level> ] " % record["level"].icon
+    prefix = (
+        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> [ <level>%s</level> ] <cyan>({module})</cyan> "
+        % record["level"].icon
+    )
 
     if record["exception"] is not None:
         # stackprinter.set_excepthook(style='darkbg2')
@@ -29,7 +32,6 @@ def formatter(record):
 
 
 class QubxLogConfig:
-
     @staticmethod
     def get_log_level():
         return os.getenv("QUBX_LOG_LEVEL", "DEBUG")
@@ -107,14 +109,15 @@ if runtime_env() in ["notebook", "shell"]:
             if line:
                 if "dark" in line.lower():
                     set_mpl_theme("dark")
+                    # - temporary workaround for vscode - dark theme not applying to ipywidgets in notebook
+                    # - see https://github.com/microsoft/vscode-jupyter/issues/7161
+                    if runtime_env() == "notebook":
+                        _vscode_clr_trick = """from IPython.display import display, HTML; display(HTML("<style> .cell-output-ipywidget-background { background-color: transparent !important; } :root { --jp-widgets-color: var(--vscode-editor-foreground); --jp-widgets-font-size: var(--vscode-editor-font-size); } </style>"))"""
+                        exec(_vscode_clr_trick, self.shell.user_ns)
 
                 elif "light" in line.lower():
                     set_mpl_theme("light")
 
-            # install additional plotly helpers
-            # from qube.charting.plot_helpers import install_plotly_helpers
-            # install_plotly_helpers()
-
         def _get_manager(self):
             if self.__manager is None:
                 import multiprocessing as m

qubx/_nb_magic.py

@@ -1,4 +1,4 @@
-""""
+""" "
 Here stuff we want to have in every Jupyter notebook after calling %qubx magic
 """
 
@@ -27,7 +27,6 @@ def np_fmt_reset():
 
 
 if runtime_env() in ["notebook", "shell"]:
-
     # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
     # -- all imports below will appear in notebook after calling %%qubx magic ---
     # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -

qubx/backtester/account.py

@@ -0,0 +1,134 @@
+from qubx import logger
+from qubx.backtester.ome import OrdersManagementEngine
+from qubx.core.account import BasicAccountProcessor
+from qubx.core.basics import (
+    ZERO_COSTS,
+    CtrlChannel,
+    Instrument,
+    Order,
+    Position,
+    TransactionCostsCalculator,
+    dt_64,
+)
+from qubx.core.interfaces import ITimeProvider
+from qubx.core.series import Bar, Quote, Trade
+
+
+class SimulatedAccountProcessor(BasicAccountProcessor):
+    ome: dict[Instrument, OrdersManagementEngine]
+    order_to_instrument: dict[str, Instrument]
+
+    _channel: CtrlChannel
+    _fill_stop_order_at_price: bool
+    _half_tick_size: dict[Instrument, float]
+
+    def __init__(
+        self,
+        account_id: str,
+        channel: CtrlChannel,
+        base_currency: str,
+        initial_capital: float,
+        time_provider: ITimeProvider,
+        tcc: TransactionCostsCalculator = ZERO_COSTS,
+        accurate_stop_orders_execution: bool = False,
+    ) -> None:
+        super().__init__(
+            account_id=account_id,
+            time_provider=time_provider,
+            base_currency=base_currency,
+            tcc=tcc,
+            initial_capital=initial_capital,
+        )
+        self.ome = {}
+        self.order_to_instrument = {}
+        self._channel = channel
+        self._half_tick_size = {}
+        self._fill_stop_order_at_price = accurate_stop_orders_execution
+        if self._fill_stop_order_at_price:
+            logger.info(f"{self.__class__.__name__} emulates stop orders executions at exact price")
+
+    def get_orders(self, instrument: Instrument | None = None) -> list[Order]:
+        if instrument is not None:
+            ome = self.ome.get(instrument)
+            if ome is None:
+                raise ValueError(f"ExchangeService:get_orders :: No OME configured for '{instrument}'!")
+            return ome.get_open_orders()
+
+        return [o for ome in self.ome.values() for o in ome.get_open_orders()]
+
+    def get_position(self, instrument: Instrument) -> Position:
+        if instrument in self.positions:
+            return self.positions[instrument]
+
+        # - initiolize OME for this instrument
+        self.ome[instrument] = OrdersManagementEngine(
+            instrument=instrument,
+            time_provider=self.time_provider,
+            tcc=self._tcc,  # type: ignore
+            fill_stop_order_at_price=self._fill_stop_order_at_price,
+        )
+
+        # - initiolize empty position
+        position = Position(instrument)  # type: ignore
+        self._half_tick_size[instrument] = instrument.tick_size / 2  # type: ignore
+        self.attach_positions(position)
+        return self.positions[instrument]
+
+    def update_position_price(self, time: dt_64, instrument: Instrument, price: float) -> None:
+        super().update_position_price(time, instrument, price)
+
+        # - first we need to update OME with new quote.
+        # - if update is not a quote we need 'emulate' it.
+        # - actually if SimulatedExchangeService is used in backtesting mode it will recieve only quotes
+        # - case when we need that - SimulatedExchangeService is used for paper trading and data provider configured to listen to OHLC or TAS.
+        # - probably we need to subscribe to quotes in real data provider in any case and then this emulation won't be needed.
+        quote = price if isinstance(price, Quote) else self.emulate_quote_from_data(instrument, time, price)
+        if quote is None:
+            return
+
+        # - process new quote
+        self._process_new_quote(instrument, quote)
+
+    def process_order(self, order: Order, update_locked_value: bool = True) -> None:
+        _new = order.status == "NEW"
+        _open = order.status == "OPEN"
+        _cancel = order.status == "CANCELED"
+        _closed = order.status == "CLOSED"
+        if _new or _open:
+            self.order_to_instrument[order.id] = order.instrument
+        if (_cancel or _closed) and order.id in self.order_to_instrument:
+            self.order_to_instrument.pop(order.id)
+        return super().process_order(order, update_locked_value)
+
+    def emulate_quote_from_data(
+        self, instrument: Instrument, timestamp: dt_64, data: float | Trade | Bar
+    ) -> Quote | None:
+        if instrument not in self._half_tick_size:
+            _ = self.get_position(instrument)
+
+        _ts2 = self._half_tick_size[instrument]
+        if isinstance(data, Quote):
+            return data
+        elif isinstance(data, Trade):
+            if data.taker:  # type: ignore
+                return Quote(timestamp, data.price - _ts2 * 2, data.price, 0, 0)  # type: ignore
+            else:
+                return Quote(timestamp, data.price, data.price + _ts2 * 2, 0, 0)  # type: ignore
+        elif isinstance(data, Bar):
+            return Quote(timestamp, data.close - _ts2, data.close + _ts2, 0, 0)  # type: ignore
+        elif isinstance(data, float):
+            return Quote(timestamp, data - _ts2, data + _ts2, 0, 0)
+        else:
+            return None
+
+    def _process_new_quote(self, instrument: Instrument, data: Quote) -> None:
+        ome = self.ome.get(instrument)
+        if ome is None:
+            logger.warning("ExchangeService:update :: No OME configured for '{symbol}' yet !")
+            return
+        for r in ome.update_bbo(data):
+            if r.exec is not None:
+                self.order_to_instrument.pop(r.order.id)
+                # - process methods will be called from stg context
+                self._channel.send((instrument, "order", r.order, False))
+                self._channel.send((instrument, "deals", [r.exec], False))

qubx/backtester/broker.py

@@ -0,0 +1,82 @@
+from qubx.backtester.ome import OmeReport
+from qubx.core.basics import (
+    CtrlChannel,
+    Instrument,
+    Order,
+)
+from qubx.core.interfaces import IBroker
+
+from .account import SimulatedAccountProcessor
+
+
+class SimulatedBroker(IBroker):
+    channel: CtrlChannel
+
+    _account: SimulatedAccountProcessor
+
+    def __init__(
+        self,
+        channel: CtrlChannel,
+        account: SimulatedAccountProcessor,
+    ) -> None:
+        self.channel = channel
+        self._account = account
+
+    @property
+    def is_simulated_trading(self) -> bool:
+        return True
+
+    def send_order(
+        self,
+        instrument: Instrument,
+        order_side: str,
+        order_type: str,
+        amount: float,
+        price: float | None = None,
+        client_id: str | None = None,
+        time_in_force: str = "gtc",
+        **options,
+    ) -> Order:
+        ome = self._account.ome.get(instrument)
+        if ome is None:
+            raise ValueError(f"ExchangeService:send_order :: No OME configured for '{instrument.symbol}'!")
+
+        # - try to place order in OME
+        report = ome.place_order(
+            order_side.upper(),  # type: ignore
+            order_type.upper(),  # type: ignore
+            amount,
+            price,
+            client_id,
+            time_in_force,
+            **options,
+        )
+
+        self._send_exec_report(instrument, report)
+        return report.order
+
+    def cancel_order(self, order_id: str) -> Order | None:
+        instrument = self._account.order_to_instrument.get(order_id)
+        if instrument is None:
+            raise ValueError(f"ExchangeService:cancel_order :: can't find order with id = '{order_id}'!")
+
+        ome = self._account.ome.get(instrument)
+        if ome is None:
+            raise ValueError(f"ExchangeService:send_order :: No OME configured for '{instrument}'!")
+
+        # - cancel order in OME and remove from the map to free memory
+        order_update = ome.cancel_order(order_id)
+        self._send_exec_report(instrument, order_update)
+
+        return order_update.order
+
+    def cancel_orders(self, instrument: Instrument) -> None:
+        raise NotImplementedError("Not implemented yet")
+
+    def update_order(self, order_id: str, price: float | None = None, amount: float | None = None) -> Order:
+        raise NotImplementedError("Not implemented yet")
+
+    def _send_exec_report(self, instrument: Instrument, report: OmeReport):
+        self.channel.send((instrument, "order", report.order, False))
+        if report.exec is not None:
+            self.channel.send((instrument, "deals", [report.exec], False))

qubx/backtester/data.py

@@ -0,0 +1,269 @@
+from collections import defaultdict
+from typing import Any, Dict, Optional
+
+import numpy as np
+import pandas as pd
+from tqdm.auto import tqdm
+
+from qubx import logger
+from qubx.backtester.simulated_data import EventBatcher, IterableSimulationData
+from qubx.core.basics import (
+    CtrlChannel,
+    DataType,
+    Instrument,
+    TimestampedDict,
+)
+from qubx.core.helpers import BasicScheduler
+from qubx.core.interfaces import IDataProvider
+from qubx.core.series import Bar, Quote, time_as_nsec
+from qubx.data.readers import AsDict, DataReader
+from qubx.utils.time import infer_series_frequency
+
+from .account import SimulatedAccountProcessor
+from .utils import SimulatedTimeProvider
+
+
+class SimulatedDataProvider(IDataProvider):
+    time_provider: SimulatedTimeProvider
+    channel: CtrlChannel
+
+    _scheduler: BasicScheduler
+    _account: SimulatedAccountProcessor
+    _last_quotes: Dict[Instrument, Optional[Quote]]
+    _readers: dict[str, DataReader]
+    _scheduler: BasicScheduler
+    _pregenerated_signals: dict[Instrument, pd.Series | pd.DataFrame]
+    _to_process: dict[Instrument, list]
+    _data_source: IterableSimulationData
+    _open_close_time_indent_ns: int
+
+    def __init__(
+        self,
+        exchange_id: str,
+        channel: CtrlChannel,
+        scheduler: BasicScheduler,
+        time_provider: SimulatedTimeProvider,
+        account: SimulatedAccountProcessor,
+        readers: dict[str, DataReader],
+        open_close_time_indent_secs=1,
+    ):
+        self.channel = channel
+        self.time_provider = time_provider
+        self._exchange_id = exchange_id
+        self._scheduler = scheduler
+        self._account = account
+        self._readers = readers
+
+        # - create exchange's instance
+        self._last_quotes = defaultdict(lambda: None)
+
+        # - pregenerated signals storage
+        self._pregenerated_signals = dict()
+        self._to_process = {}
+
+        # - simulation data source
+        self._data_source = IterableSimulationData(
+            self._readers, open_close_time_indent_secs=open_close_time_indent_secs
+        )
+        self._open_close_time_indent_ns = open_close_time_indent_secs * 1_000_000_000  # convert seconds to nanoseconds
+
+        logger.info(f"{self.__class__.__name__}.{exchange_id} is initialized")
+
+    def run(
+        self,
+        start: str | pd.Timestamp,
+        end: str | pd.Timestamp,
+        silent: bool = False,
+        enable_event_batching: bool = True,
+    ) -> None:
+        logger.info(f"{self.__class__.__name__} ::: Simulation started at {start} :::")
+
+        if self._pregenerated_signals:
+            self._prepare_generated_signals(start, end)
+            _run = self._run_generated_signals
+            enable_event_batching = False  # no batching for pre-generated signals
+        else:
+            _run = self._run_as_strategy
+
+        qiter = EventBatcher(self._data_source.create_iterable(start, end), passthrough=not enable_event_batching)
+        start, end = pd.Timestamp(start), pd.Timestamp(end)
+        total_duration = end - start
+        update_delta = total_duration / 100
+        prev_dt = pd.Timestamp(start)
+
+        if silent:
+            for instrument, data_type, event, is_hist in qiter:
+                if not _run(instrument, data_type, event, is_hist):
+                    break
+        else:
+            _p = 0
+            with tqdm(total=100, desc="Simulating", unit="%", leave=False) as pbar:
+                for instrument, data_type, event, is_hist in qiter:
+                    if not _run(instrument, data_type, event, is_hist):
+                        break
+                    dt = pd.Timestamp(event.time)
+                    # update only if date has changed
+                    if dt - prev_dt > update_delta:
+                        _p += 1
+                        pbar.n = _p
+                        pbar.refresh()
+                        prev_dt = dt
+                pbar.n = 100
+                pbar.refresh()
+
+        logger.info(f"{self.__class__.__name__} ::: Simulation finished at {end} :::")
+
+    def set_generated_signals(self, signals: pd.Series | pd.DataFrame):
+        logger.debug(f"Using pre-generated signals:\n {str(signals.count()).strip('ndtype: int64')}")
+        # - sanity check
+        signals.index = pd.DatetimeIndex(signals.index)
+
+        if isinstance(signals, pd.Series):
+            self._pregenerated_signals[str(signals.name)] = signals  # type: ignore
+
+        elif isinstance(signals, pd.DataFrame):
+            for col in signals.columns:
+                self._pregenerated_signals[col] = signals[col]  # type: ignore
+        else:
+            raise ValueError("Invalid signals or strategy configuration")
+
+    @property
+    def is_simulation(self) -> bool:
+        return True
+
+    def subscribe(self, subscription_type: str, instruments: set[Instrument], reset: bool) -> None:
+        logger.debug(f" | subscribe: {subscription_type} -> {instruments}")
+        self._data_source.add_instruments_for_subscription(subscription_type, list(instruments))
+
+    def unsubscribe(self, subscription_type: str, instruments: set[Instrument] | Instrument | None = None) -> None:
+        logger.debug(f" | unsubscribe: {subscription_type} -> {instruments}")
+        if instruments is not None:
+            self._data_source.remove_instruments_from_subscription(
+                subscription_type, [instruments] if isinstance(instruments, Instrument) else list(instruments)
+            )
+
+    def has_subscription(self, instrument: Instrument, subscription_type: str) -> bool:
+        return self._data_source.has_subscription(instrument, subscription_type)
+
+    def get_subscriptions(self, instrument: Instrument) -> list[str]:
+        _s_lst = self._data_source.get_subscriptions_for_instrument(instrument)
+        logger.debug(f" | get_subscriptions {instrument} -> {_s_lst}")
+        return _s_lst
+
+    def get_subscribed_instruments(self, subscription_type: str | None = None) -> list[Instrument]:
+        _in_lst = self._data_source.get_instruments_for_subscription(subscription_type or DataType.ALL)
+        logger.debug(f" | get_subscribed_instruments {subscription_type} -> {_in_lst}")
+        return _in_lst
+
+    def warmup(self, configs: dict[tuple[str, Instrument], str]) -> None:
+        for si, warm_period in configs.items():
+            logger.debug(f" | Warming up {si} -> {warm_period}")
+            self._data_source.set_warmup_period(si[0], warm_period)
+
+    def get_ohlc(self, instrument: Instrument, timeframe: str, nbarsback: int) -> list[Bar]:
+        _reader = self._readers.get(DataType.OHLC)
+        if _reader is None:
+            logger.error(f"Reader for {DataType.OHLC} data not configured")
+            return []
+
+        start = pd.Timestamp(self.time_provider.time())
+        end = start - nbarsback * (_timeframe := pd.Timedelta(timeframe))
+        _spec = f"{instrument.exchange}:{instrument.symbol}"
+        return self._convert_records_to_bars(
+            _reader.read(data_id=_spec, start=start, stop=end, transform=AsDict()),  # type: ignore
+            time_as_nsec(self.time_provider.time()),
+            _timeframe.asm8.item(),
+        )
+
+    def get_quote(self, instrument: Instrument) -> Quote | None:
+        return self._last_quotes[instrument]
+
+    def close(self):
+        pass
+
+    def _prepare_generated_signals(self, start: str | pd.Timestamp, end: str | pd.Timestamp):
+        for s, v in self._pregenerated_signals.items():
+            _s_inst = None
+
+            for i in self.get_subscribed_instruments():
+                # - we can process series with variable id's if we can find some similar instrument
+                if s == i.symbol or s == str(i) or s == f"{i.exchange}:{i.symbol}" or str(s) == str(i):
+                    _start, _end = pd.Timestamp(start), pd.Timestamp(end)
+                    _start_idx, _end_idx = v.index.get_indexer([_start, _end], method="ffill")
+                    sel = v.iloc[max(_start_idx, 0) : _end_idx + 1]  # sel = v[pd.Timestamp(start) : pd.Timestamp(end)]
+
+                    self._to_process[i] = list(zip(sel.index, sel.values))
+                    _s_inst = i
+                    break
+
+            if _s_inst is None:
+                logger.error(f"Can't find instrument for pregenerated signals with id '{s}'")
+                raise ValueError(f"Can't find instrument for pregenerated signals with id '{s}'")
+
+    def _convert_records_to_bars(
+        self, records: list[TimestampedDict], cut_time_ns: int, timeframe_ns: int
+    ) -> list[Bar]:
+        """
+        Convert records to bars and we need to cut last bar up to the cut_time_ns
+        """
+        bars = []
+
+        _data_tf = infer_series_frequency([r.time for r in records[:50]])
+        timeframe_ns = _data_tf.item()
+
+        if records is not None:
+            for r in records:
+                # _b_ts_0 = np.datetime64(r.time, "ns").item()
+                _b_ts_0 = r.time
+                _b_ts_1 = _b_ts_0 + timeframe_ns - self._open_close_time_indent_ns
+
+                if _b_ts_0 <= cut_time_ns and cut_time_ns < _b_ts_1:
+                    break
+
+                bars.append(
+                    Bar(
+                        _b_ts_0, r.data["open"], r.data["high"], r.data["low"], r.data["close"], r.data.get("volume", 0)
+                    )
+                )
+
+        return bars
+
+    def _run_generated_signals(self, instrument: Instrument, data_type: str, data: Any, is_hist) -> bool:
+        if is_hist:
+            raise ValueError("Historical data is not supported for pre-generated signals !")
+
+        t = data.time  # type: ignore
+        self.time_provider.set_time(np.datetime64(t, "ns"))
+
+        q = self._account.emulate_quote_from_data(instrument, np.datetime64(t, "ns"), data)
+        self._last_quotes[instrument] = q
+        cc = self.channel
+
+        # - we need to send quotes for invoking portfolio logging etc
+        cc.send((instrument, data_type, data, is_hist))
+        sigs = self._to_process[instrument]
+        _current_time = self.time_provider.time()
+        while sigs and sigs[0][0].as_unit("ns").asm8 <= _current_time:
+            cc.send((instrument, "event", {"order": sigs[0][1]}, is_hist))
+            sigs.pop(0)
+
+        return cc.control.is_set()
+
+    def _run_as_strategy(self, instrument: Instrument, data_type: str, data: Any, is_hist: bool) -> bool:
+        t = data.time  # type: ignore
+        self.time_provider.set_time(np.datetime64(t, "ns"))
+
+        q = self._account.emulate_quote_from_data(instrument, np.datetime64(t, "ns"), data)
+        cc = self.channel
+
+        if not is_hist and q is not None:
+            self._last_quotes[instrument] = q
+
+            # we have to schedule possible crons before sending the data event itself
+            if self._scheduler.check_and_run_tasks():
+                # - push nothing - it will force to process last event
+                cc.send((None, "service_time", None, False))
+
+        cc.send((instrument, data_type, data, is_hist))
+
+        return cc.control.is_set()