Qubx 0.5.7__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/core/mixins/subscription.py

@@ -0,0 +1,203 @@
+from collections import defaultdict
+from typing import Any
+
+from qubx.core.basics import DataType, Instrument
+from qubx.core.interfaces import IDataProvider, ISubscriptionManager
+from qubx.utils.misc import synchronized
+
+
+class SubscriptionManager(ISubscriptionManager):
+    _data_provider: IDataProvider
+    _base_sub: str
+    _sub_to_warmup: dict[str, str]
+    _auto_subscribe: bool
+
+    _pending_global_subscriptions: set[str]
+    _pending_global_unsubscriptions: set[str]
+
+    _pending_stream_subscriptions: dict[str, set[Instrument]]
+    _pending_stream_unsubscriptions: dict[str, set[Instrument]]
+    _pending_warmups: dict[tuple[str, Instrument], str]
+
+    def __init__(
+        self,
+        data_provider: IDataProvider,
+        auto_subscribe: bool = True,
+        default_base_subscription: DataType = DataType.NONE,
+    ) -> None:
+        self._data_provider = data_provider
+        self._base_sub = default_base_subscription
+        self._sub_to_warmup = {}
+        self._pending_warmups = {}
+        self._pending_global_subscriptions = set()
+        self._pending_global_unsubscriptions = set()
+        self._pending_stream_subscriptions = defaultdict(set)
+        self._pending_stream_unsubscriptions = defaultdict(set)
+        self._auto_subscribe = auto_subscribe
+
+    def subscribe(self, subscription_type: str, instruments: list[Instrument] | Instrument | None = None) -> None:
+        # - figure out which instruments to subscribe to (all or specific)
+        if instruments is None:
+            self._pending_global_subscriptions.add(subscription_type)
+            return
+
+        if isinstance(instruments, Instrument):
+            instruments = [instruments]
+
+        # - get instruments that are not already subscribed to
+        _current_instruments = self._data_provider.get_subscribed_instruments(subscription_type)
+        instruments = list(set(instruments).difference(_current_instruments))
+
+        # - subscribe to all existing subscriptions if subscription_type is ALL
+        if subscription_type == DataType.ALL:
+            subscriptions = self.get_subscriptions()
+            for sub in subscriptions:
+                self.subscribe(sub, instruments)
+            return
+
+        self._pending_stream_subscriptions[subscription_type].update(instruments)
+        self._update_pending_warmups(subscription_type, instruments)
+
+    def unsubscribe(self, subscription_type: str, instruments: list[Instrument] | Instrument | None = None) -> None:
+        if instruments is None:
+            self._pending_global_unsubscriptions.add(subscription_type)
+            return
+
+        if isinstance(instruments, Instrument):
+            instruments = [instruments]
+
+        # - subscribe to all existing subscriptions if subscription_type is ALL
+        if subscription_type == DataType.ALL:
+            subscriptions = self.get_subscriptions()
+            for sub in subscriptions:
+                self.unsubscribe(sub, instruments)
+            return
+
+        self._pending_stream_unsubscriptions[subscription_type].update(instruments)
+
+    @synchronized
+    def commit(self) -> None:
+        if not self._has_operations_to_commit():
+            return
+
+        # - warm up subscriptions
+        self._run_warmup()
+
+        # - update subscriptions
+        for _sub in self._get_updated_subs():
+            _current_sub_instruments = set(self._data_provider.get_subscribed_instruments(_sub))
+            _removed_instruments = self._pending_stream_unsubscriptions.get(_sub, set())
+            _added_instruments = self._pending_stream_subscriptions.get(_sub, set())
+
+            if _sub in self._pending_global_unsubscriptions:
+                _removed_instruments.update(_current_sub_instruments)
+
+            if _sub in self._pending_global_subscriptions:
+                _added_instruments.update(self._data_provider.get_subscribed_instruments())
+
+            # - subscribe collection
+            _updated_instruments = _current_sub_instruments.union(_added_instruments).difference(_removed_instruments)
+            if _updated_instruments != _current_sub_instruments:
+                self._data_provider.subscribe(_sub, _updated_instruments, reset=True)
+
+            # - unsubscribe instruments
+            if _removed_instruments:
+                self._data_provider.unsubscribe(_sub, _removed_instruments)
+
+        # - clean up pending subs and unsubs
+        self._pending_stream_subscriptions.clear()
+        self._pending_stream_unsubscriptions.clear()
+        self._pending_global_subscriptions.clear()
+        self._pending_global_unsubscriptions.clear()
+
+    def has_subscription(self, instrument: Instrument, subscription_type: str) -> bool:
+        return self._data_provider.has_subscription(instrument, subscription_type)
+
+    def get_subscriptions(self, instrument: Instrument | None = None) -> list[str]:
+        return list(
+            set(self._data_provider.get_subscriptions(instrument))
+            | {self.get_base_subscription()}
+            | self._pending_global_subscriptions
+        )
+
+    def get_subscribed_instruments(self, subscription_type: str | None = None) -> list[Instrument]:
+        return self._data_provider.get_subscribed_instruments(subscription_type)
+
+    def get_base_subscription(self) -> str:
+        return self._base_sub
+
+    def set_base_subscription(self, subscription_type: str) -> None:
+        self._base_sub = subscription_type
+
+    def get_warmup(self, subscription_type: str) -> str | None:
+        return self._sub_to_warmup.get(subscription_type)
+
+    def set_warmup(self, configs: dict[Any, str]) -> None:
+        for subscription_type, period in configs.items():
+            self._sub_to_warmup[subscription_type] = period
+
+    @property
+    def auto_subscribe(self) -> bool:
+        return self._auto_subscribe
+
+    @auto_subscribe.setter
+    def auto_subscribe(self, value: bool) -> None:
+        self._auto_subscribe = value
+
+    def _get_updated_subs(self) -> list[str]:
+        return list(
+            set(self._pending_stream_unsubscriptions.keys())
+            | set(self._pending_stream_subscriptions.keys())
+            | self._pending_global_subscriptions
+            | self._pending_global_unsubscriptions
+        )
+
+    def _has_operations_to_commit(self) -> bool:
+        return any(
+            (
+                self._pending_stream_unsubscriptions,
+                self._pending_stream_subscriptions,
+                self._pending_global_subscriptions,
+                self._pending_global_unsubscriptions,
+            )
+        )
+
+    def _update_pending_warmups(self, subscription_type: str, instruments: list[Instrument]) -> None:
+        # TODO: refactor pending warmups in a way that would allow to subscribe and then call set_warmup in the same iteration
+        # - ohlc is handled separately
+        if DataType.from_str(subscription_type) != DataType.OHLC:
+            _warmup_period = self._sub_to_warmup.get(subscription_type)
+            if _warmup_period is not None:
+                for instrument in instruments:
+                    self._pending_warmups[(subscription_type, instrument)] = _warmup_period
+
+        # - if base subscription, then we need to fetch historical OHLC data for warmup
+        if subscription_type == self._base_sub:
+            self._pending_warmups.update(
+                {
+                    (sub, instrument): period
+                    for sub, period in self._sub_to_warmup.items()
+                    for instrument in instruments
+                    if DataType.OHLC == sub
+                }
+            )
+
+    def _run_warmup(self) -> None:
+        # - handle warmup for global subscriptions
+        _subscribed_instruments = set(self._data_provider.get_subscribed_instruments())
+        _new_instruments = (
+            set.union(*self._pending_stream_subscriptions.values()) if self._pending_stream_subscriptions else set()
+        )
+
+        for sub in self._pending_global_subscriptions:
+            _warmup_period = self._sub_to_warmup.get(sub)
+            if _warmup_period is None:
+                continue
+            _sub_instruments = self._data_provider.get_subscribed_instruments(sub)
+            _add_instruments = _subscribed_instruments.union(_new_instruments).difference(_sub_instruments)
+            for instr in _add_instruments:
+                self._pending_warmups[(sub, instr)] = _warmup_period
+
+        # TODO: think about appropriate handling of timeouts
+        self._data_provider.warmup(self._pending_warmups.copy())
+        self._pending_warmups.clear()

qubx/core/mixins/trading.py

@@ -0,0 +1,88 @@
+from qubx import logger
+from qubx.core.basics import Instrument, Order, OrderRequest
+from qubx.core.interfaces import IAccountProcessor, IBroker, ITimeProvider, ITradingManager
+
+
+class TradingManager(ITradingManager):
+    _time_provider: ITimeProvider
+    _broker: IBroker
+    _account: IAccountProcessor
+    _strategy_name: str
+
+    _order_id: int | None = None
+
+    def __init__(
+        self, time_provider: ITimeProvider, broker: IBroker, account: IAccountProcessor, strategy_name: str
+    ) -> None:
+        self._time_provider = time_provider
+        self._broker = broker
+        self._account = account
+        self._strategy_name = strategy_name
+
+    def trade(
+        self,
+        instrument: Instrument,
+        amount: float,
+        price: float | None = None,
+        time_in_force="gtc",
+        **options,
+    ) -> Order:
+        # - adjust size
+        size_adj = instrument.round_size_down(abs(amount))
+        if size_adj < instrument.min_size:
+            raise ValueError(f"Attempt to trade size {abs(amount)} less than minimal allowed {instrument.min_size} !")
+
+        side = "buy" if amount > 0 else "sell"
+        type = "market"
+        if price is not None:
+            price = instrument.round_price_down(price) if amount > 0 else instrument.round_price_up(price)
+            type = "limit"
+            if (stp_type := options.get("stop_type")) is not None:
+                type = f"stop_{stp_type}"
+
+        client_id = self._generate_order_client_id(instrument.symbol)
+        logger.debug(
+            f"  [<y>{self.__class__.__name__}</y>(<g>{instrument.symbol}</g>)] :: Sending {type} {side} {size_adj} { ' @ ' + str(price) if price else ''} -> (client_id: <r>{client_id})</r> ..."
+        )
+
+        order = self._broker.send_order(
+            instrument=instrument,
+            order_side=side,
+            order_type=type,
+            amount=size_adj,
+            price=price,
+            time_in_force=time_in_force,
+            client_id=client_id,
+            **options,
+        )
+        return order
+
+    def submit_orders(self, order_requests: list[OrderRequest]) -> list[Order]:
+        raise NotImplementedError("Not implemented yet")
+
+    def set_target_position(
+        self, instrument: Instrument, target: float, price: float | None = None, time_in_force="gtc", **options
+    ) -> Order:
+        raise NotImplementedError("Not implemented yet")
+
+    def close_position(self, instrument: Instrument) -> None:
+        raise NotImplementedError("Not implemented yet")
+
+    def cancel_order(self, order_id: str) -> None:
+        if not order_id:
+            return
+        self._broker.cancel_order(order_id)
+
+    def cancel_orders(self, instrument: Instrument) -> None:
+        for o in self._account.get_orders(instrument).values():
+            self._broker.cancel_order(o.id)
+
+    def _generate_order_client_id(self, symbol: str) -> str:
+        if self._order_id is None:
+            self._order_id = self._time_provider.time().astype("int64") // 100_000_000
+        assert self._order_id is not None
+        self._order_id += 1
+        return "_".join(["qubx", symbol, str(self._order_id)])
+
+    def exchanges(self) -> list[str]:
+        return [self._broker.exchange()]

qubx/core/mixins/universe.py

@@ -0,0 +1,270 @@
+from qubx.core.basics import DataType, Instrument, TargetPosition
+from qubx.core.helpers import CachedMarketDataHolder
+from qubx.core.interfaces import (
+    IAccountProcessor,
+    IBroker,
+    IDataProvider,
+    IPositionGathering,
+    IStrategy,
+    IStrategyContext,
+    ISubscriptionManager,
+    ITimeProvider,
+    ITradingManager,
+    IUniverseManager,
+    RemovalPolicy,
+)
+from qubx.core.loggers import StrategyLogging
+
+
+class UniverseManager(IUniverseManager):
+    _context: IStrategyContext
+    _strategy: IStrategy
+    _broker: IDataProvider
+    _trading_service: IBroker
+    _cache: CachedMarketDataHolder
+    _logging: StrategyLogging
+    _subscription_manager: ISubscriptionManager
+    _trading_manager: ITradingManager
+    _time_provider: ITimeProvider
+    _account: IAccountProcessor
+    _position_gathering: IPositionGathering
+    _removal_queue: dict[Instrument, tuple[RemovalPolicy, bool]]
+
+    def __init__(
+        self,
+        context: IStrategyContext,
+        strategy: IStrategy,
+        broker: IDataProvider,
+        trading_service: IBroker,
+        cache: CachedMarketDataHolder,
+        logging: StrategyLogging,
+        subscription_manager: ISubscriptionManager,
+        trading_manager: ITradingManager,
+        time_provider: ITimeProvider,
+        account: IAccountProcessor,
+        position_gathering: IPositionGathering,
+    ):
+        self._context = context
+        self._strategy = strategy
+        self._broker = broker
+        self._trading_service = trading_service
+        self._cache = cache
+        self._logging = logging
+        self._subscription_manager = subscription_manager
+        self._trading_manager = trading_manager
+        self._time_provider = time_provider
+        self._account = account
+        self._position_gathering = position_gathering
+        self._instruments = []
+        self._removal_queue = {}
+
+    def _has_position(self, instrument: Instrument) -> bool:
+        return (
+            instrument in self._account.positions
+            and abs(self._account.positions[instrument].quantity) > instrument.min_size
+        )
+
+    def set_universe(
+        self,
+        instruments: list[Instrument],
+        skip_callback: bool = False,
+        if_has_position_then: RemovalPolicy = "close",
+    ) -> None:
+        assert if_has_position_then in (
+            "close",
+            "wait_for_close",
+            "wait_for_change",
+        ), "Invalid if_has_position_then policy"
+
+        new_set = set(instruments)
+        prev_set = set(self._instruments)
+
+        # - determine instruments to remove depending on if_has_position_then policy
+        may_be_removed = list(prev_set - new_set)
+
+        # - split instruments into removable and keepable
+        to_remove, to_keep = self._get_what_can_be_removed_or_kept(may_be_removed, skip_callback, if_has_position_then)
+
+        to_add = list(new_set - prev_set)
+        self.__do_add_instruments(to_add)
+        self.__do_remove_instruments(to_remove)
+
+        # - cleanup removal queue
+        self.__cleanup_removal_queue(instruments)
+
+        if not skip_callback and (to_add or to_remove):
+            self._strategy.on_universe_change(self._context, to_add, to_remove)
+
+        self._subscription_manager.commit()  # apply pending changes
+
+        # set new instruments
+        self._instruments.clear()
+        self._instruments.extend(instruments)
+        self._instruments.extend(to_keep)
+
+    def _get_what_can_be_removed_or_kept(
+        self, may_be_removed: list[Instrument], skip_callback: bool, if_has_position_then: RemovalPolicy
+    ) -> tuple[list[Instrument], list[Instrument]]:
+        immediately_close = if_has_position_then == "close"
+        to_remove, to_keep = [], []
+        for instr in may_be_removed:
+            if immediately_close:
+                to_remove.append(instr)
+            else:
+                if self._has_position(instr):
+                    self._removal_queue[instr] = (if_has_position_then, skip_callback)
+                    to_keep.append(instr)
+        return to_remove, to_keep
+
+    def __cleanup_removal_queue(self, instruments: list[Instrument]):
+        for instr in instruments:
+            # - if it's still in the removal queue, remove it
+            if instr in self._removal_queue:
+                self._removal_queue.pop(instr)
+
+    def add_instruments(self, instruments: list[Instrument]):
+        self.__do_add_instruments(instruments)
+        self.__cleanup_removal_queue(instruments)
+        self._strategy.on_universe_change(self._context, instruments, [])
+        self._subscription_manager.commit()
+        self._instruments.extend(instruments)
+
+    def remove_instruments(
+        self,
+        instruments: list[Instrument],
+        if_has_position_then: RemovalPolicy = "close",
+    ):
+        assert if_has_position_then in (
+            "close",
+            "wait_for_close",
+            "wait_for_change",
+        ), "Invalid if_has_position_then policy"
+
+        # - split instruments into removable and keepable
+        to_remove, to_keep = self._get_what_can_be_removed_or_kept(instruments, False, if_has_position_then)
+
+        # - remove ones that can be removed immediately
+        self.__do_remove_instruments(to_remove)
+        self._strategy.on_universe_change(self._context, [], to_remove)
+        self._subscription_manager.commit()
+
+        # - update instruments list
+        self._instruments = list(set(self._instruments) - set(to_remove))
+        self._instruments.extend(to_keep)
+
+    @property
+    def instruments(self) -> list[Instrument]:
+        return self._instruments
+
+    def __do_remove_instruments(self, instruments: list[Instrument]):
+        """
+        Remove symbols from universe. Steps:
+        - [v] cancel all open orders
+        - [v] close all open positions
+        - [v] unsubscribe from market data
+        - [v] remove from data cache
+
+        We are still keeping the symbols in the positions dictionary.
+        """
+        if not instruments:
+            return
+
+        # - preprocess instruments and cancel all open orders
+        for instr in instruments:
+            # - remove instrument from the removal queue if it's there
+            self._removal_queue.pop(instr, None)
+
+            # - cancel all open orders
+            self._trading_manager.cancel_orders(instr)
+
+        # - close all open positions
+        exit_targets = [
+            TargetPosition.zero(self._context, instr.signal(0, group="Universe", comment="Universe change"))
+            for instr in instruments
+            if self._has_position(instr)
+        ]
+        self._position_gathering.alter_positions(self._context, exit_targets)
+
+        # - if still open positions close them manually
+        for instr in instruments:
+            pos = self._account.positions.get(instr)
+            if pos and abs(pos.quantity) > instr.min_size:
+                self._trading_manager.trade(instr, -pos.quantity)
+
+        # - unsubscribe from market data
+        for instr in instruments:
+            self._subscription_manager.unsubscribe(DataType.ALL, instr)
+
+        # - remove from data cache
+        for instr in instruments:
+            self._cache.remove(instr)
+
+    def __do_add_instruments(self, instruments: list[Instrument]) -> None:
+        # - create positions for instruments
+        self._create_and_update_positions(instruments)
+
+        # - initialize ohlcv for new instruments
+        for instr in instruments:
+            self._cache.init_ohlcv(instr)
+
+        # - subscribe to market data
+        self._subscription_manager.subscribe(
+            (
+                DataType.ALL
+                if self._subscription_manager.auto_subscribe
+                else self._subscription_manager.get_base_subscription()
+            ),
+            instruments,
+        )
+
+        # - reinitialize strategy loggers
+        self._logging.initialize(self._time_provider.time(), self._account.positions, self._account.get_balances())
+
+    def _create_and_update_positions(self, instruments: list[Instrument]):
+        for instrument in instruments:
+            _ = self._account.get_position(instrument)
+
+            # - check if we need any aux instrument for calculating pnl ?
+            # TODO: test edge cases for aux symbols (UniverseManager)
+            # aux = lookup.find_aux_instrument_for(instrument, self._account.get_base_currency())
+            # if aux is not None:
+            #     instrument._aux_instrument = aux
+            #     instruments.append(aux)
+            #     _ = self._trading_service.get_position(aux)
+
+    def on_alter_position(self, instrument: Instrument) -> None:
+        """
+        Called when the position of an instrument changes.
+        It can be used for postponed unsubscribed events
+        """
+        # - check if need to remove instrument from the universe
+        if instrument in self._removal_queue:
+            _, skip_callback = self._removal_queue[instrument]
+
+            # - if no position, remove instrument from the universe
+            if not self._has_position(instrument):
+                self.__do_remove_instruments([instrument])
+
+                if not skip_callback:
+                    self._strategy.on_universe_change(self._context, [], [instrument])
+
+                # - commit changes and remove instrument from the universe
+                self._subscription_manager.commit()
+                self._instruments.remove(instrument)
+
+    def is_trading_allowed(self, instrument: Instrument) -> bool:
+        if instrument in self._removal_queue:
+            policy, skip_callback = self._removal_queue[instrument]
+
+            if policy == "wait_for_change":
+                self.__do_remove_instruments([instrument])
+
+                if not skip_callback:
+                    self._strategy.on_universe_change(self._context, [], [instrument])
+
+                # - commit changes and remove instrument from the universe
+                self._subscription_manager.commit()
+                self._instruments.remove(instrument)
+                return False
+
+        return True

qubx/core/series.pxd

@@ -0,0 +1,125 @@
+import numpy as np
+cimport numpy as np
+
+
+cdef np.ndarray nans(int dims)
+
+
+cdef class Indexed:
+    cdef public list values
+    cdef public float max_series_length
+    cdef unsigned short _is_empty
+
+
+cdef class Locator:
+    cdef TimeSeries _series
+
+
+cdef class TimeSeries:
+    cdef public long long timeframe
+    cdef public Indexed times
+    cdef public Indexed values
+    cdef public float max_series_length
+    cdef public Locator loc
+    cdef unsigned short _is_new_item
+    cdef public str name
+    cdef dict indicators        # it's used for indicators caching
+    cdef list calculation_order # calculation order as list: [ (input_id, indicator_obj, indicator_id) ]
+    cdef double _process_every_update
+    cdef double _last_bar_update_value
+    cdef long long _last_bar_update_time
+
+    cdef _update_indicators(TimeSeries self, long long time, object value, short new_item_started)
+
+
+cdef class Indicator(TimeSeries):
+    cdef public TimeSeries series
+    cdef public TimeSeries parent
+
+
+cdef class RollingSum:
+    """
+    Rolling fast summator
+    """
+    cdef unsigned int period
+    cdef np.ndarray __s
+    cdef unsigned int __i
+    cdef double rsum
+    cdef public unsigned short is_init_stage 
+
+    cpdef double update(RollingSum self, double value, short new_item_started)
+
+
+cdef class Bar:
+    cdef public long long time
+    cdef public double open
+    cdef public double high
+    cdef public double low
+    cdef public double close
+    cdef public double volume          # total volume (in quote asset)
+    cdef public double bought_volume   # volume bought (in quote asset) if presented
+
+    cpdef Bar update(Bar self, double price, double volume, double bought_volume=*)
+
+    cpdef dict to_dict(Bar self, unsigned short skip_time=*)
+
+
+cdef class OHLCV(TimeSeries):
+    cdef public TimeSeries open
+    cdef public TimeSeries high
+    cdef public TimeSeries low
+    cdef public TimeSeries close
+    cdef public TimeSeries volume
+    cdef public TimeSeries bvolume
+
+    cpdef short update(OHLCV self, long long time, double price, double volume=*, double bvolume=*)
+
+    cpdef short update_by_bar(OHLCV self, long long time, double open, double high, double low, double close, double vol_incr=*, double b_vol_incr=*)
+
+    cpdef _update_indicators(OHLCV self, long long time, object value, short new_item_started)
+
+    cpdef object append_data(
+        OHLCV self, 
+        np.ndarray times, 
+        np.ndarray opens,
+        np.ndarray highs, 
+        np.ndarray lows,
+        np.ndarray closes, 
+        np.ndarray volumes,
+        np.ndarray bvolumes
+    )
+
+
+cdef class Trade:
+    cdef public long long time
+    cdef public double price
+    cdef public double size
+    cdef public short taker
+    cdef public long long trade_id
+
+
+cdef class Quote:
+    cdef public long long time
+    cdef public double bid
+    cdef public double ask
+    cdef public double bid_size
+    cdef public double ask_size
+
+    cpdef double mid_price(Quote self)
+
+
+cdef class OrderBook:
+    cdef public long long time
+    cdef public double top_bid
+    cdef public double top_ask
+    cdef public double tick_size
+    cdef public np.ndarray bids
+    cdef public np.ndarray asks
+
+    cpdef Quote to_quote(OrderBook self)
+    cpdef double mid_price(OrderBook self)
+
+
+cdef class IndicatorOHLC(Indicator):
+    pass
+