Qubx 0.5.7__cp312-cp312-manylinux_2_39_x86_64.whl

qubx/backtester/ome.py

@@ -0,0 +1,296 @@
+from dataclasses import dataclass
+from operator import neg
+
+import numpy as np
+from sortedcontainers import SortedDict
+
+from qubx import logger
+from qubx.core.basics import (
+    OPTION_FILL_AT_SIGNAL_PRICE,
+    Deal,
+    Instrument,
+    ITimeProvider,
+    Order,
+    OrderSide,
+    OrderType,
+    TransactionCostsCalculator,
+    dt_64,
+)
+from qubx.core.exceptions import (
+    ExchangeError,
+    InvalidOrder,
+)
+from qubx.core.series import Quote
+
+
+@dataclass
+class OmeReport:
+    timestamp: dt_64
+    order: Order
+    exec: Deal | None
+
+
+class OrdersManagementEngine:
+    instrument: Instrument
+    time_service: ITimeProvider
+    active_orders: dict[str, Order]
+    stop_orders: dict[str, Order]
+    asks: SortedDict[float, list[str]]
+    bids: SortedDict[float, list[str]]
+    bbo: Quote | None  # current best bid/ask order book (simplest impl)
+    __order_id: int
+    __trade_id: int
+    _fill_stops_at_price: bool
+
+    def __init__(
+        self,
+        instrument: Instrument,
+        time_provider: ITimeProvider,
+        tcc: TransactionCostsCalculator,
+        fill_stop_order_at_price: bool = False,  # emulate stop orders execution at order's exact limit price
+        debug: bool = True,
+    ) -> None:
+        self.instrument = instrument
+        self.time_service = time_provider
+        self.tcc = tcc
+        self.asks = SortedDict()
+        self.bids = SortedDict(neg)
+        self.active_orders = dict()
+        self.stop_orders = dict()
+        self.bbo = None
+        self.__order_id = 100000
+        self.__trade_id = 100000
+        self._fill_stops_at_price = fill_stop_order_at_price
+        if not debug:
+            self._dbg = lambda message, **kwargs: None
+
+    def _generate_order_id(self) -> str:
+        self.__order_id += 1
+        return "SIM-ORDER-" + self.instrument.symbol + "-" + str(self.__order_id)
+
+    def _generate_trade_id(self) -> str:
+        self.__trade_id += 1
+        return "SIM-EXEC-" + self.instrument.symbol + "-" + str(self.__trade_id)
+
+    def get_quote(self) -> Quote:
+        return self.bbo
+
+    def get_open_orders(self) -> list[Order]:
+        return list(self.active_orders.values()) + list(self.stop_orders.values())
+
+    def update_bbo(self, quote: Quote) -> list[OmeReport]:
+        timestamp = self.time_service.time()
+        rep = []
+
+        if self.bbo is not None:
+            if quote.bid >= self.bbo.ask:
+                for level in self.asks.irange(0, quote.bid):
+                    for order_id in self.asks[level]:
+                        order = self.active_orders.pop(order_id)
+                        rep.append(self._execute_order(timestamp, order.price, order, False))
+                    self.asks.pop(level)
+
+            if quote.ask <= self.bbo.bid:
+                for level in self.bids.irange(np.inf, quote.ask):
+                    for order_id in self.bids[level]:
+                        order = self.active_orders.pop(order_id)
+                        rep.append(self._execute_order(timestamp, order.price, order, False))
+                    self.bids.pop(level)
+
+            # - processing stop orders
+            for soid in list(self.stop_orders.keys()):
+                so = self.stop_orders[soid]
+                _emulate_price_exec = self._fill_stops_at_price or so.options.get(OPTION_FILL_AT_SIGNAL_PRICE, False)
+
+                if so.side == "BUY" and quote.ask >= so.price:
+                    _exec_price = quote.ask if not _emulate_price_exec else so.price
+                    self.stop_orders.pop(soid)
+                    rep.append(self._execute_order(timestamp, _exec_price, so, True))
+                elif so.side == "SELL" and quote.bid <= so.price:
+                    _exec_price = quote.bid if not _emulate_price_exec else so.price
+                    self.stop_orders.pop(soid)
+                    rep.append(self._execute_order(timestamp, _exec_price, so, True))
+
+        self.bbo = quote
+        return rep
+
+    def place_order(
+        self,
+        order_side: OrderSide,
+        order_type: OrderType,
+        amount: float,
+        price: float | None = None,
+        client_id: str | None = None,
+        time_in_force: str = "gtc",
+        **options,
+    ) -> OmeReport:
+        if self.bbo is None:
+            raise ExchangeError(f"Simulator is not ready for order management - no quote for {self.instrument.symbol}")
+
+        # - validate order parameters
+        self._validate_order(order_side, order_type, amount, price, time_in_force)
+
+        timestamp = self.time_service.time()
+        order = Order(
+            self._generate_order_id(),
+            order_type,
+            self.instrument,
+            timestamp,
+            amount,
+            price if price is not None else 0,
+            order_side,
+            "NEW",
+            time_in_force,
+            client_id,
+            options=options,
+        )
+
+        return self._process_order(timestamp, order)
+
+    def _dbg(self, message, **kwargs) -> None:
+        logger.debug(f"    [<y>OME</y>(<g>{self.instrument}</g>)] :: {message}", **kwargs)
+
+    def _process_order(self, timestamp: dt_64, order: Order) -> OmeReport:
+        if order.status in ["CLOSED", "CANCELED"]:
+            raise InvalidOrder(f"Order {order.id} is already closed or canceled.")
+
+        buy_side = order.side == "BUY"
+        c_ask = self.bbo.ask
+        c_bid = self.bbo.bid
+
+        # - check if order can be "executed" immediately
+        exec_price = None
+        _need_update_book = False
+
+        if order.type == "MARKET":
+            exec_price = c_ask if buy_side else c_bid
+
+        elif order.type == "LIMIT":
+            _need_update_book = True
+            if (buy_side and order.price >= c_ask) or (not buy_side and order.price <= c_bid):
+                exec_price = c_ask if buy_side else c_bid
+
+        elif order.type == "STOP_MARKET":
+            # - it processes stop orders separately without adding to orderbook (as on real exchanges)
+            order.status = "OPEN"
+            self.stop_orders[order.id] = order
+
+        elif order.type == "STOP_LIMIT":
+            # TODO: (OME) check trigger conditions in options etc
+            raise NotImplementedError("'STOP_LIMIT' order is not supported in Qubx simulator yet !")
+
+        # - if order must be "executed" immediately
+        if exec_price is not None:
+            return self._execute_order(timestamp, exec_price, order, True)
+
+        # - processing limit orders
+        if _need_update_book:
+            if buy_side:
+                self.bids.setdefault(order.price, list()).append(order.id)
+            else:
+                self.asks.setdefault(order.price, list()).append(order.id)
+
+            order.status = "OPEN"
+            self.active_orders[order.id] = order
+
+        self._dbg(f"registered {order.id} {order.type} {order.side} {order.quantity} {order.price}")
+        return OmeReport(timestamp, order, None)
+
+    def _execute_order(self, timestamp: dt_64, exec_price: float, order: Order, taker: bool) -> OmeReport:
+        order.status = "CLOSED"
+        self._dbg(f"<red>{order.id}</red> {order.type} {order.side} {order.quantity} executed at {exec_price}")
+        return OmeReport(
+            timestamp,
+            order,
+            Deal(
+                id=self._generate_trade_id(),
+                order_id=order.id,
+                time=timestamp,
+                amount=order.quantity if order.side == "BUY" else -order.quantity,
+                price=exec_price,
+                aggressive=taker,
+                fee_amount=self.tcc.get_execution_fees(
+                    instrument=self.instrument, exec_price=exec_price, amount=order.quantity, crossed_market=taker
+                ),
+                fee_currency=self.instrument.quote,
+            ),
+        )
+
+    def _validate_order(
+        self, order_side: str, order_type: str, amount: float, price: float | None, time_in_force: str
+    ) -> None:
+        if order_side.upper() not in ["BUY", "SELL"]:
+            raise InvalidOrder("Invalid order side. Only BUY or SELL is allowed.")
+
+        _ot = order_type.upper()
+        if _ot not in ["LIMIT", "MARKET", "STOP_MARKET", "STOP_LIMIT"]:
+            raise InvalidOrder("Invalid order type. Only LIMIT, MARKET, STOP_MARKET, STOP_LIMIT are supported.")
+
+        if amount <= 0:
+            raise InvalidOrder("Invalid order amount. Amount must be positive.")
+
+        if (_ot == "LIMIT" or _ot.startswith("STOP")) and (price is None or price <= 0):
+            raise InvalidOrder("Invalid order price. Price must be positively defined for LIMIT or STOP orders.")
+
+        if time_in_force.upper() not in ["GTC", "IOC"]:
+            raise InvalidOrder("Invalid time in force. Only GTC or IOC is supported for now.")
+
+        if _ot.startswith("STOP"):
+            assert price is not None
+            c_ask, c_bid = self.bbo.ask, self.bbo.bid
+            if (order_side == "BUY" and c_ask >= price) or (order_side == "SELL" and c_bid <= price):
+                raise ExchangeError(
+                    f"Stop price would trigger immediately: STOP_MARKET {order_side} {amount} of {self.instrument.symbol} at {price} | market: {c_ask} / {c_bid}"
+                )
+
+    def cancel_order(self, order_id: str) -> OmeReport:
+        # - check limit orders
+        if order_id in self.active_orders:
+            order = self.active_orders.pop(order_id)
+            if order.side == "BUY":
+                oids = self.bids[order.price]
+                oids.remove(order_id)
+                if not oids:
+                    self.bids.pop(order.price)
+            else:
+                oids = self.asks[order.price]
+                oids.remove(order_id)
+                if not oids:
+                    self.asks.pop(order.price)
+        # - check stop orders
+        elif order_id in self.stop_orders:
+            order = self.stop_orders.pop(order_id)
+        # - wrong order_id
+        else:
+            raise InvalidOrder(f"Order {order_id} not found for {self.instrument.symbol}")
+
+        order.status = "CANCELED"
+        self._dbg(f"{order.id} {order.type} {order.side} {order.quantity} canceled")
+        return OmeReport(self.time_service.time(), order, None)
+
+    def __str__(self) -> str:
+        _a, _b = True, True
+
+        timestamp = self.time_service.time()
+        _s = f"= = ({np.datetime64(timestamp, 'ns')}) = =\n"
+        for k, v in reversed(self.asks.items()):
+            _sizes = ",".join([f"{self.active_orders[o].quantity}" for o in v])
+            _s += f"  {k} : [{_sizes}]\n"
+            if k == self.bbo.ask:
+                _a = False
+
+        if _a:
+            _s += f"  {self.bbo.ask} : \n"
+        _s += "- - - - - - - - - - - - - - - - - - - -\n"
+
+        _s1 = ""
+        for k, v in self.bids.items():
+            _sizes = ",".join([f"{self.active_orders[o].quantity}" for o in v])
+            _s1 += f"  {k} : [{_sizes}]\n"
+            if k == self.bbo.bid:
+                _b = False
+        _s1 += "= = = = = = = = = = = = = = = = = = = =\n"
+
+        _s1 = f"  {self.bbo.bid} : \n" + _s1 if _b else _s1
+
+        return _s + _s1

qubx/backtester/optimization.py

@@ -0,0 +1,201 @@
+import re
+from itertools import product
+from types import FunctionType
+from typing import Any, Callable, Type
+
+import numpy as np
+
+from qubx.utils.misc import generate_name
+
+
+def _wrap_single_list(param_grid: list | dict) -> dict[str, Any] | list:
+    """
+    Wraps all non list values as single
+    :param param_grid:
+    :return:
+    """
+    as_list = lambda x: x if isinstance(x, (tuple, list, dict, np.ndarray)) else [x]  # noqa: E731
+    if isinstance(param_grid, list):
+        return [_wrap_single_list(ps) for ps in param_grid]
+    return {k: as_list(v) for k, v in param_grid.items()}
+
+
+def permutate_params(
+    parameters: dict[str, list | tuple | Any],
+    conditions: FunctionType | list | tuple | None = None,
+    wrap_as_list=False,
+) -> list[dict]:
+    """
+    Generate list of all permutations for given parameters and theirs possible values
+
+    Example:
+
+    >>> def foo(par1, par2):
+    >>>     print(par1)
+    >>>     print(par2)
+    >>>
+    >>> # permutate all values and call function for every permutation
+    >>> [foo(**z) for z in permutate_params({
+    >>>                                       'par1' : [1,2,3],
+    >>>                                       'par2' : [True, False]
+    >>>                                     }, conditions=lambda par1, par2: par1<=2 and par2==True)]
+
+    1
+    True
+    2
+    True
+
+    :param conditions: list of filtering functions
+    :param parameters: dictionary
+    :param wrap_as_list: if True (default) it wraps all non list values as single lists (required for sklearn)
+    :return: list of permutations
+    """
+    if conditions is None:
+        conditions = []
+    elif isinstance(conditions, FunctionType):
+        conditions = [conditions]
+    elif isinstance(conditions, (tuple, list)):
+        if not all([isinstance(e, FunctionType) for e in conditions]):
+            raise ValueError("every condition must be a function")
+    else:
+        raise ValueError("conditions must be of type of function, list or tuple")
+
+    args = []
+    vals = []
+    for k, v in parameters.items():
+        args.append(k)
+        # vals.append([v] if not isinstance(v, (list, tuple)) else list(v) if isinstance(v, range) else v)
+        match v:
+            case list() | tuple():
+                vals.append(v)
+            case range():
+                vals.append(list(v))
+            case str():
+                vals.append([v])
+            case _:
+                vals.append([v])
+        # vals.append(v if isinstance(v, (List, Tuple)) else list(v) if isinstance(v, range) else [v])
+    d = [dict(zip(args, p)) for p in product(*vals)]
+    result = []
+    for params_set in d:
+        conditions_met = True
+        for cond_func in conditions:
+            func_param_args = cond_func.__code__.co_varnames
+            func_param_values = [params_set[arg] for arg in func_param_args]
+            if not cond_func(*func_param_values):
+                conditions_met = False
+                break
+        if conditions_met:
+            result.append(params_set)
+
+    # if we need to follow sklearn rules we should wrap every noniterable as list
+    return _wrap_single_list(result) if wrap_as_list else result
+
+
+def dicts_product(d1: dict, d2: dict) -> dict:
+    """
+    Product of two dictionaries.
+
+    Example:
+    -------
+
+    dicts_product({
+        'A': 1,
+        'B': 2,
+    }, {
+        'C': 3,
+        'D': 4,
+    })
+
+    Output:
+    ------
+    {
+        'A + C': [1, 3],
+        'A + D': [1, 4],
+        'B + C': [2, 3],
+        'B + D': [2, 4]
+    }
+
+    """
+
+    def flatten(lst):
+        return [item for sublist in lst for item in (sublist if isinstance(sublist, list) else [sublist])]
+
+    return {(a + " + " + b): flatten([d1[a], d2[b]]) for a, b in product(d1.keys(), d2.keys())}
+
+
+class _dict(dict):
+    def __add__(self, other: dict) -> dict:
+        return _dict(dicts_product(self, other))
+
+
+def variate(clz: Type[Any] | list[Type[Any]], *args, conditions=None, **kwargs) -> _dict:
+    """
+    Make variations of parameters for simulations (micro optimizer)
+
+    Example:
+
+    >>>    class MomentumStrategy_Ex1_test:
+    >>>       def __init__(self, p1, lookback_period=10, filter_type='sma', skip_entries_flag=False):
+    >>>            self.p1, self.lookback_period, self.filter_type, self.skip_entries_flag = p1, lookback_period, filter_type, skip_entries_flag
+    >>>
+    >>>        def __repr__(self):
+    >>>            return self.__class__.__name__ + f"({self.p1},{self.lookback_period},{self.filter_type},{self.skip_entries_flag})"
+    >>>
+    >>>    variate(MomentumStrategy_Ex1_test, 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False])
+
+    Output:
+    >>>    {
+    >>>        'MSE1t_(lp=1,ft=ema,sef=True)':  MomentumStrategy_Ex1_test(10,1,ema,True),
+    >>>        'MSE1t_(lp=1,ft=ema,sef=False)': MomentumStrategy_Ex1_test(10,1,ema,False),
+    >>>        'MSE1t_(lp=1,ft=sma,sef=True)':  MomentumStrategy_Ex1_test(10,1,sma,True),
+    >>>        'MSE1t_(lp=1,ft=sma,sef=False)': MomentumStrategy_Ex1_test(10,1,sma,False),
+    >>>        'MSE1t_(lp=2,ft=ema,sef=True)':  MomentumStrategy_Ex1_test(10,2,ema,True),
+    >>>        'MSE1t_(lp=2,ft=ema,sef=False)': MomentumStrategy_Ex1_test(10,2,ema,False),
+    >>>        'MSE1t_(lp=2,ft=sma,sef=True)':  MomentumStrategy_Ex1_test(10,2,sma,True),
+    >>>        'MSE1t_(lp=2,ft=sma,sef=False)': MomentumStrategy_Ex1_test(10,2,sma,False),
+    >>>        'MSE1t_(lp=3,ft=ema,sef=True)':  MomentumStrategy_Ex1_test(10,3,ema,True),
+    >>>        'MSE1t_(lp=3,ft=ema,sef=False)': MomentumStrategy_Ex1_test(10,3,ema,False),
+    >>>        'MSE1t_(lp=3,ft=sma,sef=True)':  MomentumStrategy_Ex1_test(10,3,sma,True),
+    >>>        'MSE1t_(lp=3,ft=sma,sef=False)': MomentumStrategy_Ex1_test(10,3,sma,False)
+    >>>    }
+
+    and using in simuation:
+
+    >>>    r = simulate(
+    >>>             variate(MomentumStrategy_Ex1_test, 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False]),
+    >>>             data, capital, ["BINANCE.UM:BTCUSDT"], dict(type="ohlc", timeframe="5Min", nback=0), "5Min -1Sec", "vip0_usdt", "2024-01-01", "2024-01-02"
+    >>>    )
+
+    Also it's possible to pass a class with tracker:
+    >>>    variate([MomentumStrategy_Ex1_test, AtrTracker(2, 1)], 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False])
+    """
+
+    def _cmprss(xs: str):
+        return "".join([x[0] for x in re.split(r"((?<!-)(?=[A-Z]))|_|(\d)", xs) if x])
+
+    if isinstance(clz, (type, Callable)):
+        sfx = _cmprss(clz.__name__)
+        _mk = lambda k, *args, **kwargs: k(*args, **kwargs)  # noqa: E731
+    elif isinstance(clz, (list, tuple)) and clz and isinstance(clz[0], type):
+        sfx = _cmprss(clz[0].__name__)
+        _mk = lambda k, *args, **kwargs: [k[0](*args, **kwargs), *k[1:]]  # noqa: E731
+    else:
+        raise ValueError(
+            "Can't recognize data for variating: must be either a class type or a list where first element is class type"
+        )
+
+    def _v_to_str(x: Any) -> str:
+        if isinstance(x, (list, tuple, dict, set, np.ndarray)) and len(xs := str(x)) > 15:
+            return "[" + generate_name(xs, 8).lower() + "]"
+        return str(x)
+
+    to_excl = [s for s, v in kwargs.items() if not isinstance(v, (list, set, tuple, range))]
+    dic2str = lambda ds: [_cmprss(k) + "=" + _v_to_str(v) for k, v in ds.items() if k not in to_excl]  # noqa: E731
+
+    return _dict(
+        {
+            f"{sfx}_({','.join(dic2str(z))})": _mk(clz, *args, **z)
+            for z in permutate_params(kwargs, conditions=conditions)
+        }
+    )