Qubx 0.1.90__cp311-cp311-manylinux_2_35_x86_64.whl → 0.2.4__cp311-cp311-manylinux_2_35_x86_64.whl

qubx/__init__.py

@@ -1,3 +1,4 @@
+from typing import Callable
 from qubx.utils import set_mpl_theme, runtime_env
 from qubx.utils.misc import install_pyx_recompiler_for_dev
 
@@ -10,20 +11,13 @@ def formatter(record):
     end = record["extra"].get("end", "\n")
     fmt = "<lvl>{message}</lvl>%s" % end
     if record["level"].name in {"WARNING", "SNAKY"}:
-        fmt = (
-            "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - %s" % fmt
-        )
+        fmt = "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - %s" % fmt
 
-    prefix = (
-        "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> [ <level>%s</level> ] "
-        % record["level"].icon
-    )
+    prefix = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> [ <level>%s</level> ] " % record["level"].icon
 
     if record["exception"] is not None:
         # stackprinter.set_excepthook(style='darkbg2')
-        record["extra"]["stack"] = stackprinter.format(
-            record["exception"], style="darkbg"
-        )
+        record["extra"]["stack"] = stackprinter.format(record["exception"], style="darkbg3")
         fmt += "\n{extra[stack]}\n"
 
     if record["level"].name in {"TEXT"}:
@@ -44,7 +38,7 @@ class QubxLogConfig:
         QubxLogConfig.setup_logger(level)
 
     @staticmethod
-    def setup_logger(level: str | None = None):
+    def setup_logger(level: str | None = None, custom_formatter: Callable | None = None):
         global logger
         config = {
             "handlers": [
@@ -55,7 +49,7 @@ class QubxLogConfig:
         logger.configure(**config)
         logger.remove(None)
         level = level or QubxLogConfig.get_log_level()
-        logger.add(sys.stdout, format=formatter, colorize=True, level=level)
+        logger.add(sys.stdout, format=custom_formatter or formatter, colorize=True, level=level, enqueue=True)
         logger = logger.opt(colors=True)
 
 

qubx/backtester/__init__.py

@@ -0,0 +1,2 @@
+from .simulator import simulate
+from .optimization import variate

qubx/backtester/ome.py

@@ -0,0 +1,241 @@
+from typing import List, Dict
+from dataclasses import dataclass
+from operator import neg
+
+import numpy as np
+from sortedcontainers import SortedDict
+
+from qubx import logger
+from qubx.core.basics import Deal, Instrument, Order, Position, Signal, TransactionCostsCalculator, dt_64, ITimeProvider
+from qubx.core.series import Quote, Trade
+from qubx.core.exceptions import (
+    ExchangeError,
+    InvalidOrder,
+)
+
+
+@dataclass
+class OmeReport:
+    timestamp: dt_64
+    order: Order
+    exec: Deal | None
+
+
+class OrdersManagementEngine:
+    instrument: Instrument
+    time_service: ITimeProvider
+    active_orders: Dict[str, Order]
+    asks: SortedDict[float, List[str]]
+    bids: SortedDict[float, List[str]]
+    bbo: Quote | None  # current best bid/ask order book (simplest impl)
+    __order_id: int
+    __trade_id: int
+
+    def __init__(
+        self, instrument: Instrument, time_provider: ITimeProvider, tcc: TransactionCostsCalculator, debug: bool = True
+    ) -> None:
+        self.instrument = instrument
+        self.time_service = time_provider
+        self.tcc = tcc
+        self.asks = SortedDict()
+        self.bids = SortedDict(neg)
+        self.active_orders = dict()
+        self.bbo = None
+        self.__order_id = 100000
+        self.__trade_id = 100000
+        if not debug:
+            self._dbg = lambda message, **kwargs: None
+
+    def _generate_order_id(self) -> str:
+        self.__order_id += 1
+        return "SIM-ORDER-" + self.instrument.symbol + "-" + str(self.__order_id)
+
+    def _generate_trade_id(self) -> str:
+        self.__trade_id += 1
+        return "SIM-EXEC-" + self.instrument.symbol + "-" + str(self.__trade_id)
+
+    def get_quote(self) -> Quote:
+        return self.bbo
+
+    def get_open_orders(self) -> List[Order]:
+        return list(self.active_orders.values())
+
+    def update_bbo(self, quote: Quote) -> List[OmeReport]:
+        timestamp = self.time_service.time()
+        rep = []
+
+        if self.bbo is not None:
+            if quote.bid >= self.bbo.ask:
+                for level in self.asks.irange(0, quote.bid):
+                    for order_id in self.asks[level]:
+                        order = self.active_orders.pop(order_id)
+                        rep.append(self._execute_order(timestamp, order.price, order, False))
+                    self.asks.pop(level)
+
+            if quote.ask <= self.bbo.bid:
+                for level in self.bids.irange(np.inf, quote.ask):
+                    for order_id in self.bids[level]:
+                        order = self.active_orders.pop(order_id)
+                        rep.append(self._execute_order(timestamp, order.price, order, False))
+                    self.bids.pop(level)
+
+        self.bbo = quote
+        return rep
+
+    def place_order(
+        self,
+        order_side: str,
+        order_type: str,
+        amount: float,
+        price: float | None = None,
+        client_id: str | None = None,
+        time_in_force: str = "gtc",
+        fill_at_price: bool = False,
+    ) -> OmeReport:
+
+        if self.bbo is None:
+            raise ExchangeError(
+                f"Simulator is not ready for order management - no any quote for {self.instrument.symbol}"
+            )
+
+        # - validate order parameters
+        self._validate_order(order_side, order_type, amount, price, time_in_force)
+
+        timestamp = self.time_service.time()
+        order = Order(
+            self._generate_order_id(),
+            order_type,
+            self.instrument.symbol,
+            timestamp,
+            amount,
+            price if price is not None else 0,
+            order_side,
+            "NEW",
+            time_in_force,
+            client_id,
+        )
+
+        return self._process_order(timestamp, order, fill_at_price=fill_at_price)
+
+    def _dbg(self, message, **kwargs) -> None:
+        logger.debug(f"[OMS] {self.instrument.symbol} - {message}", **kwargs)
+
+    def _process_order(self, timestamp: dt_64, order: Order, fill_at_price: bool = False) -> OmeReport:
+        if order.status in ["CLOSED", "CANCELED"]:
+            raise InvalidOrder(f"Order {order.id} is already closed or canceled.")
+
+        buy_side = order.side == "BUY"
+        c_ask = self.bbo.ask
+        c_bid = self.bbo.bid
+
+        # - check if order can be "executed" immediately
+        exec_price = None
+        if fill_at_price and order.price:
+            exec_price = order.price
+
+        elif order.type == "MARKET":
+            exec_price = c_ask if buy_side else c_bid
+
+        elif order.type == "LIMIT":
+            if (buy_side and order.price >= c_ask) or (not buy_side and order.price <= c_bid):
+                exec_price = c_ask if buy_side else c_bid
+
+        # - if order must be "executed" immediately
+        if exec_price is not None:
+            return self._execute_order(timestamp, exec_price, order, True)
+
+        # - processing limit orders
+        if buy_side:
+            self.bids.setdefault(order.price, list()).append(order.id)
+        else:
+            self.asks.setdefault(order.price, list()).append(order.id)
+        order.status = "OPEN"
+        self._dbg(f"registered {order.id} {order.type} {order.side} {order.quantity} {order.price}")
+        self.active_orders[order.id] = order
+        return OmeReport(timestamp, order, None)
+
+    def _execute_order(self, timestamp: dt_64, exec_price: float, order: Order, taker: bool) -> OmeReport:
+        order.status = "CLOSED"
+        self._dbg(f"{order.id} {order.type} {order.side} {order.quantity} executed at {exec_price}")
+        return OmeReport(
+            timestamp,
+            order,
+            Deal(
+                id=self._generate_trade_id(),
+                order_id=order.id,
+                time=timestamp,
+                amount=order.quantity if order.side == "BUY" else -order.quantity,
+                price=exec_price,
+                aggressive=taker,
+                fee_amount=self.tcc.get_execution_fees(
+                    instrument=self.instrument, exec_price=exec_price, amount=order.quantity, crossed_market=taker
+                ),
+                fee_currency=self.instrument.quote,
+            ),
+        )
+
+    def _validate_order(
+        self, order_side: str, order_type: str, amount: float, price: float | None, time_in_force: str
+    ) -> None:
+        if order_side.upper() not in ["BUY", "SELL"]:
+            raise InvalidOrder("Invalid order side. Only BUY or SELL is allowed.")
+
+        if order_type.upper() not in ["LIMIT", "MARKET"]:
+            raise InvalidOrder("Invalid order type. Only LIMIT or MARKET is supported.")
+
+        if amount <= 0:
+            raise InvalidOrder("Invalid order amount. Amount must be positive.")
+
+        if order_type.upper() == "LIMIT" and (price is None or price <= 0):
+            raise InvalidOrder("Invalid order price. Price must be positively defined for LIMIT orders.")
+
+        if time_in_force.upper() not in ["GTC", "IOC"]:
+            raise InvalidOrder("Invalid time in force. Only GTC or IOC is supported for now.")
+
+    def cancel_order(self, order_id: str) -> OmeReport:
+        if order_id not in self.active_orders:
+            raise InvalidOrder(f"Order {order_id} not found for {self.instrument.symbol}")
+
+        timestamp = self.time_service.time()
+        order = self.active_orders.pop(order_id)
+        if order.side == "BUY":
+            oids = self.bids[order.price]
+            oids.remove(order_id)
+            if not oids:
+                self.bids.pop(order.price)
+        else:
+            oids = self.asks[order.price]
+            oids.remove(order_id)
+            if not oids:
+                self.asks.pop(order.price)
+
+        order.status = "CANCELED"
+        self._dbg(f"{order.id} {order.type} {order.side} {order.quantity} canceled")
+        return OmeReport(timestamp, order, None)
+
+    def __str__(self) -> str:
+        _a, _b = True, True
+
+        timestamp = self.time_service.time()
+        _s = f"= = ({np.datetime64(timestamp, 'ns')}) = =\n"
+        for k, v in reversed(self.asks.items()):
+            _sizes = ",".join([f"{self.active_orders[o].quantity}" for o in v])
+            _s += f"  {k} : [{ _sizes }]\n"
+            if k == self.bbo.ask:
+                _a = False
+
+        if _a:
+            _s += f"  {self.bbo.ask} : \n"
+        _s += "- - - - - - - - - - - - - - - - - - - -\n"
+
+        _s1 = ""
+        for k, v in self.bids.items():
+            _sizes = ",".join([f"{self.active_orders[o].quantity}" for o in v])
+            _s1 += f"  {k} : [{ _sizes }]\n"
+            if k == self.bbo.bid:
+                _b = False
+        _s1 += "= = = = = = = = = = = = = = = = = = = =\n"
+
+        _s1 = f"  {self.bbo.bid} : \n" + _s1 if _b else _s1
+
+        return _s + _s1

qubx/backtester/optimization.py

@@ -0,0 +1,155 @@
+from typing import Any, Dict, List, Sequence, Tuple, Type
+import numpy as np
+import re
+
+from types import FunctionType
+from itertools import product
+
+
+def _wrap_single_list(param_grid: List | Dict) -> Dict[str, Any] | List:
+    """
+    Wraps all non list values as single
+    :param param_grid:
+    :return:
+    """
+    as_list = lambda x: x if isinstance(x, (tuple, list, dict, np.ndarray)) else [x]
+    if isinstance(param_grid, list):
+        return [_wrap_single_list(ps) for ps in param_grid]
+    return {k: as_list(v) for k, v in param_grid.items()}
+
+
+def permutate_params(
+    parameters: Dict[str, List | Tuple | Any],
+    conditions: FunctionType | List | Tuple | None = None,
+    wrap_as_list=False,
+) -> List[Dict]:
+    """
+    Generate list of all permutations for given parameters and theirs possible values
+
+    Example:
+
+    >>> def foo(par1, par2):
+    >>>     print(par1)
+    >>>     print(par2)
+    >>>
+    >>> # permutate all values and call function for every permutation
+    >>> [foo(**z) for z in permutate_params({
+    >>>                                       'par1' : [1,2,3],
+    >>>                                       'par2' : [True, False]
+    >>>                                     }, conditions=lambda par1, par2: par1<=2 and par2==True)]
+
+    1
+    True
+    2
+    True
+
+    :param conditions: list of filtering functions
+    :param parameters: dictionary
+    :param wrap_as_list: if True (default) it wraps all non list values as single lists (required for sklearn)
+    :return: list of permutations
+    """
+    if conditions is None:
+        conditions = []
+    elif isinstance(conditions, FunctionType):
+        conditions = [conditions]
+    elif isinstance(conditions, (tuple, list)):
+        if not all([isinstance(e, FunctionType) for e in conditions]):
+            raise ValueError("every condition must be a function")
+    else:
+        raise ValueError("conditions must be of type of function, list or tuple")
+
+    args = []
+    vals = []
+    for k, v in parameters.items():
+        args.append(k)
+        # vals.append([v] if not isinstance(v, (list, tuple)) else list(v) if isinstance(v, range) else v)
+        match v:
+            case list() | tuple():
+                vals.append(v)
+            case range():
+                vals.append(list(v))
+            case str():
+                vals.append([v])
+            case _:
+                vals.append([v])
+        # vals.append(v if isinstance(v, (List, Tuple)) else list(v) if isinstance(v, range) else [v])
+    d = [dict(zip(args, p)) for p in product(*vals)]
+    result = []
+    for params_set in d:
+        conditions_met = True
+        for cond_func in conditions:
+            func_param_args = cond_func.__code__.co_varnames
+            func_param_values = [params_set[arg] for arg in func_param_args]
+            if not cond_func(*func_param_values):
+                conditions_met = False
+                break
+        if conditions_met:
+            result.append(params_set)
+
+    # if we need to follow sklearn rules we should wrap every noniterable as list
+    return _wrap_single_list(result) if wrap_as_list else result
+
+
+def variate(clz: Type[Any] | List[Type[Any]], *args, conditions=None, **kwargs) -> Dict[str, Any]:
+    """
+    Make variations of parameters for simulations (micro optimizer)
+
+    Example:
+
+    >>>    class MomentumStrategy_Ex1_test:
+    >>>       def __init__(self, p1, lookback_period=10, filter_type='sma', skip_entries_flag=False):
+    >>>            self.p1, self.lookback_period, self.filter_type, self.skip_entries_flag = p1, lookback_period, filter_type, skip_entries_flag
+    >>>
+    >>>        def __repr__(self):
+    >>>            return self.__class__.__name__ + f"({self.p1},{self.lookback_period},{self.filter_type},{self.skip_entries_flag})"
+    >>>
+    >>>    variate(MomentumStrategy_Ex1_test, 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False])
+
+    Output:
+    >>>    {
+    >>>        'MSE1t_(lp=1,ft=ema,sef=True)':  MomentumStrategy_Ex1_test(10,1,ema,True),
+    >>>        'MSE1t_(lp=1,ft=ema,sef=False)': MomentumStrategy_Ex1_test(10,1,ema,False),
+    >>>        'MSE1t_(lp=1,ft=sma,sef=True)':  MomentumStrategy_Ex1_test(10,1,sma,True),
+    >>>        'MSE1t_(lp=1,ft=sma,sef=False)': MomentumStrategy_Ex1_test(10,1,sma,False),
+    >>>        'MSE1t_(lp=2,ft=ema,sef=True)':  MomentumStrategy_Ex1_test(10,2,ema,True),
+    >>>        'MSE1t_(lp=2,ft=ema,sef=False)': MomentumStrategy_Ex1_test(10,2,ema,False),
+    >>>        'MSE1t_(lp=2,ft=sma,sef=True)':  MomentumStrategy_Ex1_test(10,2,sma,True),
+    >>>        'MSE1t_(lp=2,ft=sma,sef=False)': MomentumStrategy_Ex1_test(10,2,sma,False),
+    >>>        'MSE1t_(lp=3,ft=ema,sef=True)':  MomentumStrategy_Ex1_test(10,3,ema,True),
+    >>>        'MSE1t_(lp=3,ft=ema,sef=False)': MomentumStrategy_Ex1_test(10,3,ema,False),
+    >>>        'MSE1t_(lp=3,ft=sma,sef=True)':  MomentumStrategy_Ex1_test(10,3,sma,True),
+    >>>        'MSE1t_(lp=3,ft=sma,sef=False)': MomentumStrategy_Ex1_test(10,3,sma,False)
+    >>>    }
+
+    and using in simuation:
+
+    >>>    r = simulate(
+    >>>             variate(MomentumStrategy_Ex1_test, 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False]),
+    >>>             data, capital, ["BINANCE.UM:BTCUSDT"], dict(type="ohlc", timeframe="5Min", nback=0), "5Min -1Sec", "vip0_usdt", "2024-01-01", "2024-01-02"
+    >>>    )
+
+    Also it's possible to pass a class with tracker:
+    >>>    variate([MomentumStrategy_Ex1_test, AtrTracker(2, 1)], 10, lookback_period=[1,2,3], filter_type=['ema', 'sma'], skip_entries_flag=[True, False])
+    """
+
+    def _cmprss(xs: str):
+        return "".join([x[0] for x in re.split("((?<!-)(?=[A-Z]))|_|(\d)", xs) if x])
+
+    if isinstance(clz, type):
+        sfx = _cmprss(clz.__name__)
+        _mk = lambda k, *args, **kwargs: k(*args, **kwargs)
+    elif isinstance(clz, (list, tuple)) and clz and isinstance(clz[0], type):
+        sfx = _cmprss(clz[0].__name__)
+        _mk = lambda k, *args, **kwargs: [k[0](*args, **kwargs), *k[1:]]
+    else:
+        raise ValueError(
+            "Can't recognize data for variating: must be either a class type or a list where first element is class type"
+        )
+
+    to_excl = [s for s, v in kwargs.items() if not isinstance(v, (list, set, tuple, range))]
+    dic2str = lambda ds: [_cmprss(k) + "=" + str(v) for k, v in ds.items() if k not in to_excl]
+
+    return {
+        f"{sfx}_({ ','.join(dic2str(z)) })": _mk(clz, *args, **z)
+        for z in permutate_params(kwargs, conditions=conditions)
+    }