Qubx 0.0.1__cp311-cp311-manylinux_2_35_x86_64.whl

qubx/__init__.py

@@ -0,0 +1,164 @@
+from qubx.utils import set_mpl_theme, runtime_env, install_pyx_recompiler_for_dev
+install_pyx_recompiler_for_dev()
+
+from loguru import logger
+import os, sys, stackprinter
+from qubx.core.lookups import InstrumentsLookup
+
+
+def formatter(record):
+    end = record["extra"].get("end", "\n")
+    fmt = "<lvl>{message}</lvl>%s" % end
+    if record["level"].name in {"WARNING", "SNAKY"}:
+        fmt = "<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - %s" % fmt
+
+    prefix = "<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green> [ <level>%s</level> ] " % record["level"].icon
+
+    if record["exception"] is not None:
+        # stackprinter.set_excepthook(style='darkbg2')
+        record["extra"]["stack"] = stackprinter.format(record["exception"], style="darkbg")
+        fmt += "\n{extra[stack]}\n"
+
+    if record["level"].name in {"TEXT"}:
+        prefix = ""
+
+    return prefix + fmt
+
+
+config = {
+    "handlers": [ {"sink": sys.stdout, "format": "{time} - {message}"}, ],
+    "extra": {"user": "someone"},
+}
+
+
+logger.configure(**config)
+logger.remove(None)
+logger.add(sys.stdout, format=formatter, colorize=True)
+logger = logger.opt(colors=True)
+
+lookup = InstrumentsLookup()
+
+# registering magic for jupyter notebook
+if runtime_env() in ['notebook', 'shell']:
+    from IPython.core.magic import (Magics, magics_class, line_magic, line_cell_magic)
+    from IPython import get_ipython
+
+    @magics_class
+    class QubxMagics(Magics):
+        # process data manager
+        __manager = None
+
+        @line_magic
+        def qubxd(self, line: str):
+            self.qubx_setup('dark')
+
+        @line_magic
+        def qubxl(self, line: str):
+            self.qubx_setup('light')
+
+        @line_magic
+        def qubx_setup(self, line: str):
+            """
+            QUBE framework initialization
+            """
+            import os
+
+            tpl_path = os.path.join(os.path.dirname(__file__), "_nb_magic.py")
+            # print("TPL:", tpl_path)
+            with open(tpl_path, 'r', encoding="utf8") as myfile:
+                s = myfile.read()
+
+            exec(s, self.shell.user_ns)
+
+            # setup more funcy mpl theme instead of ugly default
+            if line:
+                if 'dark' in line.lower():
+                    set_mpl_theme('dark')
+
+                elif 'light' in line.lower():
+                    set_mpl_theme('light')
+
+            # install additional plotly helpers
+            # from qube.charting.plot_helpers import install_plotly_helpers
+            # install_plotly_helpers()
+
+        def _get_manager(self):
+            if self.__manager is None:
+                import multiprocessing as m
+                self.__manager = m.Manager()
+            return self.__manager
+
+        @line_cell_magic
+        def proc(self, line, cell=None):
+            """
+            Run cell in separate process
+
+            >>> %%proc x, y as MyProc1
+            >>> x.set('Hello')
+            >>> y.set([1,2,3,4])
+            
+            """
+            import multiprocessing as m
+            import time, re
+
+            # create ext args
+            name = None
+            if line:
+                # check if custom process name was provided
+                if ' as ' in line:
+                    line, name = line.split('as')
+                    if not name.isspace():
+                        name = name.strip()
+                    else:
+                        print('>>> Process name must be specified afer "as" keyword !')
+                        return
+
+                ipy = get_ipython()
+                for a in [x for x in re.split('[\ ,;]', line.strip()) if x]:
+                    ipy.push({a: self._get_manager().Value(None, None)})
+
+            # code to run
+            lines = '\n'.join(['    %s' % x for x in cell.split('\n')])
+
+            def fn():
+                result = get_ipython().run_cell(lines)
+
+                # send errors to parent
+                if result.error_before_exec:
+                    raise result.error_before_exec
+
+                if result.error_in_exec:
+                    raise result.error_in_exec
+
+            t_start = str(time.time()).replace('.', '_')
+            f_id = f'proc_{t_start}' if name is None else name
+            if self._is_task_name_already_used(f_id):
+                f_id = f"{f_id}_{t_start}"
+
+            task = m.Process(target=fn, name=f_id)
+            task.start()
+            print(' -> Task %s is started' % f_id)
+
+        def _is_task_name_already_used(self, name):
+            import multiprocessing as m
+            for p in m.active_children():
+                if p.name == name:
+                    return True
+            return False
+
+        @line_magic
+        def list_proc(self, line):
+            import multiprocessing as m
+            for p in m.active_children():
+                print(p.name)
+
+        @line_magic
+        def kill_proc(self, line):
+            import multiprocessing as m
+            for p in m.active_children():
+                if line and p.name.startswith(line):
+                    p.terminate()
+
+
+    # - registering magic here
+    get_ipython().register_magics(QubxMagics)

qubx/_nb_magic.py

@@ -0,0 +1,69 @@
+""""
+Here stuff we want to have in every Jupyter notebook after calling %qube magic
+"""
+import importlib_metadata
+
+import qubx
+from qubx.utils import runtime_env
+from qubx.utils.misc import add_project_to_system_path
+
+
+def np_fmt_short():
+    # default np output is 75 columns so extend it a bit and suppress scientific fmt for small floats
+    np.set_printoptions(linewidth=240, suppress=True)
+
+
+def np_fmt_reset():
+    # reset default np printing options
+    np.set_printoptions(edgeitems=3, infstr='inf', linewidth=75, nanstr='nan', precision=8,
+                        suppress=False, threshold=1000, formatter=None)
+
+
+if runtime_env() in ['notebook', 'shell']:
+
+    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    # -- all imports below will appear in notebook after calling %%alphalab magic ---
+    # - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+
+    # - - - - Common stuff - - - -
+    import numpy as np
+    import pandas as pd
+    from datetime import time, timedelta
+    from tqdm.auto import tqdm
+
+    # - - - - TA stuff and indicators - - - -
+    # - - - - Portfolio analysis - - - -
+    # - - - - Simulator stuff - - - -
+    # - - - - Learn stuff - - - -
+    # - - - - Charting stuff - - - -
+    from matplotlib import pyplot as plt
+    from qubx.utils.charting.mpl_helpers import fig, subplot, sbp
+    # - - - - Utils - - - -
+
+    # - setup short numpy output format
+    np_fmt_short()
+    
+    # - add project home to system path
+    add_project_to_system_path()
+
+    # - check current version
+    try: 
+        version = importlib_metadata.version('qube2')
+    except:
+        version = 'Dev'
+
+    # some new logo
+    if not hasattr(qubx.QubxMagics, '__already_initialized__'):
+        from qubx.utils.misc import (green, yellow, cyan, magenta, white, blue, red)
+
+        print(
+        f"""
+                   {red("╻")}
+   {green("┏┓      ╻     ")}  {red("┃")}  {yellow("┏┓")}       {cyan("Quantitative Backtesting Environment")} 
+   {green("┃┃  ┓┏  ┣┓  ┏┓")}  {red("┃")}  {yellow("┏┛")}       
+   {green("┗┻  ┗┻  ┗┛  ┗ ")}  {red("┃")}  {yellow("┗━")}       (c) 2024,  ver. {magenta(version.rstrip())}
+                   {red("╹")}       
+"""
+        )
+        qubx.QubxMagics.__already_initialized__ = True
+

qubx/core/basics.py

@@ -0,0 +1,224 @@
+from datetime import datetime
+from typing import Callable, Dict, List, Optional, Union
+import numpy as np
+import math
+from dataclasses import dataclass, field
+from qubx.core.series import Quote, Trade, time_as_nsec
+from qubx.core.utils import time_to_str, time_delta_to_str, recognize_timeframe
+
+
+dt_64 = np.datetime64
+
+
+@dataclass
+class FuturesInfo:
+    contract_type: Optional[str] = None             # contract type  
+    delivery_date: Optional[datetime] = None        # delivery date
+    onboard_date: Optional[datetime] = None         # futures contract size
+    contract_size: float = 1.0                      # futures contract size
+    maint_margin: float = 0.0                       # maintanance margin
+    required_margin: float = 0.0                    # required margin
+    liquidation_fee: float = 0.0                    # liquidation cost
+
+    def __str__(self) -> str:
+        return f"{self.contract_type} ({self.contract_size}) {self.onboard_date.isoformat()} -> {self.delivery_date.isoformat()}"
+
+
+@dataclass
+class Instrument:
+    symbol: str                                     # instrument's name
+    market_type: str                                # market type (CRYPTO, STOCK, FX, etc)
+    exchange: str                                   # exchange id
+    base: str                                       # base symbol
+    quote: str                                      # quote symbol
+    margin_symbol: str                              # margin asset
+    min_tick: float = 0.0                           # tick size - minimal price change
+    min_size_step: float = 0.0                      # minimal position change step size
+    min_size: float = 0.0                           # minimal allowed position size
+
+    # - futures section
+    futures_info: Optional[FuturesInfo] = None          
+
+    _aux_instrument: Optional['Instrument'] = None  # instrument used for conversion to main asset basis
+                                                    #  | let's say we trade BTC/ETH with main account in USDT
+                                                    #  | so we need to use ETH/USDT for convert profits/losses to USDT
+    _tick_precision: int = field(repr=False ,default=-1) #type: check
+    _size_precision: int = field(repr=False ,default=-1)
+
+    @property
+    def is_futures(self) -> bool:
+        return self.futures_info is not None
+
+    @property
+    def price_precision(self):
+        if self._tick_precision < 0:
+            self._tick_precision = int(abs(np.log10(self.min_tick)))
+        return self._tick_precision
+
+    @property
+    def size_precision(self):
+        if self._size_precision < 0:
+            self._size_precision = int(abs(np.log10(self.min_size_step)))
+        return self._size_precision
+
+    def __str__(self) -> str:
+        return f"{self.exchange}:{self.symbol} [{self.market_type} {str(self.futures_info) if self.futures_info else 'SPOT ' + self.base + '/' + self.quote }]" 
+
+
+@dataclass
+class Signal: 
+    """
+    Class for presenting signals generated by strategy
+    """
+    instrument: Instrument
+    signal: float
+    price: Optional[float] = None
+    stop: Optional[float] = None
+    take: Optional[float] = None
+    group: Optional[str] = None 
+    comment: Optional[str] = None
+
+
+class TransactionCostsCalculator:
+    def __init__(self, maker: float, taker: float):
+        self.maker = maker
+        self.taker = taker
+
+    def get_execution_fees(self, instrument: Instrument, exec_price: float, amount: float, crossed_market=False, conversion_rate=1.0):
+        if crossed_market:
+            return abs(amount * exec_price) * self.taker / conversion_rate
+        else:
+            return abs(amount * exec_price) * self.maker / conversion_rate
+
+    def get_overnight_fees(self, instrument: Instrument, amount: float):
+        return 0.0
+
+    def get_funding_rates_fees(self, instrument: Instrument, amount: float):
+        return 0.0
+
+    def __repr__(self):
+        return f'<TCC: {self.maker * 100:.4f} / {self.taker * 100:.4f}>'
+
+
+ZERO_COSTS = TransactionCostsCalculator(0.0, 0.0)
+
+
+class Position:
+    instrument: Instrument                      # instrument for this poisition
+    quantity: float = 0.0                       # quantity positive for long and negative for short
+    tcc: TransactionCostsCalculator             # transaction costs calculator
+    pnl: float = 0.0                            # total cumulative position PnL in portfolio basic funds currency
+    r_pnl: float = 0.0                          # total cumulative position PnL in portfolio basic funds currency
+    market_value: float = 0.0                   # position's market value in quote currency
+    market_value_funds: float = 0.0             # position market value in portfolio funded currency
+    position_avg_price: float = 0.0             # average position price
+    position_avg_price_funds: float = 0.0       # average position price
+    commissions: float = 0.0                    # cumulative commissions paid for this position
+
+    last_update_time: Optional[int] = None      # when price updated or position changed
+    last_update_price: Optional[float] = None   # last update price (actually instrument's price) in quoted currency
+
+    # - helpers for position processing 
+    _formatter: str
+    _prc_formatter: str
+    _qty_multiplier: float = 1.0
+
+    def __init__(self, instrument: Instrument, tcc: TransactionCostsCalculator, 
+                 quantity=0.0, average_price=0.0, aux_price=1.0, 
+                 ) -> None:
+        self.instrument = instrument
+        self.tcc = tcc
+        
+        # - size/price formaters
+        #                 time         [symbol]                                                        qty                                                  
+        self._formatter = f'%s [{instrument.exchange}:{instrument.symbol}] %{instrument.size_precision+8}.{instrument.size_precision}f'
+        #                           pos_avg_px                 pnl  | mkt_price mkt_value
+        self._formatter += f'%10.{instrument.price_precision}f %+10.4f | %s  %10.2f'
+        self._prc_formatter = f"%.{instrument.price_precision}f"
+        if instrument.is_futures:
+            self._qty_multiplier = instrument.futures_info.contract_size
+
+        if quantity != 0.0 and average_price > 0.0:
+            self.quantity = quantity
+            raise ValueError("[TODO] Position: restore state by quantity and avg price !!!!")
+
+    def _price(self, update: Union[Quote, Trade]) -> float:
+        if isinstance(update, Quote):
+            return update.bid if np.sign(self.quantity) > 0 else update.ask
+        elif isinstance(update, Trade):
+            return update.price
+        raise ValueError(f"Unknown update type: {type(update)}")
+
+    def update_position(self, timestamp: dt_64, position: float, exec_price: float, aggressive=True, conversion_rate:float=1) -> float:
+        # - realized PnL of this fill
+        deal_pnl = 0
+        quantity = self.quantity
+
+        if quantity != position:
+            pos_change = position - quantity
+            direction = np.sign(pos_change)
+            prev_direction = np.sign(quantity)
+
+            # how many shares are closed/open
+            qty_closing = min(abs(self.quantity), abs(pos_change)) * direction if prev_direction != direction else 0
+            qty_opening = pos_change if prev_direction == direction else pos_change - qty_closing
+
+            # - extract realized part of PnL
+            if qty_closing != 0:
+                deal_pnl = qty_closing * (self.position_avg_price - exec_price)
+                quantity += qty_closing
+                # - reset average price to 0 if smaller than minimal price change to avoid cumulative error
+                if abs(quantity) < self.instrument.min_size_step:
+                    quantity = 0.0
+                    self.position_avg_price = 0.0
+
+            # - if it has something to add to position let's update price and cost
+            if qty_opening != 0:
+                qa_open, qas = abs(qty_opening), abs(quantity)
+                self.position_avg_price = (qa_open * exec_price + qas * self.position_avg_price) / (qas + qa_open)
+
+            # - update position and position's price
+            self.position_avg_price_funds = self.position_avg_price / conversion_rate
+            self.quantity = position
+
+            # - convert PnL to fund currency
+            self.r_pnl += deal_pnl / conversion_rate
+
+            # - update pnl
+            self._update_market_price(time_as_nsec(timestamp), exec_price, conversion_rate)
+
+            # - calculate transaction costs
+            comms = self.tcc.get_execution_fees(self.instrument, exec_price, pos_change, aggressive, conversion_rate)
+            self.commissions += comms
+
+        return deal_pnl
+
+    def update_market_price(self, price: Union[Quote, Trade], conversion_rate:float=1) -> float:
+        return self._update_market_price(price.time, self._price(price), conversion_rate)
+
+    def _update_market_price(self, timestamp: dt_64, price: float, conversion_rate:float) -> float:
+        self.last_update_time = timestamp
+        self.last_update_price = price
+
+        if not np.isnan(price):
+            self.pnl = self.quantity * (price - self.position_avg_price) / conversion_rate + self.r_pnl
+            self.market_value = self.quantity * self.last_update_price * self._qty_multiplier
+
+            # calculate mkt value in funded currency
+            self.market_value_funds = self.market_value / conversion_rate
+        return self.pnl
+
+    def total_pnl(self, conversion_rate:float=1.0) -> float:
+        pnl = self.r_pnl
+        if not np.isnan(self.last_update_price):
+            pnl += self.quantity * (self.last_update_price - self.position_avg_price) / conversion_rate
+        return pnl
+
+    @staticmethod
+    def _t2s(t) -> str:
+        return np.datetime64(t, 'ns').astype('datetime64[ms]').item().strftime('%Y-%m-%d %H:%M:%S') if t else '---'
+
+    def __str__(self):
+        _mkt_price = (self._prc_formatter % self.last_update_price) if self.last_update_price else "---"
+        return self._formatter % (Position._t2s(self.last_update_time), self.quantity, self.position_avg_price_funds,self.pnl, _mkt_price,  self.market_value_funds)
+    

qubx/core/lookups.py

@@ -0,0 +1,152 @@
+import glob, re
+import json, os, dataclasses
+from datetime import datetime
+from typing import Dict, List, Optional
+
+from qubx.core.basics import Instrument, FuturesInfo
+from qubx.utils.marketdata.binance import get_binance_symbol_info_for_type
+from qubx import logger
+from qubx.utils.misc import makedirs, get_local_qubx_folder
+
+
+class _InstrumentEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if dataclasses.is_dataclass(obj):
+            return {k:v for k,v in dataclasses.asdict(obj).items() if not k.startswith('_')}
+        if isinstance(obj, (datetime)):
+            return obj.isoformat()
+        return super().default(obj)
+
+
+class _InstrumentDecoder(json.JSONDecoder):
+    def _preprocess(d, ks):
+        fi = d.get(ks)
+        if fi:
+            fi['delivery_date'] = datetime.strptime(fi.get('delivery_date', '5000-01-01T00:00:00'),'%Y-%m-%dT%H:%M:%S')
+            fi['onboard_date'] = datetime.strptime(fi.get('onboard_date', '1970-01-01T00:00:00'),'%Y-%m-%dT%H:%M:%S')
+        return d | {ks: FuturesInfo(**fi) if fi else None}
+
+    def decode(self, json_string):
+        obj = super(_InstrumentDecoder, self).decode(json_string)
+        if isinstance(obj, dict):
+            return Instrument(**_InstrumentDecoder._preprocess(obj, 'futures_info'))
+        elif isinstance(obj, list):
+            return [Instrument(**_InstrumentDecoder._preprocess(o, 'futures_info')) for o in obj]
+        return obj
+
+
+class InstrumentsLookup:
+    lookup: Dict[str, Instrument]
+    path: str
+
+    def __init__(self, path: str=makedirs(get_local_qubx_folder(), 'instruments')) -> None:
+        self.path = path
+        if not self.load():
+            self.refresh()
+        self.load()
+
+    def load(self) -> bool:
+        self.lookup = {}
+        data_exists = False
+        for fs in glob.glob(self.path + '/*.json'):
+            try:
+                with open(fs, 'r') as f:
+                    instrs = json.load(f, cls=_InstrumentDecoder)
+                    for i in instrs:  
+                        self.lookup[f"{i.exchange}:{i.symbol}"] = i
+                    data_exists = True
+            except Exception as ex:
+                logger.warning(ex)
+
+        return data_exists
+
+    def find(self, exchange: str, base: str, quote: str) -> Optional[Instrument]:
+        for i in self.lookup.values():
+            if i.exchange == exchange and (
+                (i.base == base and i.quote == quote) or (i.base == quote and i.quote == base)
+            ):
+                return i
+        return None
+    
+    def find_aux_instrument_for(self, instrument: Instrument, base_currency: str) -> Optional[Instrument]:
+        """
+        Tries to find aux instrument (for conversions to funded currency)
+        for example: 
+            ETHBTC -> BTCUSDT for base_currency USDT
+            EURGBP -> GBPUSD for base_currency USD
+            ...
+        """
+        base_currency = base_currency.upper()
+        if instrument.quote != base_currency and instrument._aux_instrument is None:
+            return self.find(instrument.exchange, instrument.quote, base_currency)
+        return instrument._aux_instrument
+
+    def __getitem__(self, spath: str) -> List[Instrument]:
+        res = []
+        c = re.compile(spath)
+        for k, v in self.lookup.items():
+            if re.match(c, k):
+                res.append(v)
+        return res
+    
+    def refresh(self):
+        for mn in dir(self):
+            if mn.startswith('_update_'):
+                getattr(self, mn)(self.path)
+
+    def _update_kraken(self, path: str):
+        # TODO
+        pass
+
+    def _update_dukas(self, path: str):
+        instruments = [
+            Instrument('EURUSD', 'FX', 'DUKAS', 'EUR', 'USD', 'USD', 0.00001, 1, 1000),
+            Instrument('GBPUSD', 'FX', 'DUKAS', 'GBP', 'USD', 'USD', 0.00001, 1, 1000),
+            Instrument('USDJPY', 'FX', 'DUKAS', 'USD', 'JPY', 'USD', 0.001,   1, 1000),
+            Instrument('USDCAD', 'FX', 'DUKAS', 'USD', 'CAD', 'USD', 0.00001, 1, 1000),
+            Instrument('AUDUSD', 'FX', 'DUKAS', 'AUD', 'USD', 'USD', 0.00001, 1, 1000),
+            Instrument('USDPLN', 'FX', 'DUKAS', 'USD', 'PLN', 'USD', 0.00001, 1, 1000),
+            Instrument('EURGBP', 'FX', 'DUKAS', 'EUR', 'GBP', 'USD', 0.00001, 1, 1000),
+            # TODO: addd all or find how to get it from site
+        ]
+        logger.info(f'Updates {len(instruments)} for DUKASCOPY')
+        with open(os.path.join(path, f'dukas.json'), 'w') as f:
+            json.dump(instruments, f, cls=_InstrumentEncoder)
+
+    def _update_binance(self, path: str):
+        infos = get_binance_symbol_info_for_type(['UM', 'CM', 'SPOT'])
+        for exchange, info in infos.items():
+            instruments = []
+            for s in info['symbols']:
+                tick_size, size_step = None, None 
+                for i in s['filters']:
+                    if i['filterType'] == 'PRICE_FILTER':
+                        tick_size = float(i['tickSize'])
+                    if i['filterType'] == 'LOT_SIZE':
+                        size_step = float(i['stepSize'])
+
+                fut_info = None
+                if 'contractType' in s:
+                    fut_info = FuturesInfo( 
+                        s.get('contractType', 'UNKNOWN'),
+                        datetime.fromtimestamp(s.get('deliveryDate', 0)/1000.0),
+                        datetime.fromtimestamp(s.get('onboardDate', 0)/1000.0),
+                        float(s.get('contractSize', 1)), 
+                        float(s.get('maintMarginPercent', 0)), 
+                        float(s.get('requiredMarginPercent', 0)), 
+                        float(s.get('liquidationFee', 0)),
+                    )
+
+                instruments.append(
+                    Instrument(
+                        s['symbol'], 'CRYPTO', exchange.upper(), s['baseAsset'], s['quoteAsset'], s.get('marginAsset', None), 
+                        tick_size, size_step, 
+                        min_size=size_step,    # TODO: not sure about minimal position for Binance
+                        futures_info=fut_info
+                ))
+
+            logger.info(f'Loaded {len(instruments)} for {exchange}')
+
+            with open(os.path.join(path, f'{exchange}.json'), 'w') as f:
+                json.dump(instruments, f, cls=_InstrumentEncoder)
+