siglab-py 0.5.30__py3-none-any.whl → 0.6.12__py3-none-any.whl

siglab_py/util/market_data_util.py

@@ -9,9 +9,9 @@ import math
 import pandas as pd
 import numpy as np
 import asyncio
+from tabulate import tabulate
 
 from ccxt.base.exchange import Exchange as CcxtExchange
-from ccxt import deribit
 import ccxt
 import ccxt.pro as ccxtpro
 
@@ -24,6 +24,8 @@ import yfinance as yf
 from siglab_py.util.retry_util import retry
 from siglab_py.exchanges.futubull import Futubull
 from siglab_py.exchanges.any_exchange import AnyExchange
+from siglab_py.exchanges.deribit import Deribit, DeribitAsync
+from siglab_py.exchanges.binance import Binance, BinanceAsync
 
 def instantiate_exchange(
     exchange_name : str,
@@ -31,6 +33,7 @@ def instantiate_exchange(
     secret : Union[str, None]  = None,
     passphrase : Union[str, None] = None,
     default_type : Union[str, None] = 'spot',
+    default_sub_type : Union[str, None] = None,
     rate_limit_ms : float = 100
 ) -> Union[AnyExchange, None]:
     exchange_name = exchange_name.lower().strip()
@@ -46,6 +49,10 @@ def instantiate_exchange(
                             'defaultType' : default_type
                         }
                     }
+    
+    if default_sub_type:
+        exchange_params['defaultSubType'] = default_sub_type
+
     if api_key:
         exchange_params['apiKey'] = api_key
     if secret:
@@ -54,13 +61,13 @@ def instantiate_exchange(
         exchange_params['passphrase'] = passphrase
 
     if exchange_name=='binance':
-        exchange = ccxt.binance(exchange_params)  # type: ignore
+        exchange = Binance(exchange_params)  # type: ignore
     elif exchange_name=='bybit':
         exchange = ccxt.bybit(exchange_params) # type: ignore
     elif exchange_name=='okx':
         exchange = ccxt.okx(exchange_params) # type: ignore
     elif exchange_name=='deribit':
-        exchange = ccxt.deribit(exchange_params)  # type: ignore
+        exchange = Deribit(exchange_params)  # type: ignore
     elif exchange_name=='hyperliquid':
         exchange = ccxt.hyperliquid(
             {
@@ -82,8 +89,10 @@ async def async_instantiate_exchange(
     api_key : str,
     secret : str,
     passphrase : str,
-    default_type : str,
-    rate_limit_ms : float = 100
+    default_type : Union[str, None] = 'spot',
+    default_sub_type : Union[str, None] = None,
+    rate_limit_ms : float = 100,
+    verbose : bool = False
 ) -> Union[AnyExchange, None]:
     exchange : Union[AnyExchange, None] = None
     exchange_name : str = gateway_id.split('_')[0]
@@ -98,13 +107,17 @@ async def async_instantiate_exchange(
                         'rateLimit' : rate_limit_ms,
                         'options' : {
                             'defaultType' : default_type
-                        }
+                        },
+                        'verbose': verbose
                     }
+    
+    if default_sub_type:
+        exchange_params['defaultSubType'] = default_sub_type
 
     if exchange_name=='binance':
         # spot, future, margin, delivery, option
         # https://github.com/ccxt/ccxt/blob/master/python/ccxt/binance.py#L1298
-        exchange = ccxtpro.binance(exchange_params)  # type: ignore
+        exchange = BinanceAsync(exchange_params)  # type: ignore
     elif exchange_name=='bybit':
         # spot, linear, inverse, futures
         # https://github.com/ccxt/ccxt/blob/master/python/ccxt/bybit.py#L1041
@@ -117,7 +130,7 @@ async def async_instantiate_exchange(
     elif exchange_name=='deribit':
         # spot, swap, future
         # https://github.com/ccxt/ccxt/blob/master/python/ccxt/deribit.py#L360
-        exchange = ccxtpro.deribit(exchange_params)  # type: ignore
+        exchange = DeribitAsync(exchange_params)  # type: ignore
     elif exchange_name=='kraken':
         exchange = ccxtpro.kraken(exchange_params) # type: ignore
     elif exchange_name=='hyperliquid':
@@ -156,9 +169,10 @@ async def async_instantiate_exchange(
                 "walletAddress" : api_key,
                 "privateKey" : secret,
                 'enableRateLimit'  : True,
-                'rateLimit' : rate_limit_ms
-            }
-        )  # type: ignore
+                'rateLimit' : rate_limit_ms,
+                'verbose': verbose
+            }  # type: ignore
+        )
     else:
         raise ValueError(f"Unsupported exchange {exchange_name}, check gateway_id {gateway_id}.")
 
@@ -192,18 +206,18 @@ def timestamp_to_datetime_cols(pd_candles : pd.DataFrame):
     pd_candles['timestamp_ms'] = pd_candles['timestamp_ms'].apply(_fix_timestamp_ms)
     pd_candles['datetime'] = pd_candles['timestamp_ms'].apply(lambda x: datetime.fromtimestamp(int(x/1000)))
     pd_candles['datetime'] = pd.to_datetime(pd_candles['datetime'])
-    pd_candles['datetime'] = pd_candles['datetime'].dt.tz_localize(None)
+    pd_candles['datetime'] = pd_candles['datetime'].dt.tz_localize(None)  # type: ignore
     pd_candles['datetime_utc'] = pd_candles['timestamp_ms'].apply(
         lambda x: datetime.fromtimestamp(int(x.timestamp()) if isinstance(x, pd.Timestamp) else int(x / 1000), tz=timezone.utc)
     )
     
     # This is to make it easy to do grouping with Excel pivot table
-    pd_candles['year'] = pd_candles['datetime'].dt.year
-    pd_candles['month'] = pd_candles['datetime'].dt.month
-    pd_candles['day'] = pd_candles['datetime'].dt.day
-    pd_candles['hour'] = pd_candles['datetime'].dt.hour
-    pd_candles['minute'] = pd_candles['datetime'].dt.minute
-    pd_candles['dayofweek'] = pd_candles['datetime'].dt.dayofweek  # dayofweek: Monday is 0 and Sunday is 6
+    pd_candles['year'] = pd_candles['datetime'].dt.year  # type: ignore
+    pd_candles['month'] = pd_candles['datetime'].dt.month  # type: ignore
+    pd_candles['day'] = pd_candles['datetime'].dt.day  # type: ignore
+    pd_candles['hour'] = pd_candles['datetime'].dt.hour  # type: ignore
+    pd_candles['minute'] = pd_candles['datetime'].dt.minute  # type: ignore
+    pd_candles['dayofweek'] = pd_candles['datetime'].dt.dayofweek  # type: ignore dayofweek: Monday is 0 and Sunday is 6
 
     pd_candles['week_of_month'] = pd_candles['timestamp_ms'].apply(
         lambda x: timestamp_to_week_of_month(int(x/1000))
@@ -220,42 +234,58 @@ def timestamp_to_datetime_cols(pd_candles : pd.DataFrame):
     )
 
     pd_candles['timestamp_ms_gap'] = pd_candles['timestamp_ms'] - pd_candles['timestamp_ms'].shift(1)
-    timestamp_ms_gap = pd_candles.iloc[-1]['timestamp_ms_gap']
-    assert(pd_candles[~pd_candles.timestamp_ms_gap.isna()][pd_candles.timestamp_ms_gap!=timestamp_ms_gap].shape[0]==0)
+    
+    # Depending on asset, minutes bar may have gaps
+    timestamp_ms_gap_median = pd_candles['timestamp_ms_gap'].median()
+    NUM_MS_IN_1HR = 60*60*1000
+    if timestamp_ms_gap_median>=NUM_MS_IN_1HR:
+        num_rows_with_expected_gap = pd_candles[~pd_candles.timestamp_ms_gap.isna()][pd_candles.timestamp_ms_gap==timestamp_ms_gap_median].shape[0]
+        assert(num_rows_with_expected_gap/pd_candles.shape[0]>0.9)
     pd_candles.drop(columns=['timestamp_ms_gap'], inplace=True)
 
+'''
+APAC (Asia-Pacific) Trading Hours
+    UTC 21:00 - 09:00 (approximate range)
+    Major financial centers: Tokyo, Hong Kong, Singapore, Sydney
+
+EMEA (Europe, Middle East, Africa) Trading Hours
+    UTC 07:00 - 16:00 (approximate range)
+    Major financial centers: London, Frankfurt, Paris, Zurich, Dubai
+
+US Trading Hours
+    UTC 13:00 - 22:00 (approximate range)
+    Major financial centers: New York, Chicago
+    Key markets: NYSE, NASDAQ
+
+utcnow and utcfromtimestamp been deprecated in Python 3.12 
+https://www.pythonmorsels.com/converting-to-utc-time/
+
+Example, UTC 23:00 is 3rd hour in APAC trading session
+    utc_hour = 23
+    i = get_regions_trading_utc_hours()['APAC'].index(utc_hour)
+    assert(i==2)
+'''
+def get_regions_trading_utc_hours():
+    return {
+        'APAC' : [21,22,23,0,1,2,3,4,5,6,7,8,9],
+        'EMEA' : [7,8,9,10,11,12,13,14,15,16],
+        'AMER' : [13,14,15,16,17,18,19,20,21,22]
+    }
+			
 def timestamp_to_active_trading_regions(
         timestamp_ms : int
 ) -> List[str]:
-    
-    '''
-    APAC (Asia-Pacific) Trading Hours
-        UTC 22:00 - 09:00 (approximate range)
-        Major financial centers: Tokyo, Hong Kong, Singapore, Sydney
-
-    EMEA (Europe, Middle East, Africa) Trading Hours
-        UTC 07:00 - 16:00 (approximate range)
-        Major financial centers: London, Frankfurt, Paris, Zurich, Dubai
-
-    US Trading Hours
-        UTC 13:30 - 20:00 (approximate range)
-        Major financial centers: New York, Chicago
-        Key markets: NYSE, NASDAQ
-
-    utcnow and utcfromtimestamp been deprecated in Python 3.12 
-    https://www.pythonmorsels.com/converting-to-utc-time/
-    '''
     active_trading_regions : List[str] = []
 
     dt_utc = datetime.fromtimestamp(int(timestamp_ms / 1000), tz=timezone.utc)
     utc_hour = dt_utc.hour
-    if (utc_hour >= 22) or (utc_hour <= 9):
+    if utc_hour in get_regions_trading_utc_hours()['APAC']:
         active_trading_regions.append("APAC") 
 
-    if 7 <= utc_hour <= 16:
+    if utc_hour in get_regions_trading_utc_hours()['EMEA']:
         active_trading_regions.append("EMEA")
 
-    if 13 <= utc_hour <= 20:
+    if utc_hour in get_regions_trading_utc_hours()['AMER']:
         active_trading_regions.append("AMER")
 
     return active_trading_regions
@@ -288,6 +318,17 @@ def fix_column_types(pd_candles : pd.DataFrame):
     pd_candles.reset_index(drop=True, inplace=True)
     pd_candles.sort_values("datetime", inplace=True)
 
+def interval_to_ms(interval : str) -> int:
+    interval_ms : int = 0
+    if interval=="d":
+        interval_ms = 24*60*60*1000
+    elif interval=="h":
+        interval_ms = 60*60*1000
+    elif interval=="m":
+        interval_ms = 60*1000
+
+    return interval_ms
+
 '''
 https://polygon.io/docs/stocks
 '''
@@ -343,7 +384,7 @@ class NASDAQExchange:
             pd_daily_candles['low'] = pd_daily_candles['low'].astype(str).str.replace('$','')
             pd_daily_candles['close'] = pd_daily_candles['close'].astype(str).str.replace('$','')
             pd_daily_candles['datetime']= pd.to_datetime(pd_daily_candles['datetime'])
-            pd_daily_candles['timestamp_ms'] = pd_daily_candles.datetime.values.astype(np.int64) // 10 ** 6
+            pd_daily_candles['timestamp_ms'] = pd_daily_candles.datetime.values.astype(np.int64) // 10 ** 6  # type: ignore
             pd_daily_candles['symbol'] = symbol
             pd_daily_candles['exchange'] = 'nasdaq'
             fix_column_types(pd_daily_candles)
@@ -362,7 +403,7 @@ class NASDAQExchange:
                 )
 
                 # When you fill foward, a few candles before start date can have null values (open, high, low, close, volume ...)
-                first_candle_dt = pd_hourly_candles[(~pd_hourly_candles.close.isna())  & (pd_hourly_candles['datetime'].dt.time == pd.Timestamp('00:00:00').time())].iloc[0]['datetime']
+                first_candle_dt = pd_hourly_candles[(~pd_hourly_candles.close.isna())  & (pd_hourly_candles['datetime'].dt.time == pd.Timestamp('00:00:00').time())].iloc[0]['datetime']  # type: ignore
                 pd_hourly_candles = pd_hourly_candles[pd_hourly_candles.datetime>=first_candle_dt]
                 exchange_candles[symbol] = pd_hourly_candles
 
@@ -426,6 +467,45 @@ class YahooExchange:
 
         return exchange_candles
 
+def aggregate_candles(
+    interval : str,
+    pd_candles : pd.DataFrame
+) -> pd.DataFrame:
+    if interval[-1]=='m':
+        # 'm' for pandas means months!
+        interval = interval.replace('m','min')
+    pd_candles.set_index('datetime', inplace=True)
+    pd_candles_aggregated = pd_candles.resample(interval).agg({
+        'exchange' : 'first',
+        'symbol' : 'first',
+        'timestamp_ms' : 'first',
+        
+        'open': 'first',
+        'high': 'max',
+        'low': 'min',
+        'close': 'last',
+        'volume': 'sum',
+
+        'datetime_utc' : 'first',
+        'year' : 'first',
+        'month' : 'first',
+        'day' : 'first',
+        'hour' : 'first',
+        'minute' : 'first',
+        'dayofweek' : 'first',
+        'week_of_month' : 'first',
+
+        'apac_trading_hr' : 'first',
+        'emea_trading_hr' : 'first',
+        'amer_trading_hr' : 'first',
+
+        'pct_chg_on_close' : 'sum',
+
+    })
+    pd_candles.reset_index(inplace=True)
+    pd_candles_aggregated.reset_index(inplace=True)
+    return pd_candles_aggregated
+    
 def fetch_historical_price(
     exchange,
     normalized_symbol : str,
@@ -472,19 +552,21 @@ def fetch_candles(
     validation_max_gaps : int = 10,
     validation_max_end_date_intervals : int = 1
 ) -> Dict[str, Union[pd.DataFrame, None]]:
-    
+    exchange_candles = { '' : None }
+    num_intervals = int(candle_size.replace(candle_size[-1],''))
+
     if end_ts>datetime.now().timestamp():
         end_ts = int(datetime.now().timestamp())
 
     if type(exchange) is YahooExchange:
-        return exchange.fetch_candles(
+        exchange_candles = exchange.fetch_candles(
                             start_ts=start_ts,
                             end_ts=end_ts,
                             symbols=normalized_symbols,
                             candle_size=candle_size
                         )
     elif type(exchange) is NASDAQExchange:
-        return exchange.fetch_candles(
+        exchange_candles = exchange.fetch_candles(
                             start_ts=start_ts,
                             end_ts=end_ts,
                             symbols=normalized_symbols,
@@ -501,9 +583,9 @@ def fetch_candles(
             pd_candles = exchange_candles[symbol]
             if not pd_candles is None:
                 fix_column_types(pd_candles) # You don't want to do this from Futubull as you'd need import Futubull from there: Circular references
-        return exchange_candles
+                
     elif issubclass(exchange.__class__, CcxtExchange):
-        return _fetch_candles_ccxt(
+        exchange_candles = _fetch_candles_ccxt(
             start_ts=start_ts,
             end_ts=end_ts,
             exchange=exchange,
@@ -511,7 +593,40 @@ def fetch_candles(
             candle_size=candle_size,
             num_candles_limit=num_candles_limit
         )
-    return { '' : None }
+    if num_intervals!=1:
+        for symbol in exchange_candles:
+            if not exchange_candles[symbol] is None:
+                exchange_candles[symbol] = aggregate_candles(candle_size, exchange_candles[symbol]) #  type: ignore
+
+    # For invalid rows missing timestamps, o/h/l/c/v, fill forward close, set volume to zero.
+    for symbol in exchange_candles:
+        pd_candles = exchange_candles[symbol]
+        
+        if pd_candles is not None:
+            mask_invalid_candles = pd_candles["timestamp_ms"].isna()
+            if mask_invalid_candles.any():
+                pd_invalid_candles = pd_candles[mask_invalid_candles]
+
+                if logger is not None:
+                    logger.warning(f"Dropping {pd_invalid_candles.shape[0]}/{pd_candles.shape[0]} rows from {symbol} candles (null timestamp_ms)") # type: ignore
+                    logger.warning(f"{tabulate(pd_invalid_candles, headers='keys', tablefmt='psql')}") # type: ignore
+                    
+                def _to_timestamp_ms(dt):
+                    if pd.isna(dt):
+                        return pd.NA
+                    if isinstance(dt, str):
+                        dt = pd.to_datetime(dt)
+                    return int(dt.timestamp() * 1000)
+                
+                pd_candles.loc[mask_invalid_candles, "timestamp_ms"] = pd_candles.loc[mask_invalid_candles, "datetime"].apply(_to_timestamp_ms)
+
+                pd_candles["close"] = pd_candles["close"].ffill()
+                pd_candles.loc[mask_invalid_candles, ["open", "high", "low"]] = pd_candles.loc[
+                                                                                    mask_invalid_candles, ["close"]
+                                                                                ]
+                pd_candles.loc[mask_invalid_candles, "volume"] = 0.0
+
+    return exchange_candles # type: ignore
 
 '''
 Find listing date https://gist.github.com/mr-easy/5185b1dcdd5f9f908ff196446f092e9b
@@ -563,7 +678,7 @@ def _fetch_candles_ccxt(
             
         def _calc_increment(candle_size):
             increment = 1
-            num_intervals = int(candle_size[0])
+            num_intervals = int(candle_size.replace(candle_size[-1],''))
             interval_type = candle_size[-1]
             if interval_type == "m":
                 increment = 60
@@ -584,12 +699,13 @@ def _fetch_candles_ccxt(
         A more efficient way is to find listing date. Start looping from there.
         '''
         market = exchange.markets[ticker]
+        this_ticker_start_ts = start_ts
         if market['created']:
-            start_ts = max(start_ts, int(market['created']/1000))
+            this_ticker_start_ts = max(this_ticker_start_ts, int(market['created']/1000))
 
         all_candles = []
         params = {}
-        this_cutoff = start_ts
+        this_cutoff = this_ticker_start_ts
         while this_cutoff<end_ts:
             candles = _fetch_ohlcv(exchange=exchange, symbol=ticker, timeframe=candle_size, since=int(this_cutoff * 1000), limit=num_candles_limit, params=params)
             if candles and len(candles)>0:
@@ -623,7 +739,7 @@ def fetch_deribit_btc_option_expiries(
         Dict[str, Dict[str, Union[str, float]]]
     ]
 ]:
-    exchange = deribit()
+    exchange = ccxt.deribit()
     instruments = exchange.public_get_get_instruments({
         'currency': market,
         'kind': 'option',
@@ -679,7 +795,9 @@ def fetch_deribit_btc_option_expiries(
 
 def build_pair_candles(
     pd_candles1 : pd.DataFrame,
-    pd_candles2 : pd.DataFrame
+    pd_candles2 : pd.DataFrame,
+    left_columns_postfix : str = "_1",
+    right_columns_postfix : str = "_2"
 ) -> pd.DataFrame:
     min_timestamp_ms1 = int(pd_candles1.iloc[0]['timestamp_ms'])
     max_timestamp_ms1 = int(pd_candles1.iloc[-1]['timestamp_ms'])
@@ -709,16 +827,16 @@ def build_pair_candles(
     assert(max_timestamp_ms1==max_timestamp_ms2)
     assert(pd_candles1.shape[0]==pd_candles2.shape[0])
 
-    if len([ col for col in pd_candles1.columns if col[-2:]=='_1' ]) == 0:
-        pd_candles1.columns = [str(col) + '_1' for col in pd_candles1.columns]
+    if len([ col for col in pd_candles1.columns if col[-2:]==left_columns_postfix ]) == 0:
+        pd_candles1.columns = [str(col) + left_columns_postfix for col in pd_candles1.columns]
 
-    if len([ col for col in pd_candles2.columns if col[-2:]=='_2' ]) == 0:
-        pd_candles2.columns = [str(col) + '_2' for col in pd_candles2.columns]
+    if len([ col for col in pd_candles2.columns if col[-2:]==right_columns_postfix ]) == 0:
+        pd_candles2.columns = [str(col) + right_columns_postfix for col in pd_candles2.columns]
 
     pd_candles1.reset_index(drop=True, inplace=True)
     pd_candles2.reset_index(drop=True, inplace=True)
     pd_candles = pd.concat([pd_candles1, pd_candles2], axis=1)
-    pd_candles['timestamp_ms_gap'] = pd_candles['timestamp_ms_1'] - pd_candles['timestamp_ms_2']
+    pd_candles['timestamp_ms_gap'] = pd_candles[f'timestamp_ms{left_columns_postfix}'] - pd_candles[f'timestamp_ms{right_columns_postfix}']
     assert(pd_candles[pd_candles.timestamp_ms_gap!=0].shape[0]==0)
 
     pd_candles.drop(pd_candles.columns[pd_candles.columns.str.contains('unnamed',case = False)],axis = 1, inplace = True)

siglab_py/util/notification_util.py

@@ -5,7 +5,7 @@ import pandas as pd
 import numpy as np
 from tabulate import tabulate
 
-from util.slack_notification_util import slack_dispatch_notification
+from siglab_py.util.slack_notification_util import slack_dispatch_notification
 
 from siglab_py.constants import LogLevel