siglab-py 0.3.2__tar.gz → 0.3.3__tar.gz

{siglab_py-0.3.2 → siglab_py-0.3.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: siglab_py
-Version: 0.3.2
+Version: 0.3.3
 Summary: Market data fetches, TA calculations and generic order gateway.
 Author: r0bbarh00d
 Author-email: r0bbarh00d <r0bbarh00d@gmail.com>

{siglab_py-0.3.2 → siglab_py-0.3.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "siglab_py"
-version = "0.3.2"
+version = "0.3.3"
 description = "Market data fetches, TA calculations and generic order gateway."
 authors = [{name = "r0bbarh00d", email = "r0bbarh00d@gmail.com"}]
 license = {text = "MIT"}

{siglab_py-0.3.2 → siglab_py-0.3.3}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = siglab_py
-version = 0.3.2
+version = 0.3.3
 description = Market data fetches, TA calculations and generic order gateway.
 author = r0bbarh00d
 author_email = r0bbarh00d@gmail.com

{siglab_py-0.3.2 → siglab_py-0.3.3}/siglab_py/ordergateway/client.py

@@ -1,6 +1,7 @@
 
 from typing import List, Dict, Any, Union
 import json
+from  datetime import datetime
 import time
 from redis import StrictRedis
 
@@ -49,6 +50,8 @@ class Order:
         self.dispatched_amount : float = 0 # This is amount in base ccy, with rounding + multiplier applied (So for contracts with multiplier!=1, this is no longer amount in base ccy.)
         self.dispatched_price : Union[None, float] = None
 
+        self.timestamp_ms = int(datetime.now().timestamp() * 1000)
+
     def to_dict(self) -> Dict[JSON_SERIALIZABLE_TYPES, JSON_SERIALIZABLE_TYPES]:
         return {
             "ticker" : self.ticker,
@@ -59,7 +62,8 @@ class Order:
             "reduce_only" : self.reduce_only,
             "fees_ccy" : self.fees_ccy,
             "dispatched_amount" : self.dispatched_amount,
-            "dispatched_price" : self.dispatched_price
+            "dispatched_price" : self.dispatched_price,
+            "timestamp_ms" : self.timestamp_ms
         }
 
 '''
@@ -97,6 +101,8 @@ class DivisiblePosition(Order):
         self.dispatched_slices : List[Order] = []
         self.executions : Dict[str, Dict[str, Any]] = {}
 
+        self.timestamp_ms = int(datetime.now().timestamp() * 1000)
+
     def to_slices(self) -> List[Order]:
         slices : List[Order] = []
 
@@ -331,6 +337,7 @@ class DivisiblePosition(Order):
         rv['pos'] = self.pos
         rv['done'] = self.done
         rv['execution_err'] = self.execution_err
+        rv['timestamp_ms'] = self.timestamp_ms
         return rv
 
 def execute_positions(

{siglab_py-0.3.2 → siglab_py-0.3.3}/siglab_py/ordergateway/gateway.py

@@ -24,6 +24,7 @@ from util.aws_util import AwsKmsUtil
 import ccxt.pro as ccxtpro
 
 from siglab_py.exchanges.any_exchange import AnyExchange
+from util.market_data_util import async_instantiate_exchange
 from siglab_py.ordergateway.client import Order, DivisiblePosition
 from siglab_py.constants import LogLevel # type: ignore
 from util.notification_util import dispatch_notification
@@ -32,6 +33,7 @@ current_filename = os.path.basename(__file__)
 
 '''
 Usage:
+    set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab\siglab_py
     python gateway.py --gateway_id hyperliquid_01 --default_type linear --rate_limit_ms 100
 
     --default_type defaults to linear
@@ -352,102 +354,6 @@ def init_redis_client() -> StrictRedis:
     
     return redis_client
 
-async def instantiate_exchange(
-    gateway_id : str,
-    api_key : str,
-    secret : str,
-    passphrase : str,
-    default_type : str,
-    rate_limit_ms : float = 100
-) -> Union[AnyExchange, None]:
-    exchange : Union[AnyExchange, None] = None
-    exchange_name : str = gateway_id.split('_')[0]
-    exchange_name =exchange_name.lower().strip()
-
-    # Look at ccxt exchange.describe. under 'options' \ 'defaultType' (and 'defaultSubType') for what markets the exchange support.
-    # https://docs.ccxt.com/en/latest/manual.html#instantiation
-    exchange_params : Dict[str, Any]= {
-                        'apiKey' : api_key,
-                        'secret' : secret,
-                        'enableRateLimit'  : True,
-                        'rateLimit' : rate_limit_ms,
-                        'options' : {
-                            'defaultType' : default_type
-                        }
-                    }
-
-    if exchange_name=='binance':
-        # spot, future, margin, delivery, option
-        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/binance.py#L1298
-        exchange = ccxtpro.binance(exchange_params)  # type: ignore
-    elif exchange_name=='bybit':
-        # spot, linear, inverse, futures
-        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/bybit.py#L1041
-        exchange = ccxtpro.bybit(exchange_params) # type: ignore
-    elif exchange_name=='okx':
-        # 'funding', spot, margin, future, swap, option
-        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/okx.py#L1144
-        exchange_params['password'] = passphrase
-        exchange = ccxtpro.okx(exchange_params) # type: ignore
-    elif exchange_name=='deribit':
-        # spot, swap, future
-        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/deribit.py#L360
-        exchange = ccxtpro.deribit(exchange_params)  # type: ignore
-    elif exchange_name=='hyperliquid':
-        '''
-        https://app.hyperliquid.xyz/API
-        
-        defaultType from ccxt: swap
-            https://github.com/ccxt/ccxt/blob/master/python/ccxt/hyperliquid.py#L225
-        
-        How to integrate? You can skip first 6 min: https://www.youtube.com/watch?v=UuBr331wxr4&t=363s
-
-        Example, 
-            API credentials created under "\ More \ API":
-                    Ledger Arbitrum Wallet Address: 0xAAAAA <-- This is your Ledger Arbitrum wallet address with which you connect to Hyperliquid. 
-                    API Wallet Address 0xBBBBB <-- Generated
-                    privateKey 0xCCCCC
-
-        Basic connection via CCXT:
-            import asyncio
-            import ccxt.pro as ccxtpro
-
-            async def main():
-                rate_limit_ms = 100
-                exchange_params = {
-                    "walletAddress" : "0xAAAAA", # Ledger Arbitrum Wallet Address here! Not the generated address.
-                    "privateKey" : "0xCCCCC"
-                }
-                exchange = ccxtpro.hyperliquid(exchange_params) 
-                balances = await exchange.fetch_balance()
-                print(balances)
-
-            asyncio.run(main())
-        '''
-        exchange = ccxtpro.hyperliquid(
-            {
-                "walletAddress" : api_key,
-                "privateKey" : secret,
-                'enableRateLimit'  : True,
-                'rateLimit' : rate_limit_ms
-            }
-        )  # type: ignore
-    else:
-        raise ValueError(f"Unsupported exchange {exchange_name}, check gateway_id {gateway_id}.")
-
-    await exchange.load_markets() # type: ignore
-
-    '''
-    Is this necessary? The added trouble is for example bybit.authenticate requires arg 'url'. binance doesn't. And fetch_balance already test credentials.
-
-    try:
-        await exchange.authenticate() # type: ignore
-    except Exception as swallow_this_error:
-        pass
-    '''
-
-    return exchange
-
 async def watch_orders_task(
     exchange : AnyExchange,
     executions : Dict[str, Dict[str, Any]]
@@ -931,7 +837,7 @@ async def main():
     
     redis_client : StrictRedis = init_redis_client()
 
-    exchange : Union[AnyExchange, None] = await instantiate_exchange(
+    exchange : Union[AnyExchange, None] = await async_instantiate_exchange(
         gateway_id=param['gateway_id'],
         api_key=api_key,
         secret=secret,

{siglab_py-0.3.2 → siglab_py-0.3.3}/siglab_py/tests/unit/analytic_util_tests.py

@@ -50,6 +50,7 @@ class AnalyticUtilTests(unittest.TestCase):
             'is_green', 'pct_change_close', 
             'sma_short_periods', 'sma_long_periods', 'ema_short_periods', 'ema_long_periods', 'ema_close', 
             'std', 'std_percent', 
+            'normalized_close_short', 'normalized_close_long',
             'candle_height_percent', 'candle_height_percent_rounded', 
             'log_return', 'interval_hist_vol', 'annualized_hist_vol',
             'chop_against_ema', 

{siglab_py-0.3.2 → siglab_py-0.3.3}/siglab_py/util/analytic_util.py

@@ -102,9 +102,6 @@ def compute_candles_stats(
 
     pd_candles['std_percent'] = pd_candles['std'] / pd_candles['ema_close'] * 100
 
-    pd_candles['normalized_close_short'] = (pd_candles['close'] - pd_candles['sma_short_periods']) / close_short_periods_rolling.std()
-    pd_candles['normalized_close_long'] = (pd_candles['close'] - pd_candles['sma_long_periods']) / pd_candles['std']
-
     pd_candles['candle_height_percent'] = pd_candles['candle_height'] / pd_candles['ema_close'] * 100
     pd_candles['candle_height_percent_rounded'] = pd_candles['candle_height_percent'].round().astype('Int64')
 
@@ -371,13 +368,17 @@ def compute_candles_stats(
         import statsmodels.api as sm # in-compatible with pypy
 
         # Slopes
-        X = sm.add_constant(range(len(pd_candles['normalized_close_short'])))
-        rolling_slope = pd_candles['normalized_close_short'].rolling(window=int(sliding_window_how_many_candles/slow_fast_interval_ratio)).apply(lambda x: sm.OLS(x, X[:len(x)]).fit().params[1], raw=False)
+        X = sm.add_constant(range(len(pd_candles['close'])))
+        rolling_slope = pd_candles['close'].rolling(window=int(sliding_window_how_many_candles/slow_fast_interval_ratio)).apply(lambda x: sm.OLS(x, X[:len(x)]).fit().params[1], raw=False)
         pd_candles['close_short_slope'] = rolling_slope
+        max_abs_slope = pd_candles['close_short_slope'].abs().rolling(window=int(sliding_window_how_many_candles/slow_fast_interval_ratio)).max()
+        pd_candles['normalized_close_short_slope'] = pd_candles['close_short_slope'] / (2* max_abs_slope)
 
-        X = sm.add_constant(range(len(pd_candles['normalized_close_long'])))
-        rolling_slope = pd_candles['normalized_close_long'].rolling(window=sliding_window_how_many_candles).apply(lambda x: sm.OLS(x, X[:len(x)]).fit().params[1], raw=False)
+        X = sm.add_constant(range(len(pd_candles['close'])))
+        rolling_slope = pd_candles['close'].rolling(window=sliding_window_how_many_candles).apply(lambda x: sm.OLS(x, X[:len(x)]).fit().params[1], raw=False)
         pd_candles['close_long_slope'] = rolling_slope
+        max_abs_slope = pd_candles['close_long_slope'].abs().rolling(window=sliding_window_how_many_candles).max()
+        pd_candles['normalized_close_short_slope'] = pd_candles['close_long_slope'] / (2 * max_abs_slope)
         
         X = sm.add_constant(range(len(pd_candles['ema_short_periods'])))
         rolling_slope = pd_candles['ema_short_periods'].rolling(window=int(sliding_window_how_many_candles/slow_fast_interval_ratio)).apply(lambda x: sm.OLS(x, X[:len(x)]).fit().params[1], raw=False)

{siglab_py-0.3.2 → siglab_py-0.3.3}/siglab_py/util/market_data_util.py

@@ -1,3 +1,4 @@
+import incremental
 import tzlocal
 from datetime import datetime, timezone
 from typing import List, Dict, Union, NoReturn, Any, Tuple
@@ -5,9 +6,12 @@ from pathlib import Path
 import math
 import pandas as pd
 import numpy as np
+import asyncio
 
 from ccxt.base.exchange import Exchange as CcxtExchange
 from ccxt import deribit
+import ccxt
+import ccxt.pro as ccxtpro
 
 # https://www.analyticsvidhya.com/blog/2021/06/download-financial-dataset-using-yahoo-finance-in-python-a-complete-guide/
 from yahoofinancials import YahooFinancials
@@ -16,6 +20,155 @@ from yahoofinancials import YahooFinancials
 import yfinance as yf
 
 from siglab_py.exchanges.futubull import Futubull
+from siglab_py.exchanges.any_exchange import AnyExchange
+
+def instantiate_exchange(
+    exchange_name : str,
+    api_key : Union[str, None] = None,
+    secret : Union[str, None]  = None,
+    passphrase : Union[str, None] = None,
+    default_type : Union[str, None] = 'spot',
+    rate_limit_ms : float = 100
+) -> Union[AnyExchange, None]:
+    exchange_name = exchange_name.lower().strip()
+
+    # Look at ccxt exchange.describe. under 'options' \ 'defaultType' (and 'defaultSubType') for what markets the exchange support.
+    # https://docs.ccxt.com/en/latest/manual.html#instantiation
+    exchange_params : Dict[str, Any]= {
+                        'apiKey' : api_key,
+                        'secret' : secret,
+                        'enableRateLimit'  : True,
+                        'rateLimit' : rate_limit_ms,
+                        'options' : {
+                            'defaultType' : default_type
+                        }
+                    }
+    if api_key:
+        exchange_params['apiKey'] = api_key
+    if secret:
+        exchange_params['secret'] = secret
+    if passphrase:
+        exchange_params['passphrase'] = passphrase
+
+    if exchange_name=='binance':
+        exchange = ccxt.binance(exchange_params)  # type: ignore
+    elif exchange_name=='bybit':
+        exchange = ccxt.bybit(exchange_params) # type: ignore
+    elif exchange_name=='okx':
+        exchange = ccxt.okx(exchange_params) # type: ignore
+    elif exchange_name=='deribit':
+        exchange = ccxt.deribit(exchange_params)  # type: ignore
+    elif exchange_name=='hyperliquid':
+        exchange = ccxt.hyperliquid(
+            {
+                "walletAddress" : api_key, # type: ignore
+                "privateKey" : secret, 
+                'enableRateLimit'  : True,
+                'rateLimit' : rate_limit_ms
+            }
+        )  
+    else:
+        raise ValueError(f"Unsupported exchange {exchange_name}.")
+
+    exchange.load_markets() # type: ignore
+
+    return exchange # type: ignore
+
+async def async_instantiate_exchange(
+    gateway_id : str,
+    api_key : str,
+    secret : str,
+    passphrase : str,
+    default_type : str,
+    rate_limit_ms : float = 100
+) -> Union[AnyExchange, None]:
+    exchange : Union[AnyExchange, None] = None
+    exchange_name : str = gateway_id.split('_')[0]
+    exchange_name =exchange_name.lower().strip()
+
+    # Look at ccxt exchange.describe. under 'options' \ 'defaultType' (and 'defaultSubType') for what markets the exchange support.
+    # https://docs.ccxt.com/en/latest/manual.html#instantiation
+    exchange_params : Dict[str, Any]= {
+                        'apiKey' : api_key,
+                        'secret' : secret,
+                        'enableRateLimit'  : True,
+                        'rateLimit' : rate_limit_ms,
+                        'options' : {
+                            'defaultType' : default_type
+                        }
+                    }
+
+    if exchange_name=='binance':
+        # spot, future, margin, delivery, option
+        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/binance.py#L1298
+        exchange = ccxtpro.binance(exchange_params)  # type: ignore
+    elif exchange_name=='bybit':
+        # spot, linear, inverse, futures
+        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/bybit.py#L1041
+        exchange = ccxtpro.bybit(exchange_params) # type: ignore
+    elif exchange_name=='okx':
+        # 'funding', spot, margin, future, swap, option
+        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/okx.py#L1144
+        exchange_params['password'] = passphrase
+        exchange = ccxtpro.okx(exchange_params) # type: ignore
+    elif exchange_name=='deribit':
+        # spot, swap, future
+        # https://github.com/ccxt/ccxt/blob/master/python/ccxt/deribit.py#L360
+        exchange = ccxtpro.deribit(exchange_params)  # type: ignore
+    elif exchange_name=='hyperliquid':
+        '''
+        https://app.hyperliquid.xyz/API
+        
+        defaultType from ccxt: swap
+            https://github.com/ccxt/ccxt/blob/master/python/ccxt/hyperliquid.py#L225
+        
+        How to integrate? You can skip first 6 min: https://www.youtube.com/watch?v=UuBr331wxr4&t=363s
+
+        Example, 
+            API credentials created under "\ More \ API":
+                    Ledger Arbitrum Wallet Address: 0xAAAAA <-- This is your Ledger Arbitrum wallet address with which you connect to Hyperliquid. 
+                    API Wallet Address 0xBBBBB <-- Generated
+                    privateKey 0xCCCCC
+
+        Basic connection via CCXT:
+            import asyncio
+            import ccxt.pro as ccxtpro
+
+            async def main():
+                rate_limit_ms = 100
+                exchange_params = {
+                    "walletAddress" : "0xAAAAA", # Ledger Arbitrum Wallet Address here! Not the generated address.
+                    "privateKey" : "0xCCCCC"
+                }
+                exchange = ccxtpro.hyperliquid(exchange_params) 
+                balances = await exchange.fetch_balance()
+                print(balances)
+
+            asyncio.run(main())
+        '''
+        exchange = ccxtpro.hyperliquid(
+            {
+                "walletAddress" : api_key,
+                "privateKey" : secret,
+                'enableRateLimit'  : True,
+                'rateLimit' : rate_limit_ms
+            }
+        )  # type: ignore
+    else:
+        raise ValueError(f"Unsupported exchange {exchange_name}, check gateway_id {gateway_id}.")
+
+    await exchange.load_markets() # type: ignore
+
+    '''
+    Is this necessary? The added trouble is for example bybit.authenticate requires arg 'url'. binance doesn't. And fetch_balance already test credentials.
+
+    try:
+        await exchange.authenticate() # type: ignore
+    except Exception as swallow_this_error:
+        pass
+    '''
+
+    return exchange
 
 def timestamp_to_datetime_cols(pd_candles : pd.DataFrame):
     def _fix_timestamp_ms(x):
@@ -61,6 +214,11 @@ def timestamp_to_datetime_cols(pd_candles : pd.DataFrame):
         lambda x: "AMER" in timestamp_to_active_trading_regions(int(x/1000))
     )
 
+    pd_candles['timestamp_ms_gap'] = pd_candles['timestamp_ms'] - pd_candles['timestamp_ms'].shift(1)
+    timestamp_ms_gap = pd_candles.iloc[-1]['timestamp_ms_gap']
+    assert(pd_candles[~pd_candles.timestamp_ms_gap.isna()][pd_candles.timestamp_ms_gap!=timestamp_ms_gap].shape[0]==0)
+    pd_candles.drop(columns=['timestamp_ms_gap'], inplace=True)
+
 def timestamp_to_active_trading_regions(
         timestamp_ms : int
 ) -> List[str]:
@@ -368,6 +526,20 @@ def _fetch_candles_ccxt(
 
             return candles
         
+    def _calc_increment(candle_size):
+        increment = 1
+        num_intervals = int(candle_size[0])
+        interval_type = candle_size[-1]
+        if interval_type == "m":
+            increment = 60
+        elif interval_type == "h":
+            increment = 60*60
+        elif interval_type == "d":
+            increment = 60*60*24
+        else:
+            raise ValueError(f"Invalid candle_size {candle_size}")
+        return num_intervals * increment
+    
     all_candles = []
     params = {}
     this_cutoff = start_ts
@@ -380,10 +552,10 @@ def _fetch_candles_ccxt(
             record_ts_str : str = str(record_ts)
             if len(record_ts_str)==13:
                 record_ts = int(int(record_ts_str)/1000) # Convert from milli-seconds to seconds
-
-            this_cutoff = record_ts  + 1
+            
+            this_cutoff = record_ts  + _calc_increment(candle_size)
         else:
-            this_cutoff += 1
+            this_cutoff += _calc_increment(candle_size)
 
     columns = ['exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume']
     pd_all_candles = pd.DataFrame([ [ exchange.name, ticker, x[0], x[1], x[2], x[3], x[4], x[5] ] for x in all_candles], columns=columns)

siglab_py-0.3.3/siglab_py/util/test_market_data_analytic_util.py

@@ -0,0 +1,52 @@
+from datetime import datetime
+from typing import Dict, Union
+
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from ccxt.okx import okx
+
+from market_data_util import fetch_candles
+from analytic_util import compute_candles_stats
+
+base_ccy : str = "BTC"
+# ticker = "GRIFFAIN/USDT:USDT"
+# ticker = "OL/USDT:USDT"
+ticker = f"{base_ccy}/USDT:USDT"
+
+param = {
+        'rateLimit' : 100,    # In ms
+        'options' : {
+            'defaultType': 'swap', # Should test linear instead
+        }
+    }
+exchange = okx(param) # type: ignore
+
+start_date : datetime = datetime(2024,1,1)
+end_date : datetime = datetime(2025,4,20)
+candle_size : str = '1h'
+ma_long_intervals : int = 24*30
+ma_short_intervals : int = 24
+boillenger_std_multiples : int = 2
+pypy_compatible : bool = False
+
+markets = exchange.load_markets()
+assert(ticker in markets)
+
+pd_candles: Union[pd.DataFrame, None] = fetch_candles(
+            start_ts=int(start_date.timestamp()),
+            end_ts=int(end_date.timestamp()),
+            exchange=exchange,
+            normalized_symbols=[ ticker ],
+            candle_size=candle_size
+        )[ ticker ]
+
+pd_candles.to_csv(f"{base_ccy}_raw_candles.csv") # type: ignore
+
+compute_candles_stats(
+                                    pd_candles=pd_candles, # type: ignore
+                                    boillenger_std_multiples=boillenger_std_multiples, 
+                                    sliding_window_how_many_candles=ma_long_intervals, 
+                                    slow_fast_interval_ratio=(ma_long_intervals/ma_short_intervals),
+                                    pypy_compat=pypy_compatible
+                                ) 

{siglab_py-0.3.2 → siglab_py-0.3.3}/siglab_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: siglab-py
-Version: 0.3.2
+Version: 0.3.3
 Summary: Market data fetches, TA calculations and generic order gateway.
 Author: r0bbarh00d
 Author-email: r0bbarh00d <r0bbarh00d@gmail.com>

{siglab_py-0.3.2 → siglab_py-0.3.3}/siglab_py.egg-info/SOURCES.txt

@@ -38,4 +38,5 @@ siglab_py/util/market_data_util.py
 siglab_py/util/notification_util.py
 siglab_py/util/retry_util.py
 siglab_py/util/slack_notification_util.py
+siglab_py/util/test_market_data_analytic_util.py
 siglab_py/util/trading_util.py