siglab-py 0.1.24__tar.gz → 0.1.26__tar.gz

{siglab_py-0.1.24 → siglab_py-0.1.26}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: siglab_py
-Version: 0.1.24
+Version: 0.1.26
 Summary: Market data fetches, TA calculations and generic order gateway.
 Author: r0bbarh00d
 Author-email: r0bbarh00d <r0bbarh00d@gmail.com>
@@ -25,8 +25,6 @@ Requires-Dist: ccxt
 Requires-Dist: ccxtpro
 Requires-Dist: yfinance
 Requires-Dist: yahoofinancials
-Requires-Dist: scikit-learn
-Requires-Dist: sklearn.preprocessing
 Requires-Dist: hurst
 Requires-Dist: redis
 Requires-Dist: redis-py-cluster

{siglab_py-0.1.24 → siglab_py-0.1.26}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "siglab_py"
-version = "0.1.24"
+version = "0.1.26"
 description = "Market data fetches, TA calculations and generic order gateway."
 authors = [{name = "r0bbarh00d", email = "r0bbarh00d@gmail.com"}]
 license = {text = "MIT"}
@@ -25,8 +25,6 @@ dependencies = [
     "ccxtpro",
     "yfinance",
     "yahoofinancials",
-    "scikit-learn",
-    "sklearn.preprocessing",
     "hurst",
     "redis",
     "redis-py-cluster",

{siglab_py-0.1.24 → siglab_py-0.1.26}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = siglab_py
-version = 0.1.24
+version = 0.1.26
 description = Market data fetches, TA calculations and generic order gateway.
 author = r0bbarh00d
 author_email = r0bbarh00d@gmail.com
@@ -28,8 +28,6 @@ install_requires =
 	ccxtpro
 	yfinance
 	yahoofinancials
-	scikit-learn
-	sklearn.preprocessing
 	hurst
 	redis
 	redis-py-cluster

siglab_py-0.1.26/siglab_py/market_data_providers/ccxt_candles_ta_to_csv.py

@@ -0,0 +1,227 @@
+from ctypes import ArgumentError
+import sys
+import logging
+import argparse
+from datetime import datetime, timedelta
+import time
+from typing import Dict, Union, Any
+from enum import Enum
+import asyncio
+import pandas as pd
+
+from ccxt.base.exchange import Exchange as CCXTExchange
+from ccxt.binance import binance
+from ccxt.bybit import bybit
+from ccxt.okx import okx
+from ccxt.deribit import deribit
+from ccxt.kraken import kraken
+from ccxt.hyperliquid import hyperliquid
+
+from siglab_py.exchanges.futubull import Futubull
+from siglab_py.util.market_data_util import fetch_candles
+# from util.market_data_util import fetch_candles # For debug only
+from siglab_py.util.analytic_util import compute_candles_stats
+# from util.analytic_util import compute_candles_stats # For debug only
+
+'''
+Usage:
+    set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab\siglab_py
+    python ccxt_candles_ta_to_csv.py --exchange_name bybit --symbol BTC/USDT:USDT --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --default_type linear --compute_ta Y --pypy_compatible N
+
+    (Remember: python -mpip install siglab_py)
+
+This script is pypy compatible. Set "pypy_compatible" to True, in which case "compute_candles_stats" will skip calculation for TAs which requires: scipy, statsmodels, scikit-learn, sklearn.preprocessing
+    pypy ccxt_candles_ta_to_csv.py --exchange_name bybit --symbol BTC/USDT:USDT --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --default_type linear --compute_ta Y --pypy_compatible Y
+
+    (Remember: pypy -mpip install siglab_py)
+
+If debugging from VSCode, launch.json:
+
+    {
+        "version": "0.2.0",
+        "configurations": [
+            {
+                "name": "Python Debugger: Current File",
+                "type": "debugpy",
+                "request": "launch",
+                "program": "${file}",
+                "console": "integratedTerminal",
+                "args" : [
+                        "--exchange_name", "bybit",
+                        "--symbol", "BTC/USDT:USDT",
+                        "--end_date", "2025-03-11 0:0:0",
+                        "--start_date", "2024-03-11 0:0:0",
+                        "--default_type", "linear",
+                        "--compute_ta", "Y",
+                        "--pypy_compatible", "N"
+                    ],
+                "env": {
+                    "PYTHONPATH": "${workspaceFolder}"
+                }
+            }
+        ]
+    }
+'''
+end_date : datetime = datetime.today()
+end_date = datetime(end_date.year, end_date.month, end_date.day)
+start_date : datetime = end_date - timedelta(days=365)
+
+param : Dict = {
+    'exchange' : 'bybit',
+    'symbol' : None,
+    'start_date' : start_date,
+    'end_date' : end_date,
+    'exchange_params' : {
+                        'rateLimit' : 100, # in ms
+                        'options' : {
+                            'defaultType' : "linear"
+                        }
+                    },
+    'output_filename' : 'candles_ta_$SYMBOL$.csv'
+}
+
+class LogLevel(Enum):
+    CRITICAL = 50
+    ERROR = 40
+    WARNING = 30
+    INFO = 20
+    DEBUG = 10
+    NOTSET = 0
+
+logging.Formatter.converter = time.gmtime
+logger = logging.getLogger()
+log_level = logging.INFO # DEBUG --> INFO --> WARNING --> ERROR
+logger.setLevel(log_level)
+format_str = '%(asctime)s %(message)s'
+formatter = logging.Formatter(format_str)
+sh = logging.StreamHandler()
+sh.setLevel(log_level)
+sh.setFormatter(formatter)
+logger.addHandler(sh)
+
+def log(message : str, log_level : LogLevel = LogLevel.INFO):
+    if log_level.value<LogLevel.WARNING.value:
+        logger.info(f"{datetime.now()} {message}")
+
+    elif log_level.value==LogLevel.WARNING.value:
+        logger.warning(f"{datetime.now()} {message}")
+
+    elif log_level.value==LogLevel.ERROR.value:
+        logger.error(f"{datetime.now()} {message}")
+
+def parse_args():
+    parser = argparse.ArgumentParser() # type: ignore
+    parser.add_argument("--exchange_name", help="Exchange name. bybit, okx, bybit, deribit, hyperliquid ...etc, add whatever you want top of script, import them. Then add to instantiate_exchange.", default="bybit")
+    parser.add_argument("--symbol", help="symbol, CEX example BTC/USDT for spot. BTC/USDT:USDT for perpetuals. Many DEXes offer USDC pairs.", default="BTC/USDT:USDT")
+    parser.add_argument("--start_date", help="Format: %Y-%m-%d %H:%M:%S", default=None)
+    parser.add_argument("--end_date", help="Format: %Y-%m-%d %H:%M:%S", default=None)
+
+    '''
+    Enums here: 
+    https://openapi.futunn.com/futu-api-doc/en/quote/quote.html#66
+    https://openapi.futunn.com/futu-api-doc/en/trade/trade.html#9434
+    '''
+    parser.add_argument("--default_type", help="Depends on exchange. Very often, spot, linear/swap for perpetuals. Have a look at gateway.py instantiate_exchange https://github.com/r0bbar/siglab/blob/master/siglab_py/ordergateway/gateway.py", default="linear")
+
+    parser.add_argument("--compute_ta", help="Compute technical indicators?. Y or N (default).", default='N')
+    parser.add_argument("--candle_size", help="candle interval: 1m, 1h, 1d... etc", default='1h')
+    parser.add_argument("--ma_long_intervals", help="Sliding Window size in number of intervals for higher timeframe", default=24)
+    parser.add_argument("--ma_short_intervals", help="Sliding Window size in number of intervals for lower timeframe", default=8)
+    parser.add_argument("--boillenger_std_multiples", help="Boillenger bands: # std", default=2)
+
+    parser.add_argument("--pypy_compatible", help="pypy_compatible: If Y, analytic_util will import statsmodels.api (slopes and divergence calc). In any case, partition_sliding_window requires scipy.stats.linregress and cannot be used with pypy. Y or N (default).", default='N')
+
+    args = parser.parse_args()
+    param['exchange_name'] = args.exchange_name.strip().lower()
+    param['symbol'] = args.symbol.strip().upper()
+
+    param['start_date'] = datetime.strptime(args.start_date, "%Y-%m-%d %H:%M:%S") if args.start_date else start_date
+    param['end_date'] = datetime.strptime(args.end_date, "%Y-%m-%d %H:%M:%S") if args.end_date else end_date
+    
+    param['exchange_params']['options']['defaultType'] = args.default_type
+
+    param['output_filename'] = param['output_filename'].replace('$SYMBOL$', param['symbol'].replace(":",".").replace("/","."))
+
+    if args.compute_ta:
+        if args.compute_ta=='Y':
+            param['compute_ta'] = True
+        else:
+            param['compute_ta'] = False
+    else:
+        param['compute_ta'] = False
+    param['candle_size'] = args.candle_size
+    param['ma_long_intervals'] = int(args.ma_long_intervals)
+    param['ma_short_intervals'] = int(args.ma_short_intervals)
+    param['boillenger_std_multiples'] = int(args.boillenger_std_multiples)
+
+    if args.pypy_compatible:
+        if args.pypy_compatible=='Y':
+            param['pypy_compatible'] = True
+        else:
+            param['pypy_compatible'] = False
+    else:
+        param['pypy_compatible'] = False
+
+def instantiate_exchange(
+    exchange_name : str,
+    exchange_params : Dict[str, Any]
+) -> CCXTExchange:
+    if exchange_name=='binance':
+        return binance(exchange_params)
+    elif exchange_name=='bybit':
+        return bybit(exchange_params)
+    elif exchange_name=='okx':
+        return okx(exchange_params)
+    elif exchange_name=='deribit':
+        return deribit(exchange_params)
+    else:
+        raise ArgumentError(f"Unsupported exchange {exchange_name}. Please import top of script and add to instantiate_exchange.")
+    
+async def main():
+    parse_args()
+
+    fh = logging.FileHandler(f"ccxt_candles_ta_to_csv.log")
+    fh.setLevel(log_level)
+    fh.setFormatter(formatter)     
+    logger.addHandler(fh) # type: ignore
+
+    exchange = instantiate_exchange(param['exchange_name'], param['exchange_params'])
+    markets = exchange.load_markets()
+    if param['symbol'] not in markets:
+        raise ArgumentError(f"{param['symbol']} not support by {param['exchange_name']}")
+    
+    pd_candles: Union[pd.DataFrame, None] = fetch_candles(
+            start_ts=int(param['start_date'].timestamp()),
+            end_ts=int(param['end_date'].timestamp()),
+            exchange=exchange,
+            normalized_symbols=[ param['symbol'] ],
+            candle_size=param['candle_size']
+        )[param['symbol']]
+
+    assert pd_candles is not None
+
+    if pd_candles is not None:
+        assert len(pd_candles) > 0, "No candles returned."
+        expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute', 'week_of_month', 'apac_trading_hr', 'emea_trading_hr', 'amer_trading_hr'}
+        assert set(pd_candles.columns) >= expected_columns, "Missing expected columns."
+        assert pd_candles['timestamp_ms'].notna().all(), "timestamp_ms column contains NaN values."
+        assert pd_candles['timestamp_ms'].is_monotonic_increasing, "Timestamps are not in ascending order."
+
+    if param['compute_ta']:
+        start = time.time()
+        compute_candles_stats(
+                                    pd_candles=pd_candles, 
+                                    boillenger_std_multiples=param['boillenger_std_multiples'], 
+                                    sliding_window_how_many_candles=param['ma_long_intervals'], 
+                                    slow_fast_interval_ratio=(param['ma_long_intervals']/param['ma_short_intervals']),
+                                    pypy_compat=param['pypy_compatible']
+                                )
+        compute_candles_stats_elapsed_ms = int((time.time() - start) *1000)
+        log(f"TA calculated, took {compute_candles_stats_elapsed_ms} ms")
+
+    log(f"Candles (# rows: {pd_candles.shape[0]}) written to {param['output_filename']}")
+    pd_candles.to_csv(param['output_filename'])
+
+    sys.exit()
+
+asyncio.run(main())

siglab_py-0.1.24/siglab_py/market_data_providers/futu_candles.py → siglab_py-0.1.26/siglab_py/market_data_providers/futu_candles_ta_to_csv.py

@@ -1,10 +1,12 @@
 import sys
 import logging
 import argparse
-from datetime import datetime
+from datetime import datetime, timedelta
+import time
 from typing import Dict, Union
 from enum import Enum
 import asyncio
+import pandas as pd
 
 from futu import *
 
@@ -14,9 +16,17 @@ from siglab_py.util.analytic_util import compute_candles_stats
 
 '''
 Usage:
-    python futu_candles.py --symbol HK.00700 --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --market HK --trdmarket HK --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y
+    set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab\siglab_py
+    python futu_candles_ta_to_csv.py --symbol HK.00700 --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --market HK --trdmarket HK --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible N
 
-This script is NOT pypy compatible.
+    python futu_candles_ta_to_csv.py --symbol AAPL --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --market US --trdmarket US --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible N
+
+    (Remember: python -mpip install siglab_py)
+
+This script is pypy compatible. Set "pypy_compatible" to True, in which case "compute_candles_stats" will skip calculation for TAs which requires: scipy, statsmodels, scikit-learn, sklearn.preprocessing
+    pypy futu_candles_ta_to_csv.py --symbol HK.00700 --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --market HK --trdmarket HK --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible Y
+
+    (Remember: pypy -mpip install siglab_py)
 
 If debugging from VSCode, launch.json:
 
@@ -37,7 +47,8 @@ If debugging from VSCode, launch.json:
                         "--trdmarket", "HK",
                         "--security_firm", "FUTUSECURITIES",
                         "--security_type", "STOCK",
-                        "--compute_ta", "Y"
+                        "--compute_ta", "Y",
+                        "--pypy_compatible", "N"
                     ],
                 "env": {
                     "PYTHONPATH": "${workspaceFolder}"
@@ -62,7 +73,7 @@ param : Dict = {
         'host' : '127.0.0.1',
         'port' : 11111
     },
-    'output_filename' : 'candles_$SYMBOL$.csv'
+    'output_filename' : 'candles_ta_$SYMBOL$.csv'
 }
 
 class LogLevel(Enum):
@@ -112,10 +123,12 @@ def parse_args():
 
     parser.add_argument("--compute_ta", help="Compute technical indicators?. Y or N (default).", default='N')
     parser.add_argument("--candle_size", help="candle interval: 1m, 1h, 1d... etc", default='1h')
-    parser.add_argument("--ma_long_intervals", help="Window size in number of intervals for higher timeframe", default=24)
-    parser.add_argument("--ma_short_intervals", help="Window size in number of intervals for lower timeframe", default=8)
+    parser.add_argument("--ma_long_intervals", help="Sliding Window size in number of intervals for higher timeframe", default=24)
+    parser.add_argument("--ma_short_intervals", help="Sliding Window size in number of intervals for lower timeframe", default=8)
     parser.add_argument("--boillenger_std_multiples", help="Boillenger bands: # std", default=2)
 
+    parser.add_argument("--pypy_compatible", help="pypy_compatible: If Y, analytic_util will import statsmodels.api (slopes and divergence calc). In any case, partition_sliding_window requires scipy.stats.linregress and cannot be used with pypy. Y or N (default).", default='N')
+
     args = parser.parse_args()
     param['symbol'] = args.symbol.strip().upper()
 
@@ -140,10 +153,18 @@ def parse_args():
     param['ma_short_intervals'] = int(args.ma_short_intervals)
     param['boillenger_std_multiples'] = int(args.boillenger_std_multiples)
 
+    if args.pypy_compatible:
+        if args.pypy_compatible=='Y':
+            param['pypy_compatible'] = True
+        else:
+            param['pypy_compatible'] = False
+    else:
+        param['pypy_compatible'] = False
+
 async def main():
     parse_args()
 
-    fh = logging.FileHandler(f"futu_candles.log")
+    fh = logging.FileHandler(f"futu_candles_ta_to_csv.log")
     fh.setLevel(log_level)
     fh.setFormatter(formatter)     
     logger.addHandler(fh) # type: ignore
@@ -151,18 +172,18 @@ async def main():
     exchange = Futubull(param)
 
     pd_candles: Union[pd.DataFrame, None] = fetch_candles(
-            start_ts=int(start_date.timestamp()),
-            end_ts=int(end_date.timestamp()),
+            start_ts=int(param['start_date'].timestamp()),
+            end_ts=int(param['end_date'].timestamp()),
             exchange=exchange,
             normalized_symbols=[ param['symbol'] ],
-            candle_size='1d'
+            candle_size=param['candle_size']
         )[param['symbol']]
 
     assert pd_candles is not None
 
     if pd_candles is not None:
         assert len(pd_candles) > 0, "No candles returned."
-        expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute'}
+        expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute', 'week_of_month', 'apac_trading_hr', 'emea_trading_hr', 'amer_trading_hr'}
         assert set(pd_candles.columns) >= expected_columns, "Missing expected columns."
         assert pd_candles['timestamp_ms'].notna().all(), "timestamp_ms column contains NaN values."
         assert pd_candles['timestamp_ms'].is_monotonic_increasing, "Timestamps are not in ascending order."
@@ -174,7 +195,7 @@ async def main():
                                     boillenger_std_multiples=param['boillenger_std_multiples'], 
                                     sliding_window_how_many_candles=param['ma_long_intervals'], 
                                     slow_fast_interval_ratio=(param['ma_long_intervals']/param['ma_short_intervals']),
-                                    pypy_compat=False
+                                    pypy_compat=param['pypy_compatible']
                                 )
         compute_candles_stats_elapsed_ms = int((time.time() - start) *1000)
         log(f"TA calculated, took {compute_candles_stats_elapsed_ms} ms")

{siglab_py-0.1.24 → siglab_py-0.1.26}/siglab_py/tests/integration/market_data_util_tests.py

@@ -36,7 +36,7 @@ class MarketDataUtilTests(unittest.TestCase):
 
         if pd_candles is not None:
             assert len(pd_candles) > 0, "No candles returned."
-            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute'}
+            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute', 'week_of_month', 'apac_trading_hr', 'emea_trading_hr', 'amer_trading_hr'}
             assert set(pd_candles.columns) >= expected_columns, "Missing expected columns."
             assert pd_candles['timestamp_ms'].notna().all(), "timestamp_ms column contains NaN values."
             assert pd_candles['timestamp_ms'].is_monotonic_increasing, "Timestamps are not in ascending order."
@@ -81,7 +81,7 @@ class MarketDataUtilTests(unittest.TestCase):
 
         if pd_candles is not None:
             assert len(pd_candles) > 0, "No candles returned."
-            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute'}
+            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute', 'week_of_month', 'apac_trading_hr', 'emea_trading_hr', 'amer_trading_hr'}
             assert set(pd_candles.columns) >= expected_columns, "Missing expected columns."
             assert pd_candles['timestamp_ms'].notna().all(), "timestamp_ms column contains NaN values."
             assert pd_candles['timestamp_ms'].is_monotonic_increasing, "Timestamps are not in ascending order."
@@ -114,12 +114,13 @@ class MarketDataUtilTests(unittest.TestCase):
 
         if pd_candles is not None:
             assert len(pd_candles) > 0, "No candles returned."
-            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute'}
+            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute', 'week_of_month', 'apac_trading_hr', 'emea_trading_hr', 'amer_trading_hr'}
             assert set(pd_candles.columns) >= expected_columns, "Missing expected columns."
             assert pd_candles['timestamp_ms'].notna().all(), "timestamp_ms column contains NaN values."
             assert pd_candles['timestamp_ms'].is_monotonic_increasing, "Timestamps are not in ascending order."
         
     def test_fetch_candles_futubull(self):
+        # You need Futu OpenD running and you need entitlements
         end_date : datetime = datetime.today()
         start_date : datetime = end_date - timedelta(days=365*3)
 
@@ -148,7 +149,7 @@ class MarketDataUtilTests(unittest.TestCase):
 
         if pd_candles is not None:
             assert len(pd_candles) > 0, "No candles returned."
-            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute'}
+            expected_columns = {'exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime_utc', 'datetime', 'year', 'month', 'day', 'hour', 'minute', 'week_of_month', 'apac_trading_hr', 'emea_trading_hr', 'amer_trading_hr'}
             assert set(pd_candles.columns) >= expected_columns, "Missing expected columns."
             assert pd_candles['timestamp_ms'].notna().all(), "timestamp_ms column contains NaN values."
             assert pd_candles['timestamp_ms'].is_monotonic_increasing, "Timestamps are not in ascending order."

{siglab_py-0.1.24 → siglab_py-0.1.26}/siglab_py/tests/unit/analytic_util_tests.py

@@ -41,5 +41,5 @@ class AnalyticUtilTests(unittest.TestCase):
             pypy_compat=True
         )
 
-        expected_columns : List[str] = ['exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime', 'datetime_utc', 'year', 'month', 'day', 'hour', 'minute', 'dayofweek', 'pct_chg_on_close', 'candle_height', 'is_green', 'pct_change_close', 'sma_short_periods', 'sma_long_periods', 'ema_short_periods', 'ema_long_periods', 'ema_close', 'std', 'ema_volume_short_periods', 'ema_volume_long_periods', 'max_short_periods', 'max_long_periods', 'idmax_short_periods', 'idmax_long_periods', 'min_short_periods', 'min_long_periods', 'idmin_short_periods', 'idmin_long_periods', 'h_l', 'h_pc', 'l_pc', 'tr', 'atr', 'hurst_exp', 'boillenger_upper', 'boillenger_lower', 'boillenger_channel_height', 'boillenger_upper_agg', 'boillenger_lower_agg', 'boillenger_channel_height_agg', 'aggressive_up', 'aggressive_up_index', 'aggressive_up_candle_height', 'aggressive_up_candle_high', 'aggressive_up_candle_low', 'aggressive_down', 'aggressive_down_index', 'aggressive_down_candle_height', 'aggressive_down_candle_high', 'aggressive_down_candle_low', 'fvg_low', 'fvg_high', 'fvg_gap', 'fvg_mitigated', 'close_delta', 'close_delta_percent', 'up', 'down', 'rsi', 'ema_rsi', 'typical_price', 'money_flow', 'money_flow_positive', 'money_flow_negative', 'positive_flow_sum', 'negative_flow_sum', 'money_flow_ratio', 'mfi', 'macd', 'signal', 'macd_minus_signal', 'fib_618_short_periods', 'fib_618_long_periods', 'gap_close_vs_ema', 'close_above_or_below_ema', 'close_vs_ema_inflection']
+        expected_columns : List[str] = ['exchange', 'symbol', 'timestamp_ms', 'open', 'high', 'low', 'close', 'volume', 'datetime', 'datetime_utc', 'year', 'month', 'day', 'hour', 'minute', 'dayofweek', 'pct_chg_on_close', 'candle_height', 'is_green', 'pct_change_close', 'sma_short_periods', 'sma_long_periods', 'ema_short_periods', 'ema_long_periods', 'ema_close', 'std', 'candle_height_percent', 'chop_against_ema', 'ema_volume_short_periods', 'ema_volume_long_periods', 'max_short_periods', 'max_long_periods', 'idmax_short_periods', 'idmax_long_periods', 'min_short_periods', 'min_long_periods', 'idmin_short_periods', 'idmin_long_periods', 'h_l', 'h_pc', 'l_pc', 'tr', 'atr', 'hurst_exp', 'boillenger_upper', 'boillenger_lower', 'boillenger_channel_height', 'boillenger_upper_agg', 'boillenger_lower_agg', 'boillenger_channel_height_agg', 'aggressive_up', 'aggressive_up_index', 'aggressive_up_candle_height', 'aggressive_up_candle_high', 'aggressive_up_candle_low', 'aggressive_down', 'aggressive_down_index', 'aggressive_down_candle_height', 'aggressive_down_candle_high', 'aggressive_down_candle_low', 'fvg_low', 'fvg_high', 'fvg_gap', 'fvg_mitigated', 'close_delta', 'close_delta_percent', 'up', 'down', 'rsi', 'ema_rsi', 'typical_price', 'money_flow', 'money_flow_positive', 'money_flow_negative', 'positive_flow_sum', 'negative_flow_sum', 'money_flow_ratio', 'mfi', 'macd', 'signal', 'macd_minus_signal', 'fib_618_short_periods', 'fib_618_long_periods', 'gap_close_vs_ema', 'close_above_or_below_ema', 'close_vs_ema_inflection']
         assert(pd_candles.columns.to_list()==expected_columns)

siglab_py-0.1.26/siglab_py/tests/unit/market_data_util_tests.py

@@ -0,0 +1,52 @@
+import unittest
+from datetime import datetime, timedelta
+from typing import Union
+from pathlib import Path
+
+from util.market_data_util import *
+
+from futu import *
+
+
+# @unittest.skip("Skip all integration tests.")
+class MarketDataUtilTests(unittest.TestCase):
+    def test_timestamp_to_week_of_month(self):
+        test_timestamps = [
+            1672531200000,  # 2023-01-01 (Week 0)
+            1673136000000,  # 2023-01-08 (Week 1)
+            1673740800000,  # 2023-01-15 (Week 2)
+            1674345600000,  # 2023-01-22 (Week 3)
+            1674950400000,  # 2023-01-29 (Week 4)
+            1675468800000,  # 2023-02-01 (Week 0)
+            1676073600000,  # 2023-02-08 (Week 1)
+            1676678400000,  # 2023-02-15 (Week 2)
+            1677283200000,  # 2023-02-22 (Week 3)
+            1677888000000,  # 2023-03-01 (Week 0)
+        ]
+
+        expectations = [0, 1, 2, 3, 4, 0, 1, 2, 3, 0]
+        
+        for i, ts in enumerate(test_timestamps):
+            expectation = expectations[i]
+            actual = timestamp_to_week_of_month(ts)
+            assert expectation == actual, f"Test failed for timestamp {ts}. Expected: {expectation}, Actual: {actual}"
+            
+    def test_timestamp_to_active_trading_regions(self):
+        test_timestamps = [
+            1672531200000,  # 2023-01-01 00:00:00 UTC (APAC)
+            1672563600000,  # 2023-01-01 09:00:00 UTC (APAC, EMEA)
+            1672574400000,  # 2023-01-01 12:00:00 UTC (EMEA)
+            1672588800000,  # 2023-01-01 16:00:00 UTC (EMEA, AMER)
+            1672599600000,  # 2023-01-01 19:00:00 UTC (AMER)
+            1672610400000,  # 2023-01-01 22:00:00 UTC (APAC)
+        ]
+
+        expectations = [ ['APAC'], ['APAC', 'EMEA'], ['EMEA'], ['EMEA','AMER'], ['AMER'], ['APAC']]
+
+        i = 0
+        for ts in test_timestamps:
+            expectation = expectations[i]
+            actual = timestamp_to_active_trading_regions(ts)
+            assert(expectation==actual)
+            i+=1
+        

{siglab_py-0.1.24 → siglab_py-0.1.26}/siglab_py/util/analytic_util.py

@@ -94,6 +94,13 @@ def compute_candles_stats(
     pd_candles['ema_close'] = pd_candles['ema_long_periods'] # Alias, shorter name
     pd_candles['std'] = pd_candles['close'].rolling(window=sliding_window_how_many_candles).std()
 
+    pd_candles['candle_height_percent'] = pd_candles['candle_height'] / pd_candles['ema_close'] * 100
+    
+    pd_candles['chop_against_ema'] = (
+        (pd_candles['is_green'] & (pd_candles['close'] > pd_candles['ema_close'])) |  # Case 1: Green candle and close > EMA
+        (~pd_candles['is_green'] & (pd_candles['close'] < pd_candles['ema_close']))   # Case 2: Red candle and close < EMA
+    )
+
     pd_candles['ema_volume_short_periods'] = pd_candles['volume'].ewm(span=sliding_window_how_many_candles/slow_fast_interval_ratio, adjust=False).mean()
     pd_candles['ema_volume_long_periods'] = pd_candles['volume'].ewm(span=sliding_window_how_many_candles, adjust=False).mean()
 

{siglab_py-0.1.24 → siglab_py-0.1.26}/siglab_py/util/market_data_util.py

@@ -35,6 +35,66 @@ def timestamp_to_datetime_cols(pd_candles : pd.DataFrame):
     pd_candles['minute'] = pd_candles['datetime'].dt.minute
     pd_candles['dayofweek'] = pd_candles['datetime'].dt.dayofweek  # dayofweek: Monday is 0 and Sunday is 6
 
+    pd_candles['week_of_month'] = pd_candles['timestamp_ms'].apply(
+        lambda x: timestamp_to_week_of_month(x)
+    )
+
+    pd_candles['apac_trading_hr'] = pd_candles['timestamp_ms'].apply(
+        lambda x: "APAC" in timestamp_to_active_trading_regions(x)
+    )
+    pd_candles['emea_trading_hr'] = pd_candles['timestamp_ms'].apply(
+        lambda x: "EMEA" in timestamp_to_active_trading_regions(x)
+    )
+    pd_candles['amer_trading_hr'] = pd_candles['timestamp_ms'].apply(
+        lambda x: "AMER" in timestamp_to_active_trading_regions(x)
+    )
+
+def timestamp_to_active_trading_regions(
+        timestamp_ms : int
+) -> List[str]:
+    
+    '''
+    APAC (Asia-Pacific) Trading Hours
+        UTC 22:00 - 09:00 (approximate range)
+        Major financial centers: Tokyo, Hong Kong, Singapore, Sydney
+
+    EMEA (Europe, Middle East, Africa) Trading Hours
+        UTC 07:00 - 16:00 (approximate range)
+        Major financial centers: London, Frankfurt, Paris, Zurich, Dubai
+
+    US Trading Hours
+        UTC 13:30 - 20:00 (approximate range)
+        Major financial centers: New York, Chicago
+        Key markets: NYSE, NASDAQ
+
+    utcnow and utcfromtimestamp been deprecated in Python 3.12 
+    https://www.pythonmorsels.com/converting-to-utc-time/
+    '''
+    active_trading_regions : List[str] = []
+
+    dt_utc = datetime.fromtimestamp(int(timestamp_ms / 1000), tz=timezone.utc)
+    utc_hour = dt_utc.hour
+    if (utc_hour >= 22) or (utc_hour <= 9):
+        active_trading_regions.append("APAC") 
+
+    if 7 <= utc_hour <= 16:
+        active_trading_regions.append("EMEA")
+
+    if 13 <= utc_hour <= 20:
+        active_trading_regions.append("AMER")
+
+    return active_trading_regions
+
+def timestamp_to_week_of_month(timestamp_ms: int) -> int:
+    """
+    Returns:
+        int: Week of the month (0 = first week, 1 = second week, etc.).
+    """
+    dt = datetime.fromtimestamp(timestamp_ms / 1000)
+    day_of_month = dt.day
+    week_of_month = (day_of_month - 1) // 7
+    return week_of_month
+
 def fix_column_types(pd_candles : pd.DataFrame):
     pd_candles['open'] = pd_candles['open'].astype(float)
     pd_candles['high'] = pd_candles['high'].astype(float)

{siglab_py-0.1.24 → siglab_py-0.1.26}/siglab_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: siglab-py
-Version: 0.1.24
+Version: 0.1.26
 Summary: Market data fetches, TA calculations and generic order gateway.
 Author: r0bbarh00d
 Author-email: r0bbarh00d <r0bbarh00d@gmail.com>
@@ -25,8 +25,6 @@ Requires-Dist: ccxt
 Requires-Dist: ccxtpro
 Requires-Dist: yfinance
 Requires-Dist: yahoofinancials
-Requires-Dist: scikit-learn
-Requires-Dist: sklearn.preprocessing
 Requires-Dist: hurst
 Requires-Dist: redis
 Requires-Dist: redis-py-cluster

{siglab_py-0.1.24 → siglab_py-0.1.26}/siglab_py.egg-info/SOURCES.txt

@@ -14,8 +14,9 @@ siglab_py/market_data_providers/__init__.py
 siglab_py/market_data_providers/aggregated_orderbook_provider.py
 siglab_py/market_data_providers/candles_provider.py
 siglab_py/market_data_providers/candles_ta_provider.py
+siglab_py/market_data_providers/ccxt_candles_ta_to_csv.py
 siglab_py/market_data_providers/deribit_options_expiry_provider.py
-siglab_py/market_data_providers/futu_candles.py
+siglab_py/market_data_providers/futu_candles_ta_to_csv.py
 siglab_py/market_data_providers/orderbooks_provider.py
 siglab_py/market_data_providers/test_provider.py
 siglab_py/ordergateway/__init__.py
@@ -28,6 +29,7 @@ siglab_py/tests/integration/__init__.py
 siglab_py/tests/integration/market_data_util_tests.py
 siglab_py/tests/unit/__init__.py
 siglab_py/tests/unit/analytic_util_tests.py
+siglab_py/tests/unit/market_data_util_tests.py
 siglab_py/util/__init__.py
 siglab_py/util/analytic_util.py
 siglab_py/util/aws_util.py

{siglab_py-0.1.24 → siglab_py-0.1.26}/siglab_py.egg-info/requires.txt

@@ -12,8 +12,6 @@ ccxt
 ccxtpro
 yfinance
 yahoofinancials
-scikit-learn
-sklearn.preprocessing
 hurst
 redis
 redis-py-cluster