siglab-py 0.1.29__py3-none-any.whl → 0.6.12__py3-none-any.whl

siglab_py/constants.py

@@ -1,3 +1,28 @@
+import enum
 from typing import Union, List, Dict, Any
 
-JSON_SERIALIZABLE_TYPES = Union[str, bool, int, float, None, List[Any], Dict[Any, Any]]
+INVALID : int = -1
+
+JSON_SERIALIZABLE_TYPES = Union[str, bool, int, float, None, List[Any], Dict[Any, Any]]
+
+class LogLevel(enum.Enum):
+    CRITICAL = 50
+    ERROR = 40
+    WARNING = 30
+    INFO = 20
+    DEBUG = 10
+    NOTSET = 0
+
+class TrendDirection(enum.Enum):
+    UNDEFINED = 0
+    HIGHER_HIGHS = 1
+    LOWER_HIGHS = 2
+    SIDEWAYS = 3
+    HIGHER_LOWS = 4
+    LOWER_LOWS = 5
+
+    def to_string(self) -> str:
+        return self.name.lower() if self != TrendDirection.UNDEFINED else ''
+
+OrderSide = enum.Enum('OrderSide', 'UNDEFINED BUY SELL')
+PositionStatus = enum.Enum("PositionStatus", 'UNDEFINED OPEN CLOSED SL')

siglab_py/exchanges/binance.py

@@ -0,0 +1,38 @@
+from typing import Dict, Any
+
+import ccxt
+from ccxt.base.types import Balances
+import ccxt.pro as ccxtpro
+
+'''
+Why override fetch_balance?
+    balances['total'] empty
+But you can find that from exchange raw response under balances['info']['balances'] (verbose=True).
+'''
+def _populate_balance_total_if_missing(
+        balances : Dict[str, Any]
+    ):
+    for ccy_balance in balances['info']['balances']:
+        ccy = ccy_balance['asset']
+        free = float(ccy_balance.get('free', 0))
+        locked = float(ccy_balance.get('locked', 0))
+        total = free + locked
+        if total!=0 and ccy not in balances['total']:
+            balances['total'][ccy] = total
+class Binance(ccxt.binance):
+    def __init__(self, *args: Dict[str, Any]) -> None:
+        super().__init__(*args) # type: ignore
+    
+    def fetch_balance(self, params={}) -> Balances: # type: ignore
+        balances = super().fetch_balance(params=params)
+        _populate_balance_total_if_missing(balances)
+        return balances
+
+class BinanceAsync(ccxtpro.binance):
+    def __init__(self, *args: Dict[str, Any]) -> None:
+        super().__init__(*args) # type: ignore
+
+    async def fetch_balance(self, params={}) -> Balances: # type: ignore
+        balances = await super().fetch_balance(params=params)
+        _populate_balance_total_if_missing(balances)
+        return balances

siglab_py/exchanges/deribit.py

@@ -0,0 +1,83 @@
+from typing import Dict, Any
+
+import ccxt
+import ccxt.pro as ccxtpro
+
+'''
+Why override load_markets?
+    Deribit is one of the OG CEX. If you look at 'contractSize' for BTC/USDC:USDC, a linear perp, for example:
+        exchange.markets['BTC/USDC:USDC']['contractSize'] 0.0001
+    However, for Deribit, create_order actually expects 'amount' in base ccy, not in "# of contracts" as with most other exchanges supported by CCXT.
+    Also note that 'filled' in response from create_order also in base ccy, not in # contracts.
+    The general prevailing convention in CCXT is: 'amount' should be quoted in '# contracts'.
+    Why CCXT not fix Deribit, so that it follows the prevailing convention? This is because this would be a breaking changes.
+    Thus, we override 'contractSize' to 1 for all markets.
+
+Additionally, we need to override 'fetch_position' as it swapped 'notional' with 'contracts'!!! Real ugly. Example below.
+    'id' = None
+    'symbol' = 'BTC/USDC:USDC'
+    'timestamp' = None
+    'datetime' = None
+    'lastUpdateTimestamp' = None
+    'initialMargin' = ???
+    'initialMarginPercentage' = ???
+    'maintenanceMargin' = ???
+    'maintenanceMarginPercentage' = ???
+    'entryPrice' = 85657.0
+    'notional' = 0.0009	<-- This is NOT USD! And this is NOT # Contracts! This is # BTC!
+    'leverage' = 50
+    'unrealizedPnl' = ???
+    'realizedPnl' = ???
+    'contracts' = 77.081445	<-- This is NOT "# contracts"! 0.0009 BTC x markPrice 85646.05
+    'contractSize' = 1.0
+    'marginRatio' = None
+    'liquidationPrice' = None
+    'markPrice' = 85646.05  <-- They use 'markPrice' to calc 'contracts'
+    'lastPrice' = None
+    'collateral' = None
+    'marginMode' = None
+    'side' = 'long'
+    'percentage' = None
+    'hedged' = None
+    'stopLossPrice' = None
+    'takeProfitPrice' = None
+'''
+class Deribit(ccxt.deribit):
+    def __init__(self, *args: Dict[str, Any]) -> None:
+        super().__init__(*args) # type: ignore
+
+    def load_markets(self, reload=False, params={}):
+        self.markets = super().load_markets(reload=reload, params=params)
+
+        for market in self.markets:
+            self.markets[market]['contractSize'] = 1
+
+        return self.markets
+    
+    def fetch_position(self, symbol: str, params={}): # type: ignore
+        position = super().fetch_position(symbol=symbol, params=params)
+        pos_usdt = position['contracts']
+        pos_baseccy = position['notional']
+        position['contracts'] = pos_baseccy
+        position['notional'] = pos_usdt
+        return position
+
+class DeribitAsync(ccxtpro.deribit):
+    def __init__(self, *args: Dict[str, Any]) -> None:
+        super().__init__(*args) # type: ignore
+
+    async def load_markets(self, reload=False, params={}):
+        self.markets = await super().load_markets(reload=reload, params=params)
+
+        for market in self.markets:
+            self.markets[market]['contractSize'] = 1
+            
+        return self.markets
+
+    async def fetch_position(self, symbol: str, params={}): # type: ignore
+        position = await super().fetch_position(symbol=symbol, params=params)
+        pos_usdt = position['contracts']
+        pos_baseccy = position['notional']
+        position['contracts'] = pos_baseccy
+        position['notional'] = pos_usdt
+        return position

siglab_py/exchanges/futubull.py

@@ -4,7 +4,7 @@ https://www.futuhk.com/en/support/categories/909?global_content=%7B%22promote_id
 
 Fees: https://www.futuhk.com/en/commissionnew#crypto
 
-Subscribe L2 data: https://openapi.futunn.com/futu-api-doc/en/intro/authority.html#5331
+Subscribe L2 data: https://openapi.futunn.com/futu-api-doc/en/intro/authority.html
 
 Investor Protection: https://www.futuhk.com/en
 
@@ -15,8 +15,17 @@ Margin Trading:
 Download Futu OpenD
     https://www.futuhk.com/en/support/topic1_464?global_content=%7B%22promote_id%22%3A13765%2C%22sub_promote_id%22%3A10%7D
 
-    If you run the installer version "Futu_OpenD-GUI_9.0.5008_Windows.exe", it'd be installed under:
+    If you run the installer version "Futu_OpenD-GUI_9.0.5008_Windows.exe", it'd be installed under C-Drive:
         C:\\Users\\xxx\\AppData\\Roaming\\Futu_OpenD\\Futu_OpenD.exe
+    Unfortunately, log folder also under C-drive as a result, and they are big.
+
+    For command line version: https://openapi.futunn.com/futu-api-doc/opend/opend-cmd.html
+        Binary under downloaded package (You can put it under for example D-drive): 
+            ...\Futu_OpenD_9.4.5408_Windows\Futu_OpenD_9.4.5408_Windows
+            
+        Put a batch file "start_futu_opend.bat", if login_pwd include special characters, enclose pwd with double quotes:
+            FutuOpenD -login_account=1234567 -login_pwd="... Your Secret here ..."
+    Config file is "FutuOpenD.xml", you can adjust logging verbosity here.
 
 Architecture: https://openapi.futunn.com/futu-api-doc/en/intro/intro.html
 
@@ -30,6 +39,7 @@ API
         stock basic info https://openapi.futunn.com/futu-api-doc/en/quote/get-static-info.html
         historical candles https://openapi.futunn.com/futu-api-doc/en/quote/request-history-kline.html
         realtime candles https://openapi.futunn.com/futu-api-doc/en/quote/get-kl.html
+        orderbook https://openapi.futunn.com/futu-api-doc/en/quote/get-order-book.html
         real time quote https://openapi.futunn.com/futu-api-doc/en/quote/get-stock-quote.html
         open orders https://openapi.futunn.com/futu-api-doc/en/trade/get-order-list.html
         historical orders https://openapi.futunn.com/futu-api-doc/en/trade/get-history-order-list.html

siglab_py/market_data_providers/candles_provider.py

@@ -33,7 +33,7 @@ This script is pypy compatible:
     pypy candles_provider.py --provider_id aaa --candle_size 1h --how_many_candles 2169 --redis_ttl_ms 3600000
 
 Key parameters you may want to modify:
-    provider_id: You can trigger this provider instance using test_provider.py. Of course, you'd write your own.
+    provider_id: You can trigger this provider instance using trigger_provider.py. Of course, you'd write your own.
     candle_size: 1m, 5m, 15min, 1h, 1d for example.
     how_many_candles: default to 2169 (24 x 90). 
     redis_ttl_ms: This is how long orderbook snapshot will last on redis when provider publishes to it.
@@ -318,7 +318,7 @@ async def main():
             self.universe_reload_id = universe_reload_id
     task = None
 
-    log(f"candles_provider {param['provider_id']} started, waiting for trigger. (Can use test_provider.py to trigger it)")
+    log(f"candles_provider {param['provider_id']} started, waiting for trigger. (Can use trigger_provider.py to trigger it)")
 
     universe_reload_id = 1
     for message in redis_pubsub.listen():

siglab_py/market_data_providers/candles_ta_provider.py

@@ -219,11 +219,11 @@ def work(
                                     https://redis.io/commands/set/
                                     '''
                                     expiry_sec : int = 0
-                                    if candle_size=="m":
+                                    if candle_size[-1]=="m":
                                         expiry_sec = 60 + 60*15
-                                    elif candle_size=="h":
+                                    elif candle_size[-1]=="h":
                                         expiry_sec = 60*60 + 60*15
-                                    elif candle_size=="d":
+                                    elif candle_size[-1]=="d":
                                         expiry_sec = 60*60*24 
                                     expiry_sec += 60*15 # additional 15min
 

siglab_py/market_data_providers/ccxt_candles_ta_to_csv.py

@@ -26,8 +26,8 @@ from siglab_py.util.analytic_util import compute_candles_stats
 '''
 Usage:
     set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab\siglab_py
-    python ccxt_candles_ta_to_csv.py --exchange_name bybit --symbol BTC/USDT:USDT --end_date "2025-03-11 0:0:0" --start_date "2021-03-11 0:0:0" --default_type linear --compute_ta Y --pypy_compatible N
-
+    python ccxt_candles_ta_to_csv.py --exchange_name okx --symbol BTC/USDT:USDT --candle_size 1h --end_date "2025-04-22 0:0:0" --start_date "2024-01-01 0:0:0" --default_type linear --compute_ta Y --pypy_compatible N
+    
     (Remember: python -mpip install siglab_py)
 
 This script is pypy compatible. Set "pypy_compatible" to True, in which case "compute_candles_stats" will skip calculation for TAs which requires: scipy, statsmodels, scikit-learn, sklearn.preprocessing
@@ -60,8 +60,8 @@ If debugging from VSCode, launch.json:
                 "args" : [
                         "--exchange_name", "bybit",
                         "--symbol", "BTC/USDT:USDT",
-                        "--end_date", "2025-03-11 0:0:0",
-                        "--start_date", "2024-03-11 0:0:0",
+                        "--end_date", "2025-04-22 0:0:0",
+                        "--start_date", "2024-01-01 0:0:0",
                         "--default_type", "linear",
                         "--compute_ta", "Y",
                         "--pypy_compatible", "N"

siglab_py/market_data_providers/futu_candles_ta_to_csv.py

@@ -17,12 +17,17 @@ from siglab_py.util.analytic_util import compute_candles_stats
 '''
 Usage:
     set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab\siglab_py
-    python futu_candles_ta_to_csv.py --symbol HK.00700 --end_date "2025-03-11 0:0:0" --start_date "2021-03-11 0:0:0" --market HK --trdmarket HK --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible N
+    python futu_candles_ta_to_csv.py --symbol HK.00700 --end_date "2025-03-11 0:0:0" --start_date "2021-03-11 0:0:0" --candle_size 1h --market HK --trdmarket HK --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible N
+    python futu_candles_ta_to_csv.py --symbol HK.02840 --end_date "2025-07-30 0:0:0" --start_date "2018-01-01 0:0:0" --candle_size 1h --market HK --trdmarket HK --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible N
+    python futu_candles_ta_to_csv.py --symbol AAPL --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --candle_size 1h --market US --trdmarket US --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible N
 
-    python futu_candles_ta_to_csv.py --symbol AAPL --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --market US --trdmarket US --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible N
+    Gold contracts? Note, symbol is case sensitive with Futu. So, "GCmain" is correct, "GCMain" is in-correct.
+    python futu_candles_ta_to_csv.py --symbol US.GCmain --end_date "2025-03-11 0:0:0" --start_date "2021-03-11 0:0:0" --market US --trdmarket FUTURES --security_firm FUTUSECURITIES --security_type FUTURE --compute_ta Y --pypy_compatible N
 
     (Remember: python -mpip install siglab_py)
 
+    Gold future contract specification: https://www.futunn.com/en/futures/GCMAIN-US/contract-specs
+
 This script is pypy compatible. Set "pypy_compatible" to True, in which case "compute_candles_stats" will skip calculation for TAs which requires: scipy, statsmodels, scikit-learn, sklearn.preprocessing
     pypy futu_candles_ta_to_csv.py --symbol HK.00700 --end_date "2025-03-11 0:0:0" --start_date "2024-03-11 0:0:0" --market HK --trdmarket HK --security_firm FUTUSECURITIES --security_type STOCK --compute_ta Y --pypy_compatible Y
 

siglab_py/market_data_providers/google_monitor.py

@@ -0,0 +1,320 @@
+import asyncio
+import sys
+import traceback
+import os
+import argparse
+import json
+import hashlib
+import re
+from datetime import datetime, timedelta, timezone
+import time
+import pytz
+import arrow
+from enum import Enum
+import logging
+import requests
+from typing import Dict, Optional, Set, Any, Union, List
+from redis import StrictRedis
+
+from siglab_py.util.notification_util import dispatch_notification
+
+current_filename = os.path.basename(__file__)
+
+'''
+google_monitor fetches messages from particular query. Then:
+    a. Save (and accumulate) messages to message cache file (No duplicates) for further analysis.
+        message_cache_file: str = f"google_search_messages.json"
+
+    b. If any of keywords in message_keywords_filter matches words in message (--message_keywords_filter):
+        - Publish to redis for strategy consumption, topic: param['mds']['topics']['google_alert']
+        - Dispatch slack alert
+        - If scripts runs on Windows, play a wav file (Feels free make modification play sounds on Ubuntu for example)
+
+Usage:
+    set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab\siglab_py
+    python google_monitor.py --apikey xxx --search_engine_id yyy --query "site:twitter.com @user_id1 @user_id2 some topic" --slack_info_url=https://hooks.slack.com/services/xxx --slack_critial_url=https://hooks.slack.com/services/xxx --slack_alert_url=https://hooks.slack.com/services/xxx
+
+    alert_wav_path
+        Point it to wav file for alert notification. It's using 'winsound', i.e. Windows only.
+        Set to None otherwise.
+        
+Google API: https://console.cloud.google.com/apis/credentials?project=YOUR_PROJECT
+		name: YOUR_API_KEY_NAME
+		apikey: ?????
+
+Google Search Engine
+	To create
+		name: siglab_py_search: https://programmablesearchengine.google.com/controlpanel/create
+			<script async src="https://cse.google.com/cse.js?cx=YOUR_SEARCH_ENGINE_ID">
+			</script>
+			<div class="gcse-search"></div>
+	Then enable it: https://console.developers.google.com/apis/api/customsearch.googleapis.com/overview?project=?????
+
+launch.json for Debugging from VSCode:
+    {
+        "version": "0.2.0",
+        "configurations": [
+            {
+                "name": "Python: Current File",
+                "type": "python",
+                "request": "launch",
+                "program": "${file}",
+                "console": "integratedTerminal",
+                "justMyCode": false,
+                "args" : [
+                        "--apikey", "xxx",
+                        "--search_engine_id", "yyy",
+                        "--query", "site:twitter.com @user_id1 @user_id2 some topic",
+                        "--slack_info_url", "https://hooks.slack.com/services/xxx",
+                        "--slack_critial_url", "https://hooks.slack.com/services/xxx",
+                        "--slack_alert_url", "https://hooks.slack.com/services/xxx",
+                    ],
+            }
+        ]
+    }
+'''
+
+param: Dict[str, Any] = {
+    'apikey': os.getenv('GOOGLE_APIKEY', 'xxx'),
+    'search_engine_id': os.getenv('GOOGLE_SEARCH_ENGINE_ID', 'xxx'),
+    'num_results' : 10,
+    'query' : '',
+    'alert_wav_path' : r"d:\sounds\terrible.wav",
+    "num_shouts" : 5, # How many times 'alert_wav_path' is played
+    "loop_freq_ms" : 1000*60*15, # Google allow max 100 calls per day free.
+    'current_filename' : current_filename,
+
+    'notification' : {
+        'footer' : None,
+
+        # slack webhook url's for notifications
+        'slack' : {
+            'info' : { 'webhook_url' : None },
+            'critical' : { 'webhook_url' : None },
+            'alert' : { 'webhook_url' : None },
+        }
+    },
+    
+    'mds': {
+        'topics': {
+            'tg_alert': 'tg_alert'
+        },
+        'redis': {
+            'host': 'localhost',
+            'port': 6379,
+            'db': 0,
+            'ttl_ms': 1000 * 60 * 15
+        }
+    }
+}
+
+class LogLevel(Enum):
+    CRITICAL = 50
+    ERROR = 40
+    WARNING = 30
+    INFO = 20
+    DEBUG = 10
+    NOTSET = 0
+
+logging.Formatter.converter = time.gmtime
+logger: logging.Logger = logging.getLogger()
+log_level: int = logging.INFO
+logger.setLevel(log_level)
+format_str: str = '%(asctime)s %(message)s'
+formatter: logging.Formatter = logging.Formatter(format_str)
+sh: logging.StreamHandler = logging.StreamHandler()
+sh.setLevel(log_level)
+sh.setFormatter(formatter)
+logger.addHandler(sh)
+
+def log(message: str, log_level: LogLevel = LogLevel.INFO) -> None:
+    if log_level.value < LogLevel.WARNING.value:
+        logger.info(f"{datetime.now()} {message}")
+    elif log_level.value == LogLevel.WARNING.value:
+        logger.warning(f"{datetime.now()} {message}")
+    elif log_level.value == LogLevel.ERROR.value:
+        logger.error(f"{datetime.now()} {message}")
+
+def parse_args():
+    parser = argparse.ArgumentParser() # type: ignore
+    parser.add_argument("--apikey", help="API key", default=None)
+    parser.add_argument("--search_engine_id", help="Google search engine ID", default=None)
+    parser.add_argument("--num_results", help="Max number items to fetch", default=10)
+    parser.add_argument("--query", help="Query - what are you looking for?", default=None)
+    parser.add_argument("--slack_info_url", help="Slack webhook url for INFO", default=None)
+    parser.add_argument("--slack_critial_url", help="Slack webhook url for CRITICAL", default=None)
+    parser.add_argument("--slack_alert_url", help="Slack webhook url for ALERT", default=None)
+
+    args = parser.parse_args()
+    
+    param['apikey'] = args.apikey
+    param['search_engine_id'] = args.search_engine_id
+    param['num_results'] = args.num_results
+    param['query'] = args.query
+
+    param['notification']['slack']['info']['webhook_url'] = args.slack_info_url
+    param['notification']['slack']['critical']['webhook_url'] = args.slack_critial_url
+    param['notification']['slack']['alert']['webhook_url'] = args.slack_alert_url
+
+    param['notification']['footer'] = f"From {param['current_filename']}"
+
+    print(f"Startup args: {args}") # Dont use logger, not yet setup yet.
+    print(f"param: {print(json.dumps(param, indent=2))}")
+
+def init_redis_client() -> StrictRedis:
+    redis_client : StrictRedis = StrictRedis(
+                    host = param['mds']['redis']['host'],
+                    port = param['mds']['redis']['port'],
+                    db = 0,
+                    ssl = False
+                )
+    try:
+        redis_client.keys()
+    except ConnectionError as redis_conn_error:
+        err_msg = f"Failed to connect to redis: {param['mds']['redis']['host']}, port: {param['mds']['redis']['port']}"
+        log(f"Failed to init redis connection. Will skip publishes to redis. {err_msg}")
+        redis_client = None # type: ignore
+    
+    return redis_client
+
+def search_google_custom(query, api_key, search_engine_id, num_results=10):
+    url = 'https://www.googleapis.com/customsearch/v1'
+    params = {
+        'key': api_key,
+        'cx': search_engine_id,
+        'q': query,
+        'num': num_results,
+        'sort': 'date',
+        'dateRestrict': 'd1'  # Restrict to most recent (adjust as needed: d1=day, m1=month, etc.)
+    }
+    
+    response = requests.get(url, params=params)
+    
+    if response.status_code == 200:
+        return response.json()
+    else:
+        log(f"Query error: {response.status_code} - {response.text}")
+        return None
+    
+async def main() -> None:
+    parse_args()
+
+    message_cache_file: str = f"google_search_messages.json"
+    log(f"message_cache_file: {message_cache_file}")
+
+    notification_params : Dict[str, Any] = param['notification']
+
+    processed_messages : List[Dict[str, Any]] = []
+    seen_hashes : Set[str] = set()
+    if os.path.exists(message_cache_file):
+        with open(message_cache_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            for line in lines:
+                message_data = json.loads(line)
+                message_hash: str = hashlib.sha256(message_data['message'].encode('utf-8')).hexdigest()
+
+                message_data['datetime'] = pytz.UTC.localize(arrow.get(message_data['datetime']).datetime.replace(tzinfo=None))
+
+                if message_hash not in seen_hashes:
+                    seen_hashes.add(message_hash)
+                    processed_messages.append(message_data)
+
+            processed_messages = sorted(processed_messages, key=lambda m: m['datetime'])
+    
+    try:        
+        redis_client: Optional[StrictRedis] = init_redis_client()
+    except Exception as redis_err:
+        redis_client = None
+        log(f"Failed to connect to redis. Still run but not publishing to it. {redis_err}")
+
+    while True:
+        try:
+            results = search_google_custom(param['query'], param['apikey'], param['search_engine_id'], param['num_results'])
+
+            if results:
+                if 'items' in results:
+                    for item in results['items']:
+                        title = item.get('title', 'No title')
+                        snippet = item.get('snippet', 'No snippet')
+                        link = item.get('link', 'No link')
+                        published_date = item.get('pagemap', {}).get('metatags', [{}])[0].get('article:published_time', 'No date')
+
+                        dt_message = datetime.now()
+                        pattern = r'^\d+\s*(?:days?|day?|hours?|hour?|minutes?|minute?|seconds?|second?|h|m|s)\s*(?:ago)?.*?([A-Za-z]+\s+\d+,\s+\d{4},\s+\d+:\d+\s+[AP]M\s+ET)'
+                        match = re.match(pattern, snippet)
+                        if published_date == 'No date' and match:
+                            published_date = match.group(1)
+                            dt_message = datetime.strptime(published_date, '%b %d, %Y, %I:%M %p ET')
+                        
+                        snippet = re.sub(pattern, '', snippet).strip()
+                        
+                        timestamp_ms  = int(dt_message.timestamp() * 1000)
+                        message_data: Dict[str, Any] = {
+                            "timestamp_ms": timestamp_ms,
+                            "datetime": dt_message.isoformat(), # Always in UTC
+                            "title" : title,
+                            "message": snippet,
+                            "url" : link
+                        }
+                        json_str: str = json.dumps(message_data, ensure_ascii=False, sort_keys=True)
+                        message_hash: str = hashlib.sha256(snippet.encode('utf-8')).hexdigest()
+                        if (message_hash not in seen_hashes):
+                            seen_hashes.add(message_hash)
+                            processed_messages.append(message_data)
+
+                            log(f"{message_data}")
+
+                            dispatch_notification(title=f"{param['current_filename']} Incoming! {title}", message=message_data, footer=param['notification']['footer'], params=notification_params, log_level=LogLevel.CRITICAL, logger=logger) # type: ignore
+
+                            with open(message_cache_file, 'a', encoding='utf-8') as f:
+                                json.dump(message_data, f, ensure_ascii=False)
+                                f.write('\n')
+
+                            if param['alert_wav_path']and sys.platform == 'win32':
+                                import winsound
+                                for _ in range(param['num_shouts']):
+                                    winsound.PlaySound(param['alert_wav_path'], winsound.SND_FILENAME)
+
+                            if redis_client:
+                                try:
+                                    publish_topic = f"google_search"
+                                    redis_client.publish(publish_topic, json_str)
+                                    redis_client.setex(message_hash, param['mds']['redis']['ttl_ms'] // 1000, json_str)
+                                    log(f"Published message {json_str} to Redis topic {publish_topic}", LogLevel.INFO)
+                                except Exception as e:
+                                    log(f"Failed to publish to Redis: {str(e)}", LogLevel.ERROR)
+
+                await asyncio.sleep(int(param['loop_freq_ms'] / 1000))
+
+            if processed_messages:
+                oldest_message: Dict[str, Any] = min(processed_messages, key=lambda x: x['timestamp_ms'])
+                newest_message: Dict[str, Any] = max(processed_messages, key=lambda x: x['timestamp_ms'])
+                log(
+                    json.dumps(
+                        {
+                            'num_messages': len(processed_messages),
+                            'oldest': {
+                                'timestamp_ms': oldest_message['timestamp_ms'],
+                                'datetime': datetime.fromtimestamp(int(oldest_message['timestamp_ms']/1000),tz=timezone.utc).isoformat()
+                            },
+                            'latest': {
+                                'timestamp_ms': newest_message['timestamp_ms'],
+                                'datetime': datetime.fromtimestamp(int(newest_message['timestamp_ms']/1000),tz=timezone.utc).isoformat()
+                            }
+                        }, indent=2
+                    ),
+                    LogLevel.INFO
+                )
+
+        except Exception as e:
+            log(f"Oops {str(e)} {str(sys.exc_info()[0])} {str(sys.exc_info()[1])} {traceback.format_exc()}", LogLevel.ERROR)
+        finally:
+            await asyncio.sleep(int(param['loop_freq_ms'] / 1000))
+
+if __name__ == '__main__':
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        log("Stopped by user", LogLevel.INFO)
+    except Exception as e:
+        log(f"Unexpected error: {str(e)}", LogLevel.ERROR)

siglab_py/market_data_providers/orderbooks_provider.py

@@ -20,6 +20,8 @@ from redis.client import PubSub
 from ccxt.base.exchange import Exchange
 import ccxt.pro as ccxtpro 
 
+from siglab_py.util.market_data_util import async_instantiate_exchange
+
 '''
 To start from command prompt:
     set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab
@@ -28,7 +30,7 @@ To start from command prompt:
 This script is pypy compatible.
 
 Key parameters you may want to modify:
-    provider_id: You can trigger this provider instance using test_provider.py. Of course, you'd write your own.
+    provider_id: You can trigger this provider instance using trigger_provider.py. Of course, you'd write your own.
     instance_capacity: max # tickers this provider instance will handle.
     ts_delta_observation_ms_threshold: default to 150ms. "Observation Delta" is clock diff between orderbook timestamp, and your local server clock.
     ts_delta_consecutive_ms_threshold: default to 150ms. "Consecutive Delta" is time elapsed between consecutive orderbook updates.
@@ -126,16 +128,17 @@ async def instantiate_exhange(
         ) -> Exchange:
     if old_exchange:
         await old_exchange.close() # type: ignore Otherwise, Error:  Cannot access attribute "close" for class "Exchange  Attribute "close" is unknown
-    if exchange_name==f"binance_{market_type}":
-        exchange = ccxtpro.binance(exchange_params)
-    elif exchange_name==f"okx_{market_type}":
-        exchange = ccxtpro.okx(exchange_params)
-    elif exchange_name==f"bybit_{market_type}":
-        exchange = ccxtpro.bybit(exchange_params)
-    else:
-        exchange = ccxtpro.binance(exchange_params)
+    _exchange_name = exchange_name.split('_')[0]
+
+    exchange = await async_instantiate_exchange(
+                            gateway_id = _exchange_name,
+                            default_type = market_type,
+                            api_key=None,  # type: ignore
+                            secret=None,  # type: ignore
+                            passphrase=None  # type: ignore
+                            )
     exchange.name = exchange_name # type: ignore Otherwise, Error: Cannot assign to attribute "name" for class "binance" "str" is not assignable to "None"
-    return exchange
+    return exchange  # type: ignore
 
 def log(message : str, log_level : LogLevel = LogLevel.INFO):
     if log_level.value<LogLevel.WARNING.value:
@@ -259,7 +262,7 @@ class OrderBook:
         }
 
         data['best_ask'] = min(data['asks'])
-        data['best_bid'] = min(data['bids'])
+        data['best_bid'] = max(data['bids'])
         return data
         
 class ThreadTask:
@@ -348,7 +351,7 @@ async def main():
     candles_publish_topic : str = param['mds']['topics']['candles_publish_topic']
     redis_pubsub : PubSub = init_redis_channel_subscription(redis_client, partition_assign_topic)
 
-    log(f"orderbooks_provider {param['provider_id']} started, waiting for trigger. (Can use test_provider.py to trigger it)")
+    log(f"orderbooks_provider {param['provider_id']} started, waiting for trigger. (Can use trigger_provider.py to trigger it)")
     
     tasks = []
     for message in redis_pubsub.listen():