siglab-py 0.5.72__tar.gz → 0.5.78__tar.gz

{siglab_py-0.5.72 → siglab_py-0.5.78}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: siglab_py
-Version: 0.5.72
+Version: 0.5.78
 Summary: Market data fetches, TA calculations and generic order gateway.
 Author: r0bbarh00d
 Author-email: r0bbarh00d <r0bbarh00d@gmail.com>

{siglab_py-0.5.72 → siglab_py-0.5.78}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "siglab_py"
-version = "0.5.72"
+version = "0.5.78"
 description = "Market data fetches, TA calculations and generic order gateway."
 authors = [{name = "r0bbarh00d", email = "r0bbarh00d@gmail.com"}]
 license = {text = "MIT"}

{siglab_py-0.5.72 → siglab_py-0.5.78}/setup.cfg

@@ -1,6 +1,6 @@
 [metadata]
 name = siglab_py
-version = 0.5.72
+version = 0.5.78
 description = Market data fetches, TA calculations and generic order gateway.
 author = r0bbarh00d
 author_email = r0bbarh00d@gmail.com

siglab_py-0.5.78/siglab_py/market_data_providers/google_monitor.py

@@ -0,0 +1,320 @@
+import asyncio
+import sys
+import traceback
+import os
+import argparse
+import json
+import hashlib
+import re
+from datetime import datetime, timedelta, timezone
+import time
+import pytz
+import arrow
+from enum import Enum
+import logging
+import requests
+from typing import Dict, Optional, Set, Any, Union, List
+from redis import StrictRedis
+
+from siglab_py.util.notification_util import dispatch_notification
+
+current_filename = os.path.basename(__file__)
+
+'''
+google_monitor fetches messages from particular query. Then:
+    a. Save (and accumulate) messages to message cache file (No duplicates) for further analysis.
+        message_cache_file: str = f"google_search_messages.json"
+
+    b. If any of keywords in message_keywords_filter matches words in message (--message_keywords_filter):
+        - Publish to redis for strategy consumption, topic: param['mds']['topics']['google_alert']
+        - Dispatch slack alert
+        - If scripts runs on Windows, play a wav file (Feels free make modification play sounds on Ubuntu for example)
+
+Usage:
+    set PYTHONPATH=%PYTHONPATH%;D:\dev\siglab\siglab_py
+    python google_monitor.py --apikey xxx --search_engine_id yyy --query "site:twitter.com @user_id1 @user_id2 some topic" --slack_info_url=https://hooks.slack.com/services/xxx --slack_critial_url=https://hooks.slack.com/services/xxx --slack_alert_url=https://hooks.slack.com/services/xxx
+
+    alert_wav_path
+        Point it to wav file for alert notification. It's using 'winsound', i.e. Windows only.
+        Set to None otherwise.
+        
+Google API: https://console.cloud.google.com/apis/credentials?project=YOUR_PROJECT
+		name: YOUR_API_KEY_NAME
+		apikey: ?????
+
+Google Search Engine
+	To create
+		name: siglab_py_search: https://programmablesearchengine.google.com/controlpanel/create
+			<script async src="https://cse.google.com/cse.js?cx=YOUR_SEARCH_ENGINE_ID">
+			</script>
+			<div class="gcse-search"></div>
+	Then enable it: https://console.developers.google.com/apis/api/customsearch.googleapis.com/overview?project=?????
+
+launch.json for Debugging from VSCode:
+    {
+        "version": "0.2.0",
+        "configurations": [
+            {
+                "name": "Python: Current File",
+                "type": "python",
+                "request": "launch",
+                "program": "${file}",
+                "console": "integratedTerminal",
+                "justMyCode": false,
+                "args" : [
+                        "--apikey", "xxx",
+                        "--search_engine_id", "yyy",
+                        "--query", "site:twitter.com @user_id1 @user_id2 some topic",
+                        "--slack_info_url", "https://hooks.slack.com/services/xxx",
+                        "--slack_critial_url", "https://hooks.slack.com/services/xxx",
+                        "--slack_alert_url", "https://hooks.slack.com/services/xxx",
+                    ],
+            }
+        ]
+    }
+'''
+
+param: Dict[str, Any] = {
+    'apikey': os.getenv('GOOGLE_APIKEY', 'xxx'),
+    'search_engine_id': os.getenv('GOOGLE_SEARCH_ENGINE_ID', 'xxx'),
+    'num_results' : 10,
+    'query' : '',
+    'alert_wav_path' : r"d:\sounds\terrible.wav",
+    "num_shouts" : 5, # How many times 'alert_wav_path' is played
+    "loop_freq_ms" : 1000*60*15, # Google allow max 100 calls per day free.
+    'current_filename' : current_filename,
+
+    'notification' : {
+        'footer' : None,
+
+        # slack webhook url's for notifications
+        'slack' : {
+            'info' : { 'webhook_url' : None },
+            'critical' : { 'webhook_url' : None },
+            'alert' : { 'webhook_url' : None },
+        }
+    },
+    
+    'mds': {
+        'topics': {
+            'tg_alert': 'tg_alert'
+        },
+        'redis': {
+            'host': 'localhost',
+            'port': 6379,
+            'db': 0,
+            'ttl_ms': 1000 * 60 * 15
+        }
+    }
+}
+
+class LogLevel(Enum):
+    CRITICAL = 50
+    ERROR = 40
+    WARNING = 30
+    INFO = 20
+    DEBUG = 10
+    NOTSET = 0
+
+logging.Formatter.converter = time.gmtime
+logger: logging.Logger = logging.getLogger()
+log_level: int = logging.INFO
+logger.setLevel(log_level)
+format_str: str = '%(asctime)s %(message)s'
+formatter: logging.Formatter = logging.Formatter(format_str)
+sh: logging.StreamHandler = logging.StreamHandler()
+sh.setLevel(log_level)
+sh.setFormatter(formatter)
+logger.addHandler(sh)
+
+def log(message: str, log_level: LogLevel = LogLevel.INFO) -> None:
+    if log_level.value < LogLevel.WARNING.value:
+        logger.info(f"{datetime.now()} {message}")
+    elif log_level.value == LogLevel.WARNING.value:
+        logger.warning(f"{datetime.now()} {message}")
+    elif log_level.value == LogLevel.ERROR.value:
+        logger.error(f"{datetime.now()} {message}")
+
+def parse_args():
+    parser = argparse.ArgumentParser() # type: ignore
+    parser.add_argument("--apikey", help="API key", default=None)
+    parser.add_argument("--search_engine_id", help="Google search engine ID", default=None)
+    parser.add_argument("--num_results", help="Max number items to fetch", default=10)
+    parser.add_argument("--query", help="Query - what are you looking for?", default=None)
+    parser.add_argument("--slack_info_url", help="Slack webhook url for INFO", default=None)
+    parser.add_argument("--slack_critial_url", help="Slack webhook url for CRITICAL", default=None)
+    parser.add_argument("--slack_alert_url", help="Slack webhook url for ALERT", default=None)
+
+    args = parser.parse_args()
+    
+    param['apikey'] = args.apikey
+    param['search_engine_id'] = args.search_engine_id
+    param['num_results'] = args.num_results
+    param['query'] = args.query
+
+    param['notification']['slack']['info']['webhook_url'] = args.slack_info_url
+    param['notification']['slack']['critical']['webhook_url'] = args.slack_critial_url
+    param['notification']['slack']['alert']['webhook_url'] = args.slack_alert_url
+
+    param['notification']['footer'] = f"From {param['current_filename']}"
+
+    print(f"Startup args: {args}") # Dont use logger, not yet setup yet.
+    print(f"param: {print(json.dumps(param, indent=2))}")
+
+def init_redis_client() -> StrictRedis:
+    redis_client : StrictRedis = StrictRedis(
+                    host = param['mds']['redis']['host'],
+                    port = param['mds']['redis']['port'],
+                    db = 0,
+                    ssl = False
+                )
+    try:
+        redis_client.keys()
+    except ConnectionError as redis_conn_error:
+        err_msg = f"Failed to connect to redis: {param['mds']['redis']['host']}, port: {param['mds']['redis']['port']}"
+        log(f"Failed to init redis connection. Will skip publishes to redis. {err_msg}")
+        redis_client = None # type: ignore
+    
+    return redis_client
+
+def search_google_custom(query, api_key, search_engine_id, num_results=10):
+    url = 'https://www.googleapis.com/customsearch/v1'
+    params = {
+        'key': api_key,
+        'cx': search_engine_id,
+        'q': query,
+        'num': num_results,
+        'sort': 'date',
+        'dateRestrict': 'd1'  # Restrict to most recent (adjust as needed: d1=day, m1=month, etc.)
+    }
+    
+    response = requests.get(url, params=params)
+    
+    if response.status_code == 200:
+        return response.json()
+    else:
+        log(f"Query error: {response.status_code} - {response.text}")
+        return None
+    
+async def main() -> None:
+    parse_args()
+
+    message_cache_file: str = f"google_search_messages.json"
+    log(f"message_cache_file: {message_cache_file}")
+
+    notification_params : Dict[str, Any] = param['notification']
+
+    processed_messages : List[Dict[str, Any]] = []
+    seen_hashes : Set[str] = set()
+    if os.path.exists(message_cache_file):
+        with open(message_cache_file, 'r', encoding='utf-8') as f:
+            lines = f.readlines()
+            for line in lines:
+                message_data = json.loads(line)
+                message_hash: str = hashlib.sha256(message_data['message'].encode('utf-8')).hexdigest()
+
+                message_data['datetime'] = pytz.UTC.localize(arrow.get(message_data['datetime']).datetime.replace(tzinfo=None))
+
+                if message_hash not in seen_hashes:
+                    seen_hashes.add(message_hash)
+                    processed_messages.append(message_data)
+
+            processed_messages = sorted(processed_messages, key=lambda m: m['datetime'])
+    
+    try:        
+        redis_client: Optional[StrictRedis] = init_redis_client()
+    except Exception as redis_err:
+        redis_client = None
+        log(f"Failed to connect to redis. Still run but not publishing to it. {redis_err}")
+
+    while True:
+        try:
+            results = search_google_custom(param['query'], param['apikey'], param['search_engine_id'], param['num_results'])
+
+            if results:
+                if 'items' in results:
+                    for item in results['items']:
+                        title = item.get('title', 'No title')
+                        snippet = item.get('snippet', 'No snippet')
+                        link = item.get('link', 'No link')
+                        published_date = item.get('pagemap', {}).get('metatags', [{}])[0].get('article:published_time', 'No date')
+
+                        dt_message = datetime.now()
+                        pattern = r'^\d+\s*(?:days?|day?|hours?|hour?|minutes?|minute?|seconds?|second?|h|m|s)\s*(?:ago)?.*?([A-Za-z]+\s+\d+,\s+\d{4},\s+\d+:\d+\s+[AP]M\s+ET)'
+                        match = re.match(pattern, snippet)
+                        if published_date == 'No date' and match:
+                            published_date = match.group(1)
+                            dt_message = datetime.strptime(published_date, '%b %d, %Y, %I:%M %p ET')
+                        
+                        snippet = re.sub(pattern, '', snippet).strip()
+                        
+                        timestamp_ms  = int(dt_message.timestamp() * 1000)
+                        message_data: Dict[str, Any] = {
+                            "timestamp_ms": timestamp_ms,
+                            "datetime": dt_message.isoformat(), # Always in UTC
+                            "title" : title,
+                            "message": snippet,
+                            "url" : link
+                        }
+                        json_str: str = json.dumps(message_data, ensure_ascii=False, sort_keys=True)
+                        message_hash: str = hashlib.sha256(snippet.encode('utf-8')).hexdigest()
+                        if (message_hash not in seen_hashes):
+                            seen_hashes.add(message_hash)
+                            processed_messages.append(message_data)
+
+                            log(f"{message_data}")
+
+                            dispatch_notification(title=f"{param['current_filename']} Incoming! {title}", message=message_data, footer=param['notification']['footer'], params=notification_params, log_level=LogLevel.CRITICAL, logger=logger) # type: ignore
+
+                            with open(message_cache_file, 'a', encoding='utf-8') as f:
+                                json.dump(message_data, f, ensure_ascii=False)
+                                f.write('\n')
+
+                            if param['alert_wav_path']and sys.platform == 'win32':
+                                import winsound
+                                for _ in range(param['num_shouts']):
+                                    winsound.PlaySound(param['alert_wav_path'], winsound.SND_FILENAME)
+
+                            if redis_client:
+                                try:
+                                    publish_topic = f"google_search"
+                                    redis_client.publish(publish_topic, json_str)
+                                    redis_client.setex(message_hash, param['mds']['redis']['ttl_ms'] // 1000, json_str)
+                                    log(f"Published message {json_str} to Redis topic {publish_topic}", LogLevel.INFO)
+                                except Exception as e:
+                                    log(f"Failed to publish to Redis: {str(e)}", LogLevel.ERROR)
+
+                await asyncio.sleep(int(param['loop_freq_ms'] / 1000))
+
+            if processed_messages:
+                oldest_message: Dict[str, Any] = min(processed_messages, key=lambda x: x['timestamp_ms'])
+                newest_message: Dict[str, Any] = max(processed_messages, key=lambda x: x['timestamp_ms'])
+                log(
+                    json.dumps(
+                        {
+                            'num_messages': len(processed_messages),
+                            'oldest': {
+                                'timestamp_ms': oldest_message['timestamp_ms'],
+                                'datetime': datetime.fromtimestamp(int(oldest_message['timestamp_ms']/1000),tz=timezone.utc).isoformat()
+                            },
+                            'latest': {
+                                'timestamp_ms': newest_message['timestamp_ms'],
+                                'datetime': datetime.fromtimestamp(int(newest_message['timestamp_ms']/1000),tz=timezone.utc).isoformat()
+                            }
+                        }, indent=2
+                    ),
+                    LogLevel.INFO
+                )
+
+        except Exception as e:
+            log(f"Oops {str(e)} {str(sys.exc_info()[0])} {str(sys.exc_info()[1])} {traceback.format_exc()}", LogLevel.ERROR)
+        finally:
+            await asyncio.sleep(int(param['loop_freq_ms'] / 1000))
+
+if __name__ == '__main__':
+    try:
+        asyncio.run(main())
+    except KeyboardInterrupt:
+        log("Stopped by user", LogLevel.INFO)
+    except Exception as e:
+        log(f"Unexpected error: {str(e)}", LogLevel.ERROR)

{siglab_py-0.5.72 → siglab_py-0.5.78}/siglab_py/market_data_providers/tg_monitor.py

@@ -254,7 +254,11 @@ async def main() -> None:
             processed_messages = sorted(processed_messages, key=lambda m: m['datetime'])
             last_message_date = processed_messages[-1]['datetime']
             
-    redis_client: Optional[StrictRedis] = init_redis_client()
+    try:        
+        redis_client: Optional[StrictRedis] = init_redis_client()
+    except Exception as redis_err:
+        redis_client = None
+        log(f"Failed to connect to redis. Still run but not publishing to it. {redis_err}")
         
     start_date: Optional[datetime] = None
     if param.get('start_date'):
@@ -362,7 +366,7 @@ async def main() -> None:
                         processed_messages.append(message_data)
 
                         if message_date>tm1:
-                            dispatch_notification(title=f"{param['current_filename']} {param['channel_username']} started", message=message_data, footer=param['notification']['footer'], params=notification_params, log_level=LogLevel.CRITICAL, logger=logger) # type: ignore
+                            dispatch_notification(title=f"{param['current_filename']} {param['channel_username']} Incoming!", message=message_data, footer=param['notification']['footer'], params=notification_params, log_level=LogLevel.CRITICAL, logger=logger) # type: ignore
 
                             with open(message_cache_file, 'a', encoding='utf-8') as f:
                                 json.dump(message_data, f, ensure_ascii=False)

{siglab_py-0.5.72 → siglab_py-0.5.78}/siglab_py/tests/unit/analytic_util_tests.py

@@ -47,7 +47,7 @@ class AnalyticUtilTests(unittest.TestCase):
             'datetime', 'datetime_utc', 'year', 'month', 'day', 'hour', 'minute', 'dayofweek', 
             'pct_chg_on_close', 'candle_height', 'candle_body_height',
             'week_of_month', 'apac_trading_hr', 'emea_trading_hr', 'amer_trading_hr', 
-            'is_green', 'pct_change_close', 
+            'is_green', 'candle_class', 'pct_change_close', 
             'sma_short_periods', 'sma_long_periods', 'ema_short_periods', 'ema_long_periods', 'ema_close', 
             'std', 'std_percent', 
             'vwap_short_periods', 'vwap_long_periods',

{siglab_py-0.5.72 → siglab_py-0.5.78}/siglab_py/util/analytic_util.py

@@ -15,6 +15,33 @@ from siglab_py.util.simple_math import bucket_series, bucketize_val
 from siglab_py.util.market_data_util import fix_column_types
 from siglab_py.constants import TrendDirection
 
+def classify_candle(
+    candle : pd.Series,
+    min_candle_height_ratio : float = 3
+) -> Union[str, None]:
+    candle_class : Union[str, None] = None
+    open = candle['open']
+    high = candle['high']
+    low = candle['low']
+    close = candle['close']
+    candle_full_height = high - low # always positive
+    candle_body_height = close - open # can be negative
+    candle_height_ratio = candle_full_height / abs(candle_body_height) if candle_body_height!=0 else float('inf')
+
+    if (
+        candle_height_ratio>=min_candle_height_ratio
+        and close>low
+    ):
+        candle_class = 'hammer'
+    elif (
+        candle_height_ratio>=min_candle_height_ratio
+        and close<high
+    ):
+        candle_class = 'shooting_star'
+    # Keep add more ...
+
+    return candle_class
+
 # Fibonacci
 MAGIC_FIB_LEVELS = [0, 0.236, 0.382, 0.5, 0.618, 0.786, 1.00, 1.618, 2.618, 3.618, 4.236]
 
@@ -145,6 +172,8 @@ def compute_candles_stats(
 
     pd_candles['is_green'] =  pd_candles['close'] >= pd_candles['open']
 
+    pd_candles['candle_class'] = pd_candles.apply(lambda row: classify_candle(row), axis=1)
+
     close_short_periods_rolling = pd_candles['close'].rolling(window=int(sliding_window_how_many_candles/slow_fast_interval_ratio))
     close_long_periods_rolling = pd_candles['close'].rolling(window=sliding_window_how_many_candles)
     close_short_periods_ewm = pd_candles['close'].ewm(span=int(sliding_window_how_many_candles/slow_fast_interval_ratio), adjust=False)
@@ -447,7 +476,7 @@ def compute_candles_stats(
             mitigated = pd_candles.iloc[idx + 1:row.name]['close'].lt(row['fvg_high']).any()
         return mitigated
 
-    pd_candles['fvg_mitigated'] = pd_candles.apply(lambda row: compute_fvg_mitigated(row, pd_candles), axis=1)
+    pd_candles['fvg_mitigated'] = pd_candles.apply(lambda row: compute_fvg_mitigated(row, pd_candles), axis=1) # type: ignore
 
     '''
         RSI

{siglab_py-0.5.72 → siglab_py-0.5.78}/siglab_py/util/market_data_util.py

@@ -83,7 +83,8 @@ async def async_instantiate_exchange(
     secret : str,
     passphrase : str,
     default_type : str,
-    rate_limit_ms : float = 100
+    rate_limit_ms : float = 100,
+    verbose : bool = False
 ) -> Union[AnyExchange, None]:
     exchange : Union[AnyExchange, None] = None
     exchange_name : str = gateway_id.split('_')[0]
@@ -98,7 +99,8 @@ async def async_instantiate_exchange(
                         'rateLimit' : rate_limit_ms,
                         'options' : {
                             'defaultType' : default_type
-                        }
+                        },
+                        'verbose': verbose
                     }
 
     if exchange_name=='binance':
@@ -156,7 +158,8 @@ async def async_instantiate_exchange(
                 "walletAddress" : api_key,
                 "privateKey" : secret,
                 'enableRateLimit'  : True,
-                'rateLimit' : rate_limit_ms
+                'rateLimit' : rate_limit_ms,
+                'verbose': verbose
             }
         )  # type: ignore
     else:

{siglab_py-0.5.72 → siglab_py-0.5.78}/siglab_py.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: siglab_py
-Version: 0.5.72
+Version: 0.5.78
 Summary: Market data fetches, TA calculations and generic order gateway.
 Author: r0bbarh00d
 Author-email: r0bbarh00d <r0bbarh00d@gmail.com>

{siglab_py-0.5.72 → siglab_py-0.5.78}/siglab_py.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ siglab_py/market_data_providers/candles_ta_provider.py
 siglab_py/market_data_providers/ccxt_candles_ta_to_csv.py
 siglab_py/market_data_providers/deribit_options_expiry_provider.py
 siglab_py/market_data_providers/futu_candles_ta_to_csv.py
+siglab_py/market_data_providers/google_monitor.py
 siglab_py/market_data_providers/orderbooks_provider.py
 siglab_py/market_data_providers/test_provider.py
 siglab_py/market_data_providers/tg_monitor.py