kuhl-haus-mdp 0.0.1__py3-none-any.whl

kuhl_haus/mdp/components/market_data_scanner.py

@@ -0,0 +1,236 @@
+import asyncio
+import json
+import logging
+from typing import Union, Optional, List
+
+import redis.asyncio as aioredis
+from redis.exceptions import ConnectionError
+
+from kuhl_haus.mdp.analyzers.analyzer import Analyzer
+from kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
+
+
+class MarketDataScanner:
+    mdc_connected: bool
+    processed: int
+    decoding_error: int
+    dropped: int
+    error: int
+    restarts: int
+
+    def __init__(self, redis_url: str, analyzer: Analyzer, subscriptions: List[str]):
+        self.redis_url = redis_url
+        self.analyzer = analyzer
+        self.logger = logging.getLogger(__name__)
+
+        # Connection objects
+        self.redis_client = None  # : aioredis.Redis = None
+        self.pubsub_client: Optional[aioredis.client.PubSub] = None
+
+        # State
+        self.mdc_connected = False
+        self.running = False
+
+        self.subscriptions: List[str] = subscriptions
+        self._pubsub_task: Union[asyncio.Task, None] = None
+
+        # Metrics
+        self.restarts = 0
+        self.processed = 0
+        self.decoding_errors = 0
+        self.empty_results = 0
+        self.published_results = 0
+        self.errors = 0
+
+    async def start(self):
+        """Initialize Redis connections. Pub/sub task starts on first subscription."""
+        self.logger.info("mds.starting")
+        await self.connect()
+        self.pubsub_client = self.redis_client.pubsub()
+
+        scanner_cache = await self.get_cache(self.analyzer.cache_key)
+        self.logger.info(f"mds rehydrating from cache")
+        await self.analyzer.rehydrate(scanner_cache)
+        self.logger.info("mds rehydration complete")
+
+        for subscription in self.subscriptions:
+            if subscription.endswith("*"):
+                await self.pubsub_client.psubscribe(subscription)
+            else:
+                await self.pubsub_client.subscribe(subscription)
+        self._pubsub_task = asyncio.create_task(self._handle_pubsub())
+        self.logger.info("mds.started")
+
+    async def stop(self):
+        """Cleanup Redis connections."""
+        self.logger.info("mds.stopping")
+
+        if self._pubsub_task:
+            self._pubsub_task.cancel()
+            try:
+                await self._pubsub_task
+            except asyncio.CancelledError:
+                pass
+            self._pubsub_task = None
+
+        if self.pubsub_client:
+            for subscription in self.subscriptions:
+                if subscription.endswith("*"):
+                    await self.pubsub_client.punsubscribe(subscription)
+                else:
+                    await self.pubsub_client.unsubscribe(subscription)
+            await self.pubsub_client.close()
+            self.pubsub_client = None
+
+        if self.redis_client:
+            await self.redis_client.close()
+            self.redis_client = None
+            self.mdc_connected = False
+
+        self.logger.info("mds.stopped")
+
+    async def connect(self, force: bool = False):
+        """Establish async connections to Redis"""
+        if not self.mdc_connected or force:
+            # Redis connection pool
+            try:
+                self.redis_client = aioredis.from_url(
+                    self.redis_url,
+                    encoding="utf-8",
+                    decode_responses=True,
+                    max_connections=1000,
+                    socket_connect_timeout=10,  # Add timeout
+                )
+
+                # Test Redis connection
+                await self.redis_client.ping()
+                self.mdc_connected = True
+                self.logger.debug(f"Connected to Redis: {self.redis_url}")
+            except Exception as e:
+                self.logger.error(f"Failed to connect to Redis: {e}")
+                raise
+
+    async def restart(self):
+        """Restart Market Data Scanner"""
+        try:
+            await self.stop()
+            await asyncio.sleep(1)
+            await self.start()
+            self.restarts += 1
+        except Exception as e:
+            self.logger.error(f"Error restarting Market Data Scanner: {e}")
+
+    async def _handle_pubsub(self):
+        """Background task to receive Redis pub/sub messages and fan out to WebSockets."""
+        try:
+            self.logger.info("mds.pubsub.starting")
+            message_count = 0
+            retry_count = 0
+            max_retry_interval = 60
+            self.running = True
+            while True:
+                # get_message() requires active subscriptions
+                message = await self.pubsub_client.get_message(
+                    ignore_subscribe_messages=False,
+                    timeout=1.0
+                )
+
+                if message is None:
+                    # Timeout reached, no message available sleep with exponential backoff
+                    # to a maximum duration of max_retry_interval seconds
+                    retry_count += 1
+                    self.logger.debug(
+                        f"mds.pubsub.message timeout reached, no message available.  Retry count: {retry_count}")
+                    sleep_interval = min(2**retry_count, max_retry_interval)
+                    await asyncio.sleep(sleep_interval)
+                    continue
+                else:
+                    retry_count = 0
+                msg_type = message.get("type")
+                channel = message.get("channel")
+                data = message.get("data")
+                # Log subscription lifecycle events
+                if msg_type == "subscribe" or msg_type == "psubscribe":
+                    self.logger.info(f"mds.pubsub.subscribed channel:{channel}, num_subs:{data}")
+
+                elif msg_type == "unsubscribe" or msg_type == "punsubscribe":
+                    self.logger.info(f"mds.pubsub.unsubscribed channel:{channel}, num_subs:{data}")
+
+                # Process actual data messages
+                elif msg_type == "message" or msg_type == "pmessage":
+                    message_count += 1
+                    self.logger.debug(f"mds.pubsub.message channel:{channel}, data_len:{len(data)}, msg_num:{message_count}, data:{data}")
+                    await self._process_message(data=json.loads(data))
+                else:
+                    self.logger.warning(f"mds.pubsub.unknown message type: {msg_type}")
+                    self.dropped += 1
+                    continue
+        except ConnectionError as e:
+            self.logger.error(f"mds.pubsub.connection_error error:{repr(e)}", e)
+            self.running = False
+            self.mdc_connected = False
+            await self.restart()
+        except asyncio.CancelledError:
+            self.logger.info("mds.pubsub.cancelled")
+            self.running = False
+            self.mdc_connected = False
+            raise
+        except Exception as e:
+            self.logger.error(f"mds.pubsub.error error:{repr(e)}", e)
+            self.running = False
+            self.mdc_connected = False
+            raise
+
+    async def _process_message(self, data: dict):
+        """Process single message with concurrency control"""
+        try:
+            # Delegate to analyzer (async)
+            self.logger.debug(f"Processing message - data_len:{len(data)}")
+            analyzer_results = await self.analyzer.analyze_data(data)
+            self.processed += 1
+            if analyzer_results:
+                for analyzer_result in analyzer_results:
+                    # Cache in Redis
+                    self.logger.debug(f"Caching message {analyzer_result.cache_key}")
+                    await self.cache_result(analyzer_result)
+                    self.published_results += 1
+            else:
+                # Empty result - nothing to cache
+                self.empty_results += 1
+        except json.JSONDecodeError as e:
+            self.logger.error(f"JSON decode error: {e}")
+            self.decoding_errors += 1
+        except Exception as e:
+            self.logger.error(f"Processing error: {e}", exc_info=True)
+            self.errors += 1
+
+    async def get_cache(self, cache_key: str) -> Optional[dict]:
+        """Fetch current value from Redis cache (for snapshot requests)."""
+        value = await self.redis_client.get(cache_key)
+        if value:
+            return json.loads(value)
+        return None
+
+    async def cache_result(self, analyzer_result: MarketDataAnalyzerResult):
+        """
+        Async cache to Redis with pub/sub notification
+
+        Args:
+            analyzer_result: MarketDataAnalyzerResult
+        """
+        result_json = json.dumps(analyzer_result.data)
+
+        # Pipeline - no async context manager, no await on queue methods
+        pipe = self.redis_client.pipeline(transaction=False)
+        if analyzer_result.cache_key:
+            if analyzer_result.cache_ttl > 0:
+                pipe.setex(analyzer_result.cache_key, analyzer_result.cache_ttl, result_json)
+            else:
+                pipe.set(analyzer_result.cache_key, result_json)
+        if analyzer_result.publish_key:
+            pipe.publish(analyzer_result.publish_key, result_json)
+
+        await pipe.execute()
+
+        self.logger.debug(f"Cached result for {analyzer_result.cache_key}")
+

kuhl_haus/mdp/components/widget_data_service.py

@@ -0,0 +1,191 @@
+import asyncio
+import json
+import logging
+import os
+from typing import Dict, Set
+
+import redis.asyncio as redis
+from fastapi import WebSocket
+from pydantic_settings import BaseSettings
+
+
+class UnauthorizedException(Exception):
+    pass
+
+
+class Settings(BaseSettings):
+    log_level: str = os.environ.get("LOG_LEVEL", "INFO").upper()
+
+
+settings = Settings()
+logging.basicConfig(
+    level=settings.log_level,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+class WidgetDataService:
+    """WebSocket interface for client subscriptions to Redis market data."""
+
+    def __init__(self, redis_client: redis.Redis, pubsub_client: redis.client.PubSub):
+        self.redis_client: redis.Redis = redis_client
+        self.pubsub_client: redis.client.PubSub = pubsub_client
+
+        # Track active WebSocket connections per feed
+        self.subscriptions: Dict[str, Set[WebSocket]] = {}
+        self._pubsub_task: asyncio.Task = None
+        self._pubsub_lock = asyncio.Lock()
+
+        self.mdc_connected = False
+
+    async def start(self):
+        """This doesn't do anything anymore. Pub/sub task starts on first subscription."""
+        logger.info("wds.starting")
+        await self.redis_client.ping()
+        self.mdc_connected = True
+        logger.info("wds.started")
+
+    async def stop(self):
+        """Cleanup Redis connections."""
+        logger.info("wds.stopping")
+
+        if self._pubsub_task:
+            self._pubsub_task.cancel()
+            try:
+                await self._pubsub_task
+            except asyncio.CancelledError:
+                pass
+
+        logger.info("wds.stopped")
+
+    async def subscribe(self, feed: str, websocket: WebSocket):
+        """Subscribe WebSocket client to a Redis feed."""
+        async with self._pubsub_lock:
+            if feed not in self.subscriptions:
+                self.subscriptions[feed] = set()
+                if "*" in feed:
+                    await self.pubsub_client.psubscribe(feed)
+                else:
+                    await self.pubsub_client.subscribe(feed)
+                logger.info(f"wds.feed.subscribed feed:{feed}, total_feeds:{len(self.subscriptions)}")
+
+            # First subscription: start pub/sub task
+            if len(self.subscriptions.keys()) == 1 and self._pubsub_task is None:
+                self._pubsub_task = asyncio.create_task(self._handle_pubsub())
+                logger.info("wds.pubsub.task_started")
+            self.subscriptions[feed].add(websocket)
+            logger.info(f"wds.client.subscribed feed:{feed}, clients:{len(self.subscriptions[feed])}")
+
+    async def unsubscribe(self, feed: str, websocket: WebSocket):
+        """Unsubscribe WebSocket client from a Redis feed."""
+        async with self._pubsub_lock:
+            if feed in self.subscriptions:
+                self.subscriptions[feed].discard(websocket)
+
+                if not self.subscriptions[feed]:
+                    if "*" in feed:
+                        await self.pubsub_client.punsubscribe(feed)
+                    else:
+                        await self.pubsub_client.unsubscribe(feed)
+                    del self.subscriptions[feed]
+                    logger.info(f"wds.feed.unsubscribed feed:{feed}, total_feeds:{len(self.subscriptions)}")
+                else:
+                    logger.info(f"wds.client.unsubscribed feed:{feed}, clients:{len(self.subscriptions[feed])}")
+
+            # Last subscription removed: stop pub/sub task
+            if not self.subscriptions and self._pubsub_task:
+                try:
+                    self._pubsub_task.cancel()
+                    await self._pubsub_task
+                except asyncio.CancelledError:
+                    pass
+                except RuntimeError:
+                    pass
+                self._pubsub_task = None
+                logger.info("wds.pubsub.task_stopped")
+
+    async def disconnect(self, websocket: WebSocket):
+        """Disconnect WebSocket client from all feeds."""
+        subs = []
+        async with self._pubsub_lock:
+            feeds = self.subscriptions.keys()
+            for feed in feeds:
+                logger.info(f"wds.client.disconnecting feed:{feed}")
+                subs.append(f"{feed}")
+        for sub in subs:
+            await self.unsubscribe(sub, websocket)
+
+    async def get_cache(self, cache_key: str) -> dict:
+        """Fetch current value from Redis cache (for snapshot requests)."""
+        logger.info(f"wds.cache.get cache_key:{cache_key}")
+        value = await self.redis_client.get(cache_key)
+        if value:
+            logger.info(f"wds.cache.hit cache_key:{cache_key}")
+            return json.loads(value)
+        logger.info(f"wds.cache.miss cache_key:{cache_key}")
+        return None
+
+    async def _handle_pubsub(self):
+        """Background task to receive Redis pub/sub messages and fan out to WebSockets."""
+        try:
+            logger.info("wds.pubsub.starting")
+            message_count = 0
+
+            while True:
+                # get_message() requires active subscriptions
+                message = await self.pubsub_client.get_message(
+                    ignore_subscribe_messages=False,
+                    timeout=1.0
+                )
+
+                if message is None:
+                    # Timeout reached, no message available
+                    await asyncio.sleep(0.01)
+                    continue
+
+                msg_type = message.get("type")
+
+                # Log subscription lifecycle events
+                if msg_type == "subscribe":
+                    logger.info(f"wds.pubsub.subscribed channel:{message['channel']}, num_subs:{message['data']}")
+
+                elif msg_type == "unsubscribe":
+                    logger.info(f"wds.pubsub.unsubscribed channel:{message['channel']}, num_subs:{message['data']}")
+
+                # Process actual data messages
+                elif msg_type == "message":
+                    message_count += 1
+                    feed = message["channel"]
+                    data = message["data"]
+
+                    logger.debug(f"wds.pubsub.message feed:{feed}, data_len:{len(data)}, msg_num:{message_count}")
+
+                    if feed in self.subscriptions:
+                        # Fan out to all WebSocket clients subscribed to this feed
+                        disconnected = []
+                        sent_count = 0
+
+                        for ws in self.subscriptions[feed]:
+                            try:
+                                await ws.send_text(data)
+                                sent_count += 1
+                            except Exception as e:
+                                logger.error(f"wds.send.failed feed:{feed}, error:{repr(e)}")
+                                disconnected.append(ws)
+
+                        logger.debug(f"wds.fanout.complete feed:{feed}, sent:{sent_count}, failed:{len(disconnected)}")
+
+                        # Clean up disconnected clients
+                        for ws in disconnected:
+                            await self.unsubscribe(feed, ws)
+                    else:
+                        logger.warning(f"wds.pubsub.orphan feed:{feed}, msg:Received message for untracked feed")
+
+        except asyncio.CancelledError:
+            logger.info("wds.pubsub.cancelled")
+            raise
+
+        except Exception as e:
+            logger.error(f"wds.pubsub.error error:{repr(e)}", e)
+            raise

kuhl_haus/mdp/helpers/process_manager.py

@@ -0,0 +1,228 @@
+from multiprocessing import Process, Queue
+from queue import Empty, Full
+from multiprocessing.synchronize import Event as MPEvent
+from typing import Dict, Type, Any
+import signal
+import asyncio
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class ProcessManager:
+    """Manages worker processes for MDP components"""
+
+    def __init__(self):
+        self.processes: Dict[str, Process] = {}
+        self.shutdown_events: Dict[str, MPEvent] = {}
+        self.status_queues: Dict[str, Queue] = {}
+
+    def start_worker(self, name: str, worker_class: Type, **kwargs):
+        """Start any worker class in a separate process"""
+        import multiprocessing as mp  # Keep factory import local
+
+        shutdown_event = mp.Event()  # Factory call
+        status_queue = mp.Queue(maxsize=1)
+
+        process = Process(
+            target=self._run_worker,
+            args=(worker_class, shutdown_event, status_queue),
+            kwargs=kwargs,
+            name=name,
+            daemon=False
+        )
+
+        self.processes[name] = process
+        self.shutdown_events[name] = shutdown_event
+        self.status_queues[name] = status_queue
+
+        process.start()
+        logger.info(f"Started process: {name} (PID: {process.pid})")
+
+    @staticmethod
+    def _run_worker(
+        worker_class: Type[Any],
+        shutdown_event: MPEvent,
+        status_queue: Queue,
+        **kwargs: Any
+    ) -> None:
+        """Generic worker process entry point with async event loop management"""
+
+        # Signal handlers for graceful shutdown
+        def shutdown_handler(signum, frame):
+            shutdown_event.set()
+
+        signal.signal(signal.SIGTERM, shutdown_handler)
+        signal.signal(signal.SIGINT, shutdown_handler)
+
+        # Create new event loop for this process
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+
+        # Create asyncio event for coordinating shutdown
+        async_shutdown = asyncio.Event()
+
+        async def monitor_shutdown():
+            """Monitor multiprocessing shutdown_event and set async event"""
+            while not shutdown_event.is_set():
+                await asyncio.sleep(0.1)  # Poll frequently without blocking
+            async_shutdown.set()
+            logger.info(f"Shutdown signal detected for {worker_class.__name__}")
+
+        async def status_reporter(worker_instance):
+            """Periodically report worker status to parent process"""
+            while not async_shutdown.is_set():
+                try:
+                    status = {
+                        "processed": getattr(worker_instance, 'processed', 0),
+                        "errors": getattr(worker_instance, 'errors', getattr(worker_instance, 'error', 0)),
+                        "decoding_errors": getattr(worker_instance, 'decoding_errors', 0),
+                        "dropped": getattr(worker_instance, 'dropped', 0),
+                        "duplicated": getattr(worker_instance, 'duplicated', 0),
+                        "mdq_connected": getattr(worker_instance, 'mdq_connected', False),
+                        "mdc_connected": getattr(worker_instance, 'mdc_connected', False),
+                        "restarts": getattr(worker_instance, 'restarts', 0),
+                        "running": getattr(worker_instance, 'running', False),
+                    }
+
+                    try:
+                        status_queue.put_nowait(status)
+                    except Full:
+                        pass  # Queue full, skip this update
+                except Exception as ex:
+                    logger.error(f"Error updating status for {worker_class.__name__}: {ex}")
+
+                # Wait 1 second before next status update
+                try:
+                    await asyncio.wait_for(async_shutdown.wait(), timeout=1.0)
+                except asyncio.TimeoutError:
+                    pass  # Expected - just continue loop
+
+        async def run_worker():
+            """Main worker coroutine that manages worker lifecycle"""
+            worker = None
+
+            try:
+                # Instantiate worker
+                worker = worker_class(**kwargs)
+                logger.info(f"Instantiated {worker_class.__name__}")
+
+                # Start monitoring and status tasks
+                monitor_task = asyncio.create_task(monitor_shutdown())
+                status_task = asyncio.create_task(status_reporter(worker))
+
+                # Start the worker (may block indefinitely or return immediately)
+                logger.info(f"Starting {worker_class.__name__}")
+                worker_task = asyncio.create_task(worker.start())
+
+                # Wait for shutdown signal while worker and status tasks run
+                await async_shutdown.wait()
+
+                logger.info(f"Shutdown initiated for {worker_class.__name__}")
+
+                # Signal worker to stop if it has a running flag
+                if hasattr(worker, 'running'):
+                    worker.running = False
+                    logger.debug(f"Set running=False for {worker_class.__name__}")
+
+                # Cancel worker task if it's still running
+                if not worker_task.done():
+                    logger.info(f"Cancelling worker task for {worker_class.__name__}")
+                    worker_task.cancel()
+                    try:
+                        await asyncio.wait_for(worker_task, timeout=5.0)
+                    except (asyncio.CancelledError, asyncio.TimeoutError):
+                        logger.warning(f"Worker task cancellation timeout for {worker_class.__name__}")
+                    except Exception as ex:
+                        logger.error(f"Error during worker task cancellation: {ex}")
+
+                # Call worker's stop method for cleanup
+                logger.info(f"Calling stop() for {worker_class.__name__}")
+                await worker.stop()
+
+                # Cancel monitoring tasks
+                monitor_task.cancel()
+                status_task.cancel()
+
+                # Wait for cleanup with timeout
+                try:
+                    await asyncio.wait_for(
+                        asyncio.gather(monitor_task, status_task, return_exceptions=True),
+                        timeout=2.0
+                    )
+                except asyncio.TimeoutError:
+                    logger.warning(f"Monitoring task cleanup timeout for {worker_class.__name__}")
+
+                logger.info(f"Worker {worker_class.__name__} stopped cleanly")
+
+            except Exception as ex:
+                logger.error(f"Worker error in {worker_class.__name__}: {ex}", exc_info=True)
+            finally:
+                # Ensure stop is called even if errors occurred
+                if worker is not None:
+                    try:
+                        await worker.stop()
+                    except Exception as ex:
+                        logger.error(f"Error during final stop() call: {ex}", exc_info=True)
+
+        # Run the worker coroutine
+        try:
+            loop.run_until_complete(run_worker())
+        except Exception as e:
+            logger.error(f"Fatal error in worker process {worker_class.__name__}: {e}", exc_info=True)
+        finally:
+            try:
+                # Cancel any remaining tasks
+                pending = asyncio.all_tasks(loop)
+                for task in pending:
+                    task.cancel()
+
+                # Give tasks a chance to clean up
+                if pending:
+                    loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True))
+            except Exception as e:
+                logger.error(f"Error during event loop cleanup: {e}")
+            finally:
+                loop.close()
+
+    def stop_process(self, name: str, timeout: float = 10.0):
+        """Stop a specific worker process"""
+        if name not in self.processes:
+            return
+
+        process = self.processes[name]
+        shutdown_event = self.shutdown_events[name]
+
+        logger.info(f"Stopping process: {name}")
+        shutdown_event.set()
+
+        process.join(timeout=timeout)
+        if process.is_alive():
+            logger.warning(f"Force killing process: {name}")
+            process.kill()
+            process.join()
+
+    def stop_all(self, timeout: float = 10.0):
+        """Stop all worker processes"""
+        for name in list(self.processes.keys()):
+            self.stop_process(name, timeout)
+
+    def get_status(self, name: str) -> dict:
+        """Get status from worker process (non-blocking)"""
+        if name not in self.processes:
+            return {"alive": False}
+
+        process = self.processes[name]
+        status_queue = self.status_queues[name]
+
+        status = {"alive": process.is_alive(), "pid": process.pid}
+
+        try:
+            worker_status = status_queue.get_nowait()
+            status.update(worker_status)
+        except Empty:
+            pass
+        except Exception as e:
+            logger.error(f"Error getting status for process name {name}: {e}", exc_info=True)
+
+        return status

kuhl_haus/mdp/helpers/queue_name_resolver.py

@@ -0,0 +1,24 @@
+from massive.websocket.models import (
+    WebSocketMessage,
+    EquityAgg,
+    EquityQuote,
+    EquityTrade,
+    LimitUpLimitDown,
+)
+
+from kuhl_haus.mdp.models.massive_data_queue import MassiveDataQueue
+
+
+class QueueNameResolver:
+    @staticmethod
+    def queue_name_for_web_socket_message(message: WebSocketMessage):
+        if isinstance(message, EquityTrade):
+            return MassiveDataQueue.TRADES.value
+        elif isinstance(message, EquityAgg):
+            return MassiveDataQueue.AGGREGATE.value
+        elif isinstance(message, EquityQuote):
+            return MassiveDataQueue.QUOTES.value
+        elif isinstance(message, LimitUpLimitDown):
+            return MassiveDataQueue.HALTS.value
+        else:
+            return MassiveDataQueue.UNKNOWN.value