kuhl-haus-mdp-servers 0.1.5__py3-none-any.whl

kuhl_haus/servers/__init__.py

@@ -0,0 +1,10 @@
+from importlib_metadata import PackageNotFoundError, version  # pragma: no cover
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = __name__
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError

kuhl_haus/servers/mdl_server.py

@@ -0,0 +1,258 @@
+import logging
+import os
+from contextlib import asynccontextmanager
+from copy import copy
+from typing import Optional, List, Union
+
+from fastapi import FastAPI, Response, status
+from massive.websocket import Feed, Market
+from pydantic_settings import BaseSettings
+
+from kuhl_haus.mdp.integ.massive_data_queues import MassiveDataQueues
+from kuhl_haus.mdp.integ.massive_data_listener import MassiveDataListener
+from kuhl_haus.mdp.integ.utils import get_massive_api_key
+
+
+class Settings(BaseSettings):
+    # TODO: Retrieve Massive client settings from Service Control Plane API call
+    # Massive/Polygon.io API Key
+    massive_api_key: str = get_massive_api_key()
+
+    # Massive/Polygon.io Subscription Settings
+    # The default values can be overridden via environment variable; use the API to manage at runtime.
+    feed: Union[str, Feed] = os.environ.get("MASSIVE_FEED", Feed.RealTime)
+    market: Union[str, Market] = os.environ.get("MASSIVE_MARKET", Market.Stocks)
+    subscriptions: Optional[List[str]] = os.environ.get("MASSIVE_SUBSCRIPTIONS", ["AM.*"])
+
+    # Additional Massive/Polygon.io Settings - default values can be overridden via environment variables
+    raw: bool = os.environ.get("MASSIVE_RAW", False)
+    verbose: bool = os.environ.get("MASSIVE_VERBOSE", False)
+    max_reconnects: Optional[int] = os.environ.get("MASSIVE_MAX_RECONNECTS", 5)
+    secure: bool = os.environ.get("MASSIVE_SECURE", True)
+
+    # Redis Settings
+    redis_url: str = os.environ.get("REDIS_URL", "redis://redis:redis@localhost:6379/0")
+
+    # RabbitMQ Settings
+    rabbitmq_url: str = os.environ.get("RABBITMQ_URL", "amqp://crow:crow@localhost:5672/")
+    rabbitmq_host: str = os.environ.get("RABBITMQ_API", "http://crow:crow@localhost:15672/api/")
+    message_ttl_ms: int = os.environ.get("MARKET_DATA_MESSAGE_TTL", 5000)  # 5 seconds in milliseconds
+
+    # Server Settings
+    server_ip: str = os.environ.get("SERVER_IP", "0.0.0.0")
+    server_port: int = os.environ.get("SERVER_PORT", 4200)
+    log_level: str = os.environ.get("LOG_LEVEL", "INFO").upper()
+    container_image: str = os.environ.get("CONTAINER_IMAGE", "Unknown")
+    image_version: str = os.environ.get("IMAGE_VERSION", "Unknown")
+    auto_start: bool = os.environ.get("MARKET_DATA_LISTENER_AUTO_START_ENABLED", False)
+
+
+settings = Settings()
+
+logging.basicConfig(
+    level=settings.log_level,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+# Global state
+massive_data_queues: Optional[MassiveDataQueues] = None
+massive_data_listener: Optional[MassiveDataListener] = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Startup and shutdown events"""
+
+    # Startup
+    logger.info("Instantiating Market Data Listener...")
+    global massive_data_listener, massive_data_queues
+
+    massive_data_queues = MassiveDataQueues(
+        logger=logger,
+        rabbitmq_url=settings.rabbitmq_url,
+        message_ttl=settings.message_ttl_ms,
+    )
+    await massive_data_queues.setup_queues()
+
+    massive_data_listener = MassiveDataListener(
+        logger=logger,
+        message_handler=massive_data_queues.handle_messages,
+        api_key=settings.massive_api_key,
+        feed=settings.feed,
+        market=settings.market,
+        raw=settings.raw,
+        verbose=settings.verbose,
+        subscriptions=settings.subscriptions,
+        max_reconnects=settings.max_reconnects,
+        secure=settings.secure,
+    )
+    logger.info("Market Data Listener is ready.")
+    # NOTE: AUTO-START FEATURE IS DISABLED BY DEFAULT.
+    # Non-business licenses are limited to a single WebSocket connection for the entire account.
+    # The stop, start, and restart API functionality enables manual control of the WebSocket connection.
+    #
+    # To enable auto-start, set the environment variable MARKET_DATA_LISTENER_AUTO_START_ENABLED=true.
+    if settings.auto_start:
+        logger.info("[AUTO-START ENABLED]Starting Market Data Listener...")
+        await massive_data_listener.start()
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down WebSocket sidecar...")
+    await stop_websocket_client()
+    await massive_data_queues.shutdown()
+
+app = FastAPI(
+    title="Market Data Listener",
+    description="Connects to market data provider and publishes to event-specific queues",
+    lifespan=lifespan,
+)
+
+
+@app.post("/feed")
+async def feed(feed_str: str):
+    """Update Massive/Polygon.io feeds"""
+    original_feed = copy(settings.feed)
+    logger.info(f"Original feed: {original_feed}")
+    try:
+        if feed_str == Feed.RealTime.value:
+            logger.info(f"Setting feed to: {repr(Feed.RealTime)}")
+            settings.feed = Feed.RealTime
+            massive_data_listener.feed = Feed.RealTime
+        elif feed_str == Feed.Delayed.value:
+            logger.info(f"Setting feed to: {repr(Feed.Delayed)}")
+            settings.feed = Feed.Delayed
+            massive_data_listener.feed = Feed.Delayed
+        else:
+            raise ValueError(f"Invalid feed: {feed_str}")
+    except Exception as e:
+        logger.error(f"Error setting feed: {e}")
+        logger.error(f"Restoring feed to: {original_feed}")
+        settings.feed = original_feed
+        massive_data_listener.feed = original_feed
+        logger.error(f"Current feed: {settings.feed}")
+        logger.error("Rollback complete")
+
+
+@app.post("/market")
+async def market(market_str: str):
+    """Update Massive/Polygon.io market"""
+    original_market = copy(settings.market)
+    logger.info(f"Original market: {original_market}")
+    try:
+        if market_str == Market.Stocks.value:
+            logger.info(f"Setting market to: {repr(Market.Stocks)}")
+            settings.market = Market.Stocks
+            massive_data_listener.market = Market.Stocks
+        elif market_str == Market.Options.value:
+            logger.info(f"Setting market to: {repr(Market.Options)}")
+            settings.market = Market.Options
+            massive_data_listener.market = Market.Options
+        elif market_str == Market.Indices.value:
+            logger.info(f"Setting market to: {repr(Market.Indices)}")
+            settings.market = Market.Indices
+            massive_data_listener.market = Market.Indices
+        else:
+            raise ValueError(f"Invalid market: {market_str}")
+    except Exception as e:
+        logger.error(f"Error setting market: {e}")
+        logger.error(f"Restoring market to: {original_market}")
+        settings.market = original_market
+        massive_data_listener.market = original_market
+        logger.error(f"Current market: {settings.market}")
+        logger.error("Rollback complete")
+
+
+@app.post("/subscriptions")
+async def subscriptions(subscriptions_list: List[str]):
+    """Update Massive/Polygon.io subscriptions"""
+    original_subscriptions = copy(settings.subscriptions)
+    logger.info(f"Original subscriptions: {original_subscriptions}")
+    try:
+        settings.subscriptions = []
+        for sub in subscriptions_list:
+            # Only add subscriptions that start with one of the following prefixes:
+            # "A.*", "AM.*", "T.*", "Q.*", "LULD.*"
+            if (sub.startswith("A.") or
+                    sub.startswith("AM.") or
+                    sub.startswith("T.") or
+                    sub.startswith("Q.") or
+                    sub.startswith("LULD.")):
+                logger.info(f"Adding subscription: {sub}")
+                settings.subscriptions.append(sub)
+        massive_data_listener.subscriptions = settings.subscriptions
+        logger.info(f"Current subscriptions: {settings.subscriptions}")
+    except Exception as e:
+        logger.error(f"Error setting subscriptions: {e}")
+        logger.error(f"Restoring subscriptions to: {original_subscriptions}")
+        settings.subscriptions = original_subscriptions
+        massive_data_listener.subscriptions = original_subscriptions
+        logger.error(f"Current subscriptions: {settings.subscriptions}")
+        logger.error("Rollback complete")
+
+
+@app.get("/start")
+async def start_websocket_client():
+    logger.info("Starting Market Data Listener...")
+    await massive_data_listener.start()
+
+
+@app.get("/stop")
+async def stop_websocket_client():
+    logger.info("Stopping Market Data Listener...")
+    await massive_data_listener.stop()
+
+
+@app.get("/restart")
+async def restart_websocket_client():
+    logger.info("Restarting Market Data Listener...")
+    await massive_data_listener.restart()
+
+
+@app.get("/")
+async def root():
+    if massive_data_queues.connection_status["connected"] and massive_data_listener.connection_status["connected"]:
+        ret = "Running"
+    elif massive_data_queues.connection_status["connected"]:
+        ret = "Idle"
+    else:
+        ret = "Unhealthy"
+    return {
+        "service": "Market Data Listener",
+        "status": ret,
+        "auto-start": settings.auto_start,
+        "container_image": settings.container_image,
+        "image_version": settings.image_version,
+        "mdq_connection_status": massive_data_queues.connection_status,
+        "mdl_connection_status": massive_data_listener.connection_status
+    }
+
+
+@app.get("/health", status_code=200)
+async def health_check(response: Response):
+    """Health check endpoint"""
+    # The server should be connected to MDQ even when the WebSocket client is not running.
+    status_message = "OK"
+    if not massive_data_queues.connection_status["connected"]:
+        status_message = "Unhealthy"
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    # TODO: Investigate if this caused health check failures in production during off-hours.
+    # if settings.auto_start and not massive_data_listener.connection_status["connected"]:
+    #     status_message = "Unhealthy"
+    #     response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    return {
+        "service": "Market Data Listener",
+        "status": status_message,
+        "auto-start": settings.auto_start,
+        "container_image": settings.container_image,
+        "image_version": settings.image_version,
+        "mdq_connection_status": massive_data_queues.connection_status,
+        "mdl_connection_status": massive_data_listener.connection_status
+    }
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=4200)

kuhl_haus/servers/mdp_server.py

@@ -0,0 +1,245 @@
+import asyncio
+import logging
+import os
+from contextlib import asynccontextmanager
+from typing import Dict, Union
+
+# import redis.asyncio as aioredis
+from fastapi import FastAPI, Response, status
+from fastapi.responses import RedirectResponse
+from pydantic_settings import BaseSettings
+
+from massive.rest import RESTClient
+
+from kuhl_haus.mdp.analyzers.top_stocks import TopStocksAnalyzer
+from kuhl_haus.mdp.components.market_data_scanner import MarketDataScanner
+# from kuhl_haus.mdp.components.market_data_cache import MarketDataCache
+from kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
+from kuhl_haus.mdp.models.market_data_scanner_names import MarketDataScannerNames
+from kuhl_haus.mdp.models.massive_data_queue import MassiveDataQueue
+from kuhl_haus.mdp.integ.massive_data_processor import MassiveDataProcessor
+from kuhl_haus.mdp.integ.utils import get_massive_api_key
+from kuhl_haus.mdp.helpers.process_manager import ProcessManager
+
+
+class Settings(BaseSettings):
+    # Massive/Polygon.io API Key
+    massive_api_key: str = get_massive_api_key()
+
+    # RabbitMQ Settings
+    rabbitmq_url: str = os.environ.get("RABBITMQ_URL", "amqp://crow:crow@localhost:5672/")
+    rabbitmq_host: str = os.environ.get("RABBITMQ_API", "http://crow:crow@localhost:15672/api/")
+    message_ttl_ms: int = os.environ.get("MARKET_DATA_MESSAGE_TTL", 5000)  # 5 seconds in milliseconds
+
+    # Redis Settings
+    redis_url: str = os.environ.get("REDIS_URL", "redis://redis:redis@localhost:6379/0")
+
+    # Server Settings
+    server_ip: str = os.environ.get("SERVER_IP", "0.0.0.0")
+    server_port: int = os.environ.get("SERVER_PORT", 4201)
+    log_level: str = os.environ.get("LOG_LEVEL", "INFO").upper()
+    container_image: str = os.environ.get("CONTAINER_IMAGE", "Unknown")
+    image_version: str = os.environ.get("IMAGE_VERSION", "Unknown")
+
+
+settings = Settings()
+
+logging.basicConfig(
+    level=settings.log_level,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+# Global state
+# market_data_cache: MarketDataCache = None
+market_data_scanners: Dict[str, MarketDataScanner] = {}
+massive_data_processors: Dict[str, MassiveDataProcessor] = {}
+massive_data_queues = [
+    MassiveDataQueue.TRADES.value,
+    MassiveDataQueue.AGGREGATE.value,
+    MassiveDataQueue.QUOTES.value,
+    MassiveDataQueue.HALTS.value,
+    MassiveDataQueue.NEWS.value,
+    MassiveDataQueue.UNKNOWN.value,
+]
+
+# Global process manager
+process_manager: ProcessManager = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Startup and shutdown events"""
+    global process_manager
+
+    logger.info("Starting Market Data Processor...")
+    process_manager = ProcessManager()
+
+    # Start MassiveDataProcessors in separate processes
+    for queue in massive_data_queues:
+        process_manager.start_worker(
+            name=f"mdp_{queue}",
+            worker_class=MassiveDataProcessor,
+            rabbitmq_url=settings.rabbitmq_url,
+            queue_name=queue,
+            redis_url=settings.redis_url,
+        )
+    #     # Market Data Cache
+    #     redis_client = aioredis.from_url(
+    #         settings.redis_url,
+    #         encoding="utf-8",
+    #         decode_responses=True,
+    #         max_connections=1000,
+    #         socket_connect_timeout=10,  # Add timeout
+    #     )
+    #     market_data_cache = MarketDataCache(redis_client=redis_client)
+    #
+    #     # TODO: Create a component to fetch company information from FMP.
+    #
+    # Start MarketDataScanners in separate processes
+    process_manager.start_worker(
+        name=f"scanner_{MarketDataScannerNames.TOP_STOCKS.value}",
+        worker_class=MarketDataScanner,
+        redis_url=settings.redis_url,
+        analyzer=TopStocksAnalyzer(
+            rest_client=RESTClient(api_key=settings.massive_api_key)
+        ),
+        subscriptions=[f"{MarketDataCacheKeys.AGGREGATE.value}:*"]
+    )
+
+    logger.info("Market Data Processor is running.")
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down Market Data Processor...")
+    process_manager.stop_all(timeout=15.0)
+    logger.info("Market Data Processor is stopped.")
+
+
+app = FastAPI(
+    title="Market Data Processor",
+    description="The MDP is responsible for the heavy lifting which would otherwise constrain the message handling speed of the MDL.",
+    lifespan=lifespan,
+)
+
+
+@app.get("/")
+async def root():
+    # return redirect to health_check
+    return RedirectResponse(url="/health")
+
+
+@app.get("/start")
+async def start_scanners():
+    # Start all massive data processors
+    logger.info("Starting Massive Data Processors...")
+    for processor in massive_data_processors.values():
+        asyncio.create_task(processor.start())
+    logger.info("Massive Data Processors started successfully.")
+
+    logger.info("Starting Market Data Scanners...")
+    for k in market_data_scanners.keys():
+        logger.info(f"Starting {k}...")
+        await market_data_scanners[k].start()
+        logger.info(f"{k} started successfully.")
+    logger.info("Market Data Scanners started successfully.")
+
+
+@app.post("/start_scanner")
+async def start_scanner(scanner_name: str):
+    if scanner_name not in market_data_scanners.keys():
+        return {"status": "error", "message": f"Scanner {scanner_name} not found."}
+    logger.info(f"Starting {scanner_name}...")
+    await market_data_scanners[scanner_name].start()
+    logger.info(f"Started {scanner_name} successfully.")
+    return {"status": "success", "message": f"{scanner_name} started successfully."}
+
+
+@app.get("/stop")
+async def stop_scanners():
+    logger.info("Shutting down Massive Data Processors...")
+    for queue in massive_data_queues:
+        logger.info(f"Stopping {queue}...")
+        await massive_data_processors[queue].stop()
+        logger.info(f"{queue} stopped successfully.")
+    logger.info("Massive Data Processors stopped successfully.")
+    logger.info("Shutting down Market Data Scanners...")
+    for k in market_data_scanners.keys():
+        logger.info(f"Stopping {k}...")
+        await market_data_scanners[k].stop()
+        logger.info(f"{k} stopped successfully.")
+    logger.info("Market Data Scanners stopped successfully.")
+
+
+@app.post("/stop_scanner")
+async def stop_scanner(scanner_name: str):
+    if scanner_name not in market_data_scanners.keys():
+        return {"status": "error", "message": f"Scanner {scanner_name} not found."}
+    logger.info(f"Stopping {scanner_name}...")
+    await market_data_scanners[scanner_name].stop()
+    logger.info(f"Stopped {scanner_name} successfully.")
+    return {"status": "success", "message": f"{scanner_name} stopped successfully."}
+
+
+@app.get("/restart")
+async def restart_scanners():
+    logger.info("Restarting Massive Data Processors...")
+    for queue in massive_data_queues:
+        logger.info(f"Stopping {queue}...")
+        await massive_data_processors[queue].stop()
+        logger.info(f"{queue} stopped successfully.")
+    logger.info("Starting Massive Data Processors...")
+    for processor in massive_data_processors.values():
+        asyncio.create_task(processor.start())
+    logger.info("Massive Data Processors restarted successfully.")
+
+    logger.info("Restarting Market Data Scanners...")
+    for k in market_data_scanners.keys():
+        logger.info(f"Restarting {k}...")
+        await market_data_scanners[k].restart()
+        logger.info(f"{k} restarted successfully.")
+    logger.info("Restarting Market Data Scanners restarted successfully.")
+
+
+@app.post("/restart_scanner")
+async def restart_scanner(scanner_name: str):
+    if scanner_name not in market_data_scanners.keys():
+        return {"status": "error", "message": f"Scanner {scanner_name} not found."}
+    logger.info(f"Restarting {scanner_name}...")
+    await market_data_scanners[scanner_name].restart()
+    logger.info(f"Restarted {scanner_name} successfully.")
+    return {"status": "success", "message": f"{scanner_name} restarted successfully."}
+
+
+@app.get("/health", status_code=200)
+async def health_check(response: Response):
+    """Health check endpoint - always responsive"""
+    try:
+        ret: dict[str, Union[str, dict]] = {
+            "status": "OK",
+            "container_image": settings.container_image,
+            "image_version": settings.image_version,
+        }
+
+        # Non-blocking status collection
+        for queue in massive_data_queues:
+            name = f"mdp_{queue}"
+            ret[name] = process_manager.get_status(name)
+
+        for scanner_name in [MarketDataScannerNames.TOP_STOCKS.value]:
+            name = f"scanner_{scanner_name}"
+            ret[name] = process_manager.get_status(name)
+
+        return ret
+
+    except Exception as e:
+        logger.error(f"Health check error: {e}")
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+        return {"status": "ERROR", "message": "An unhandled exception occurred during health check."}
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=4201)

kuhl_haus/servers/wds_server.py

@@ -0,0 +1,190 @@
+import json
+import logging
+import os
+from contextlib import asynccontextmanager
+from typing import Set
+
+import redis.asyncio as redis
+from fastapi import FastAPI, Response, WebSocket, WebSocketDisconnect, status
+from fastapi.responses import JSONResponse, RedirectResponse
+from kuhl_haus.mdp.components.widget_data_service import WidgetDataService
+from pydantic_settings import BaseSettings
+
+
+class UnauthorizedException(Exception):
+    pass
+
+
+class Settings(BaseSettings):
+    # Redis Settings
+    redis_url: str = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+
+    # Server Settings
+    server_ip: str = os.environ.get("SERVER_IP", "0.0.0.0")
+    server_port: int = os.environ.get("SERVER_PORT", 4202)
+    log_level: str = os.environ.get("LOG_LEVEL", "INFO").upper()
+    container_image: str = os.environ.get("CONTAINER_IMAGE", "Unknown")
+    image_version: str = os.environ.get("IMAGE_VERSION", "Unknown")
+
+    # Auth Settings
+    auth_enabled: bool = os.environ.get("AUTH_ENABLED", False)
+    auth_api_key: str = os.environ.get("AUTH_API_KEY", "secret")
+
+
+settings = Settings()
+logging.basicConfig(
+    level=settings.log_level,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+# Global service instance
+wds_service: WidgetDataService = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Manage WDS lifecycle."""
+    global wds_service
+
+    # Startup
+    redis_client = redis.from_url(
+        settings.redis_url,
+        encoding="utf-8",
+        decode_responses=True
+    )
+    pubsub_client = redis_client.pubsub()
+    wds_service = WidgetDataService(redis_client=redis_client, pubsub_client=pubsub_client)
+    await wds_service.start()
+
+    yield
+
+    # Shutdown
+    await wds_service.stop()
+    await pubsub_client.close()
+    await redis_client.close()
+
+
+app = FastAPI(
+    title="Widget Data Service",
+    description="WebSocket interface for client market data subscriptions",
+    lifespan=lifespan,
+)
+
+
+@app.get("/")
+async def root():
+    # return redirect to health_check
+    return RedirectResponse(url="/health")
+
+
+@app.get("/health", status_code=200)
+async def health_check(response: Response):
+    """Kubernetes health check endpoint."""
+    try:
+        response.status_code = status.HTTP_200_OK
+        return JSONResponse({
+            "status": "OK",
+            "container_image": settings.container_image,
+            "image_version": settings.image_version,
+        })
+    except Exception as e:
+        logger.error(f"Fatal error while processing health check: {e}")
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    """WebSocket endpoint for feed subscriptions.
+
+    Client protocol:
+        Authenticate:{"action": "auth", "api_key": "secret"}
+        Subscribe:   {"action": "subscribe", "feed": "stocks:luld:*"}
+        Unsubscribe: {"action": "unsubscribe", "feed": "stocks:luld:*"}
+        Snapshot:    {"action": "get", "cache": "stocks:luld:*"}
+    """
+    await websocket.accept()
+    client_info = {
+        "headers": json.dumps(websocket.headers.items()),
+        "host": websocket.client.host,
+        "port": websocket.client.port
+    }
+    logger.info(f"wds.ws.connected client_info:{client_info}")
+
+    active_feeds: Set[str] = set()
+    authenticated: bool = not settings.auth_enabled
+    try:
+        if not authenticated:
+            message = await websocket.receive_text()
+            data = json.loads(message)
+            action = data.get("action")
+
+            if action == "auth":
+                api_key = data.get("api_key")
+                if api_key == settings.auth_api_key:
+                    authenticated = True
+                    logger.info(f"wds.ws.authenticated client_info:{client_info}")
+                    await websocket.send_json({"status": "authorized"})
+                else:
+                    await websocket.send_json({"status": "invalid key"})
+                    await websocket.close()
+                    raise UnauthorizedException("Invalid API key")
+            else:
+                await websocket.send_json({"status": "unauthorized"})
+                await websocket.close()
+                raise UnauthorizedException("Unauthorized")
+        while authenticated:
+            message = await websocket.receive_text()
+            data = json.loads(message)
+            action = data.get("action")
+
+            if action == "subscribe":
+                feed = data.get("feed")
+                if feed:
+                    await wds_service.subscribe(feed, websocket)
+                    active_feeds.add(feed)
+                    await websocket.send_json({"status": "subscribed", "feed": feed})
+
+            elif action == "unsubscribe":
+                feed = data.get("feed")
+                if feed and feed in active_feeds:
+                    await wds_service.unsubscribe(feed, websocket)
+                    active_feeds.remove(feed)
+                    await websocket.send_json({"status": "unsubscribed", "feed": feed})
+
+            elif action == "get":
+                cache_key = data.get("cache")
+                if cache_key:
+                    cached_data = await wds_service.get_cache(cache_key)
+                    await websocket.send_json({
+                        "cache": cache_key,
+                        "data": cached_data
+                    })
+            else:
+                await websocket.send_json({"status": "invalid action"})
+
+    except WebSocketDisconnect:
+        client_info = {
+            "headers": json.dumps(websocket.headers.items()),
+            "host": websocket.client.host,
+            "port": websocket.client.port
+        }
+        logger.info(f"wds.ws.disconnected client_info:{client_info}")
+        await wds_service.disconnect(websocket)
+
+    except UnauthorizedException:
+        client_info = {
+            "headers": json.dumps(websocket.headers.items()),
+            "host": websocket.client.host,
+            "port": websocket.client.port
+        }
+        logger.info(f"wds.ws.unauthorized client_info:{client_info}")
+
+    except Exception as e:
+        logger.exception(f"wds.ws.unhandled_exception {repr(e)}", exc_info=True)
+
+    finally:
+        # Clean up all subscriptions for this client
+        for feed in active_feeds:
+            await wds_service.unsubscribe(feed, websocket)

kuhl_haus_mdp_servers-0.1.5.dist-info/METADATA

@@ -0,0 +1,117 @@
+Metadata-Version: 2.4
+Name: kuhl-haus-mdp-servers
+Version: 0.1.5
+Summary: Container image build repository for market data processing servers
+Author-Email: Tom Pounders <git@oldschool.engineer>
+License-File: LICENSE.txt
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Project-URL: Homepage, https://github.com/kuhl-haus/kuhl-haus-mdp-servers
+Project-URL: Documentation, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/wiki
+Project-URL: Source, https://github.com/kuhl-haus/kuhl-haus-mdp-servers.git
+Project-URL: Changelog, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/commits
+Project-URL: Tracker, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues
+Requires-Python: <3.13,>=3.9.21
+Requires-Dist: kuhl-haus-mdp
+Requires-Dist: websockets
+Requires-Dist: aio-pika
+Requires-Dist: redis[asyncio]
+Requires-Dist: tenacity
+Requires-Dist: fastapi
+Requires-Dist: uvicorn[standard]
+Requires-Dist: pydantic-settings
+Requires-Dist: python-dotenv
+Requires-Dist: massive
+Provides-Extra: testing
+Requires-Dist: setuptools; extra == "testing"
+Requires-Dist: pdm-backend; extra == "testing"
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-cov; extra == "testing"
+Description-Content-Type: text/markdown
+
+
+[![License](https://img.shields.io/github/license/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/blob/mainline/LICENSE.txt)
+[![PyPI](https://img.shields.io/pypi/v/kuhl-haus-mdp-servers.svg)](https://pypi.org/project/kuhl-haus-mdp-servers/) 
+[![release](https://img.shields.io/github/v/release/kuhl-haus/kuhl-haus-mdp-servers?style=flat-square)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/releases)  
+[![Build Status](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml)
+[![CodeQL Advanced](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml) 
+[![Downloads](https://static.pepy.tech/badge/kuhl-haus-mdp-servers/month)](https://pepy.tech/project/kuhl-haus-mdp-servers)
+[![GitHub last commit](https://img.shields.io/github/last-commit/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/branches)
+[![GitHub issues](https://img.shields.io/github/issues/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues) 
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/pulls)
+
+
+
+# kuhl-haus-mdp-servers
+Container image build repository for market data processing servers
+
+## TL;DR
+Non-business Massive (AKA Polygon.IO) accounts are limited to a single WebSocket connection per asset class and it has to be fast enough to handle messages in a non-blocking fashion or it'll get disconnected.  The market data processing pipeline consists of loosely-coupled market data processing components so that a single WebSocket connection can handle messages fast enough to maintain a reliable connection with the market data provider.
+
+Per, https://massive.com/docs/websocket/quickstart#connecting-to-the-websocket:
+> *By default, one concurrent WebSocket connection per asset class is allowed. If you require multiple simultaneous connections for the same asset class, please [contact support](https://massive.com/contact).*
+
+# Components Summary
+
+Non-business Massive (AKA Polygon.IO) accounts are limited to a single WebSocket connection per asset class and it has to be fast enough to handle messages in a non-blocking fashion or it'll get disconnected.  The Market Data Listener (MDL) connects to the Market Data Source (Massive) and subscribes to unfiltered feeds. MDL inspects the message type for selecting the appropriate serialization method and destination Market Data Queue (MDQ).  The Market Data Processors (MDP) subscribe to raw market data in the MDQ and perform the heavy lifting that would otherwise constrain the message handling speed of the MDL.  This decoupling allows the MDP and MDL to scale independently.  Post-processed market data is stored in the MDC for consumption by the Widget Data Service (WDS).  Client-side widgets receive market data from the WDS, which provides a WebSocket interface to MDC pub/sub streams and cached data.
+
+[![Market Data Processing C4-V1.drawio.png](docs/Market_Data_Processing_C4.png)]
+
+# Component Descriptions
+
+## Market Data Listener (MDL)
+The MDL performs minimal processing on the messages.  MDL inspects the message type for selecting the appropriate serialization method and destination queue.  MDL implementations may vary as new MDS become available (for example, news).
+
+MDL runs as a container and scales independently of other components. The MDL should not be accessible outside the data plane local network.
+
+## Market Data Queues (MDQ)
+
+**Purpose:** Buffer high-velocity market data stream for server-side processing with aggressive freshness controls
+- **Queue Type:** FIFO with TTL (5-second max message age)
+- **Cleanup Strategy:** Discarded when TTL expires
+- **Message Format:** Timestamped JSON preserving original Massive.com structure
+- **Durability:** Non-persistent messages (speed over reliability for real-time data)
+- **Independence:** Queues operate completely independently - one queue per subscription
+- **Technology**: RabbitMQ
+
+The MDQ should not be accessible outside the data plane local network.
+
+## Market Data Processors (MDP)
+The purpose of the MDP is to process raw real-time market data and delegate processing to data-specific handlers.  This separation of concerns allows MDPs to handle any type of data and simplifies horizontal scaling.  The MDP stores its processed results in the Market Data Cache (MDC).
+
+The MDP:
+- Hydrates the in-memory cache on MDC
+- Processes market data 
+- Publishes messages to pub/sub channels
+- Maintains cache entries in MDC
+
+MDPs runs as containers and scale independently of other components. The MDPs should not be accessible outside the data plane local network.
+
+## Market Data Cache (MDC)
+
+**Purpose:** In-memory data store for serialized processed market data.
+* **Cache Type**: In-memory persistent or with TTL
+- **Queue Type:** pub/sub
+- **Technology**: Redis
+
+The MDC should not be accessible outside the data plane local network.
+
+## Widget Data Service (WDS)
+**Purpose**:
+1. WebSocket interface provides access to processed market data for client-side code
+2. Is the network-layer boundary between clients and the data that is available on the data plane
+
+WDS runs as a container and scales independently of other components.  WDS is the only data plane component that should be exposed to client networks.
+
+
+## Service Control Plane (SCP)
+**Purpose**: 
+1. Authentication and authorization
+2. Serve static and dynamic content via py4web
+3. Serve SPA to authenticated clients
+4. Injects authentication token and WDS url into SPA environment for authenticated access to WDS
+5. Control plane for managing application components at runtime
+6. API for programmatic access to service controls and instrumentation.
+
+The SCP requires access to the data plane network for API access to data plane components.
+

kuhl_haus_mdp_servers-0.1.5.dist-info/RECORD

@@ -0,0 +1,9 @@
+kuhl_haus/servers/__init__.py,sha256=5dEpAdB3kypH8tCRECoXwbly1WV9kFU5kh8ldGSa0VI,349
+kuhl_haus/servers/mdl_server.py,sha256=iSkP7GANluStDeDqFJhL4IOAQNo94jUiX9HbAnmuazc,10273
+kuhl_haus/servers/mdp_server.py,sha256=sUeZOmSIjzPHIiYmZKlpmIy2Lag4FLZivI9J6XGo-PQ,9017
+kuhl_haus/servers/wds_server.py,sha256=DYLbq8Al67h-p4TzUJYl-RCU61izSUYYNtzvzb5KGyM,6418
+kuhl_haus_mdp_servers-0.1.5.dist-info/METADATA,sha256=S3gIGsy_nnyite8-r5ECn4a4cVulg2mS6RQ_q4d8lUk,7347
+kuhl_haus_mdp_servers-0.1.5.dist-info/WHEEL,sha256=tsUv_t7BDeJeRHaSrczbGeuK-TtDpGsWi_JfpzD255I,90
+kuhl_haus_mdp_servers-0.1.5.dist-info/entry_points.txt,sha256=LPD1rLjALMFQF-BcJTcVMcZRqR89QPSqu05BHC4BBWc,172
+kuhl_haus_mdp_servers-0.1.5.dist-info/licenses/LICENSE.txt,sha256=j5GdO9_Y_RH0lr1tsVAY1rtBJxoyC_-MmqaFNDE3dwo,1066
+kuhl_haus_mdp_servers-0.1.5.dist-info/RECORD,,

kuhl_haus_mdp_servers-0.1.5.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: pdm-backend (2.4.6)
+Root-Is-Purelib: true
+Tag: py3-none-any

kuhl_haus_mdp_servers-0.1.5.dist-info/entry_points.txt

@@ -0,0 +1,7 @@
+[console_scripts]
+mdl_server = kuhl_haus.servers.mdl_server:app
+mdp_server = kuhl_haus.servers.mdp_server:app
+wds_server = kuhl_haus.servers.wds_server:app
+
+[gui_scripts]
+

kuhl_haus_mdp_servers-0.1.5.dist-info/licenses/LICENSE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Kuhl Haus
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.