kuhl-haus-mdp-servers 0.0.0__tar.gz

kuhl_haus_mdp_servers-0.0.0/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: kuhl-haus-mdp-servers
+Version: 0.0.0
+Summary: Container image build repository for market data processing servers
+Author-email: Tom Pounders <git@oldschool.engineer>
+Project-URL: Homepage, https://github.com/kuhl-haus/kuhl-haus-mdp-servers
+Project-URL: Documentation, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/wiki
+Project-URL: Source, https://github.com/kuhl-haus/kuhl-haus-mdp-servers.git
+Project-URL: Changelog, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/commits
+Project-URL: Tracker, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Requires-Python: <3.13,>=3.9.21
+Description-Content-Type: text/markdown
+Requires-Dist: kuhl-haus-mdp
+Provides-Extra: testing
+Requires-Dist: setuptools; extra == "testing"
+Requires-Dist: pdm-backend; extra == "testing"
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-cov; extra == "testing"
+
+
+[![License](https://img.shields.io/github/license/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/blob/mainline/LICENSE.txt)
+[![PyPI](https://img.shields.io/pypi/v/kuhl-haus-mdp-servers.svg)](https://pypi.org/project/kuhl-haus-mdp-servers/) 
+[![release](https://img.shields.io/github/v/release/kuhl-haus/kuhl-haus-mdp-servers?style=flat-square)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/releases)  
+[![Build Status](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml)
+[![CodeQL Advanced](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml) 
+[![Downloads](https://static.pepy.tech/badge/kuhl-haus-mdp-servers/month)](https://pepy.tech/project/kuhl-haus-mdp-servers)
+[![GitHub last commit](https://img.shields.io/github/last-commit/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/branches)
+[![GitHub issues](https://img.shields.io/github/issues/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues) 
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/pulls)
+
+
+
+# kuhl-haus-mdp-servers
+Container image build repository for market data processing servers

kuhl_haus_mdp_servers-0.0.0/README.md

@@ -0,0 +1,15 @@
+
+[![License](https://img.shields.io/github/license/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/blob/mainline/LICENSE.txt)
+[![PyPI](https://img.shields.io/pypi/v/kuhl-haus-mdp-servers.svg)](https://pypi.org/project/kuhl-haus-mdp-servers/) 
+[![release](https://img.shields.io/github/v/release/kuhl-haus/kuhl-haus-mdp-servers?style=flat-square)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/releases)  
+[![Build Status](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml)
+[![CodeQL Advanced](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml) 
+[![Downloads](https://static.pepy.tech/badge/kuhl-haus-mdp-servers/month)](https://pepy.tech/project/kuhl-haus-mdp-servers)
+[![GitHub last commit](https://img.shields.io/github/last-commit/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/branches)
+[![GitHub issues](https://img.shields.io/github/issues/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues) 
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/pulls)
+
+
+
+# kuhl-haus-mdp-servers
+Container image build repository for market data processing servers

kuhl_haus_mdp_servers-0.0.0/pyproject.toml

@@ -0,0 +1,79 @@
+[build-system]
+requires = ["pdm-backend", "setuptools>=61.0"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "kuhl-haus-mdp-servers"
+dynamic = ["version"]
+description = "Container image build repository for market data processing servers"
+authors = [
+    { name = "Tom Pounders", email = "git@oldschool.engineer" },
+]
+readme = "README.md"
+requires-python = "<3.13,>=3.9.21"
+license-files = ["LICENSE.txt"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Programming Language :: Python"
+]
+dependencies = [
+    "kuhl-haus-mdp",
+]
+
+[project.urls]
+Homepage = "https://github.com/kuhl-haus/kuhl-haus-mdp-servers"
+Documentation = "https://github.com/kuhl-haus/kuhl-haus-mdp-servers/wiki"
+Source = "https://github.com/kuhl-haus/kuhl-haus-mdp-servers.git"
+Changelog = "https://github.com/kuhl-haus/kuhl-haus-mdp-servers/commits"
+Tracker = "https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues"
+
+[project.optional-dependencies]
+testing = [
+    "setuptools",
+    "pdm-backend",
+    "pytest",
+    "pytest-cov",
+]
+
+[tool.setuptools_scm]
+version_scheme = "guess-next-dev"
+local_scheme = "no-local-version"
+
+[project.scripts]
+mdl_server = "kuhl_haus.servers.mdl_server:app"
+mdp_server = "kuhl_haus.servers.mdp_server:app"
+wds_server = "kuhl_haus.servers.wds_server:app"
+
+[tool.setuptools]
+zip-safe = true
+include-package-data = true
+package-dir = {"" = "src"}
+packages = {find = {where = ["src"], exclude = ["tests"]}}
+
+[tool.pytest.ini_options]
+addopts = ["--verbose"]
+norecursedirs = ["dist", "build", ".tox"]
+testpaths = ["tests"]
+
+[tool.devpi.upload]
+no_vcs = 1
+formats = ["bdist_wheel"]
+
+[tool.flake8]
+max_line_length = 88
+extend_ignore = ["E203", "W503"]
+exclude = [".tox", "build", "dist", ".eggs", "docs/conf.py"]
+
+[tool.pyscaffold]
+version = "4.6"
+package = "servers"
+extensions = ["namespace"]
+namespace = "kuhl_haus"
+
+[tool.pdm]
+distribution = true
+version = { source = "scm" }
+
+[tool.pdm.build]
+package-dir = "src"
+includes = ["src/kuhl_haus"]

kuhl_haus_mdp_servers-0.0.0/setup.cfg

@@ -0,0 +1,28 @@
+[options]
+package_dir = 
+	= src
+zip_safe = True
+include_package_data = True
+
+[options.packages.find]
+where = src
+exclude = 
+	tests
+
+[options.package_data]
+kuhl_haus.servers = 
+	py.typed
+
+[tool:pytest]
+addopts = 
+	--verbose
+norecursedirs = 
+	dist
+	build
+	.tox
+testpaths = tests
+
+[egg_info]
+tag_build = 
+tag_date = 0
+

kuhl_haus_mdp_servers-0.0.0/setup.py

@@ -0,0 +1,22 @@
+"""
+    Setup file for kuhl-haus-crow.
+    Use setup.cfg to configure your project.
+
+    This file was generated with PyScaffold 4.6.
+    PyScaffold helps you to put up the scaffold of your new Python project.
+    Learn more under: https://pyscaffold.org/
+"""
+
+from setuptools import setup
+
+if __name__ == "__main__":
+    try:
+        setup()
+    except:  # noqa
+        print(
+            "\n\nAn error occurred while building the project, "
+            "please ensure you have the most updated version of setuptools, "
+            "setuptools_scm and wheel with:\n"
+            "   pip install -U setuptools setuptools_scm wheel\n\n"
+        )
+        raise

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus/servers/__init__.py

@@ -0,0 +1,10 @@
+from importlib_metadata import PackageNotFoundError, version  # pragma: no cover
+
+try:
+    # Change here if project is renamed and does not equal the package name
+    dist_name = __name__
+    __version__ = version(dist_name)
+except PackageNotFoundError:  # pragma: no cover
+    __version__ = "unknown"
+finally:
+    del version, PackageNotFoundError

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus/servers/mdl_server.py

@@ -0,0 +1,258 @@
+import logging
+import os
+from contextlib import asynccontextmanager
+from copy import copy
+from typing import Optional, List, Union
+
+from fastapi import FastAPI, Response, status
+from massive.websocket import Feed, Market
+from pydantic_settings import BaseSettings
+
+from kuhl_haus.mdp.integ.massive_data_queues import MassiveDataQueues
+from kuhl_haus.mdp.integ.massive_data_listener import MassiveDataListener
+from kuhl_haus.mdp.integ.utils import get_massive_api_key
+
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+class Settings(BaseSettings):
+    # TODO: Retrieve Massive client settings from Service Control Plane API call
+    # Massive/Polygon.io API Key
+    massive_api_key: str = get_massive_api_key()
+
+    # Massive/Polygon.io Subscription Settings
+    # The default values can be overridden via environment variable; use the API to manage at runtime.
+    feed: Union[str, Feed] = os.environ.get("MASSIVE_FEED", Feed.RealTime)
+    market: Union[str, Market] = os.environ.get("MASSIVE_MARKET", Market.Stocks)
+    subscriptions: Optional[List[str]] = os.environ.get("MASSIVE_SUBSCRIPTIONS", ["A.*"])
+
+    # Additional Massive/Polygon.io Settings - default values can be overridden via environment variables
+    raw: bool = os.environ.get("MASSIVE_RAW", False)
+    verbose: bool = os.environ.get("MASSIVE_VERBOSE", False)
+    max_reconnects: Optional[int] = os.environ.get("MASSIVE_MAX_RECONNECTS", 5)
+    secure: bool = os.environ.get("MASSIVE_SECURE", True)
+
+    # Redis Settings
+    redis_url: str = os.environ.get("REDIS_URL", "redis://redis:redis@localhost:6379/0")
+
+    # RabbitMQ Settings
+    rabbitmq_url: str = os.environ.get("RABBITMQ_URL", "amqp://crow:crow@localhost:5672/")
+    rabbitmq_host: str = os.environ.get("RABBITMQ_API", "http://crow:crow@localhost:15672/api/")
+    message_ttl_ms: int = os.environ.get("MARKET_DATA_MESSAGE_TTL", 5000)  # 5 seconds in milliseconds
+
+    # Server Settings
+    server_ip: str = os.environ.get("SERVER_IP", "0.0.0.0")
+    server_port: int = os.environ.get("SERVER_PORT", 4200)
+    log_level: str = os.environ.get("LOG_LEVEL", "INFO").upper()
+    container_image: str = os.environ.get("CONTAINER_IMAGE", "Unknown")
+    image_version: str = os.environ.get("IMAGE_VERSION", "Unknown")
+    auto_start: bool = os.environ.get("MARKET_DATA_LISTENER_AUTO_START_ENABLED", False)
+
+
+settings = Settings()
+
+# Global state
+massive_data_queues: Optional[MassiveDataQueues] = None
+massive_data_listener: Optional[MassiveDataListener] = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Startup and shutdown events"""
+
+    # Startup
+    logger.info("Instantiating Market Data Listener...")
+    global massive_data_listener, massive_data_queues
+
+    massive_data_queues = MassiveDataQueues(
+        logger=logger,
+        rabbitmq_url=settings.rabbitmq_url,
+        message_ttl=settings.message_ttl_ms,
+    )
+    await massive_data_queues.setup_queues()
+
+    massive_data_listener = MassiveDataListener(
+        logger=logger,
+        message_handler=massive_data_queues.handle_messages,
+        api_key=settings.massive_api_key,
+        feed=settings.feed,
+        market=settings.market,
+        raw=settings.raw,
+        verbose=settings.verbose,
+        subscriptions=settings.subscriptions,
+        max_reconnects=settings.max_reconnects,
+        secure=settings.secure,
+    )
+    logger.info("Market Data Listener is ready.")
+    # NOTE: AUTO-START FEATURE IS DISABLED BY DEFAULT.
+    # Non-business licenses are limited to a single WebSocket connection for the entire account.
+    # The stop, start, and restart API functionality enables manual control of the WebSocket connection.
+    #
+    # To enable auto-start, set the environment variable MARKET_DATA_LISTENER_AUTO_START_ENABLED=true.
+    if settings.auto_start:
+        logger.info("[AUTO-START ENABLED]Starting Market Data Listener...")
+        await massive_data_listener.start()
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down WebSocket sidecar...")
+    await stop_websocket_client()
+    await massive_data_queues.shutdown()
+
+app = FastAPI(
+    title="Market Data Listener",
+    description="Connects to market data provider and publishes to event-specific queues",
+    lifespan=lifespan,
+)
+
+
+@app.post("/feed")
+async def feed(feed_str: str):
+    """Update Massive/Polygon.io feeds"""
+    original_feed = copy(settings.feed)
+    logger.info(f"Original feed: {original_feed}")
+    try:
+        if feed_str == Feed.RealTime.value:
+            logger.info(f"Setting feed to: {repr(Feed.RealTime)}")
+            settings.feed = Feed.RealTime
+            massive_data_listener.feed = Feed.RealTime
+        elif feed_str == Feed.Delayed.value:
+            logger.info(f"Setting feed to: {repr(Feed.Delayed)}")
+            settings.feed = Feed.Delayed
+            massive_data_listener.feed = Feed.Delayed
+        else:
+            raise ValueError(f"Invalid feed: {feed_str}")
+    except Exception as e:
+        logger.error(f"Error setting feed: {e}")
+        logger.error(f"Restoring feed to: {original_feed}")
+        settings.feed = original_feed
+        massive_data_listener.feed = original_feed
+        logger.error(f"Current feed: {settings.feed}")
+        logger.error("Rollback complete")
+
+
+@app.post("/market")
+async def market(market_str: str):
+    """Update Massive/Polygon.io market"""
+    original_market = copy(settings.market)
+    logger.info(f"Original market: {original_market}")
+    try:
+        if market_str == Market.Stocks.value:
+            logger.info(f"Setting market to: {repr(Market.Stocks)}")
+            settings.market = Market.Stocks
+            massive_data_listener.market = Market.Stocks
+        elif market_str == Market.Options.value:
+            logger.info(f"Setting market to: {repr(Market.Options)}")
+            settings.market = Market.Options
+            massive_data_listener.market = Market.Options
+        elif market_str == Market.Indices.value:
+            logger.info(f"Setting market to: {repr(Market.Indices)}")
+            settings.market = Market.Indices
+            massive_data_listener.market = Market.Indices
+        else:
+            raise ValueError(f"Invalid market: {market_str}")
+    except Exception as e:
+        logger.error(f"Error setting market: {e}")
+        logger.error(f"Restoring market to: {original_market}")
+        settings.market = original_market
+        massive_data_listener.market = original_market
+        logger.error(f"Current market: {settings.market}")
+        logger.error("Rollback complete")
+
+
+@app.post("/subscriptions")
+async def subscriptions(subscriptions_list: List[str]):
+    """Update Massive/Polygon.io subscriptions"""
+    original_subscriptions = copy(settings.subscriptions)
+    logger.info(f"Original subscriptions: {original_subscriptions}")
+    try:
+        settings.subscriptions = []
+        for sub in subscriptions_list:
+            # Only add subscriptions that start with one of the following prefixes:
+            # "A.*", "AM.*", "T.*", "Q.*", "LULD.*"
+            if (sub.startswith("A.") or
+                    sub.startswith("AM.") or
+                    sub.startswith("T.") or
+                    sub.startswith("Q.") or
+                    sub.startswith("LULD.")):
+                logger.info(f"Adding subscription: {sub}")
+                settings.subscriptions.append(sub)
+        massive_data_listener.subscriptions = settings.subscriptions
+        logger.info(f"Current subscriptions: {settings.subscriptions}")
+    except Exception as e:
+        logger.error(f"Error setting subscriptions: {e}")
+        logger.error(f"Restoring subscriptions to: {original_subscriptions}")
+        settings.subscriptions = original_subscriptions
+        massive_data_listener.subscriptions = original_subscriptions
+        logger.error(f"Current subscriptions: {settings.subscriptions}")
+        logger.error("Rollback complete")
+
+
+@app.get("/start")
+async def start_websocket_client():
+    logger.info("Starting Market Data Listener...")
+    await massive_data_listener.start()
+
+
+@app.get("/stop")
+async def stop_websocket_client():
+    logger.info("Stopping Market Data Listener...")
+    await massive_data_listener.stop()
+
+
+@app.get("/restart")
+async def restart_websocket_client():
+    logger.info("Restarting Market Data Listener...")
+    await massive_data_listener.restart()
+
+
+@app.get("/")
+async def root():
+    if massive_data_queues.connection_status["connected"] and massive_data_listener.connection_status["connected"]:
+        ret = "Running"
+    elif massive_data_queues.connection_status["connected"]:
+        ret = "Idle"
+    else:
+        ret = "Unhealthy"
+    return {
+        "service": "Market Data Listener",
+        "status": ret,
+        "auto-start": settings.auto_start,
+        "container_image": settings.container_image,
+        "image_version": settings.image_version,
+        "mdq_connection_status": massive_data_queues.connection_status,
+        "mdl_connection_status": massive_data_listener.connection_status
+    }
+
+
+@app.get("/health", status_code=200)
+async def health_check(response: Response):
+    """Health check endpoint"""
+    # The server should be connected to MDQ even when the WebSocket client is not running.
+    status_message = "OK"
+    if not massive_data_queues.connection_status["connected"]:
+        status_message = "Unhealthy"
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    # TODO: Investigate if this caused health check failures in production during off-hours.
+    # if settings.auto_start and not massive_data_listener.connection_status["connected"]:
+    #     status_message = "Unhealthy"
+    #     response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+    return {
+        "service": "Market Data Listener",
+        "status": status_message,
+        "auto-start": settings.auto_start,
+        "container_image": settings.container_image,
+        "image_version": settings.image_version,
+        "mdq_connection_status": massive_data_queues.connection_status,
+        "mdl_connection_status": massive_data_listener.connection_status
+    }
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=4200)

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus/servers/mdp_server.py

@@ -0,0 +1,245 @@
+import asyncio
+import logging
+import os
+from contextlib import asynccontextmanager
+from typing import Dict, Union
+
+# import redis.asyncio as aioredis
+from fastapi import FastAPI, Response, status
+from fastapi.responses import RedirectResponse
+from pydantic_settings import BaseSettings
+
+from massive.rest import RESTClient
+
+from kuhl_haus.mdp.analyzers.top_stocks import TopStocksAnalyzer
+from kuhl_haus.mdp.components.market_data_scanner import MarketDataScanner
+# from kuhl_haus.mdp.components.market_data_cache import MarketDataCache
+from kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
+from kuhl_haus.mdp.models.market_data_scanner_names import MarketDataScannerNames
+from kuhl_haus.mdp.models.massive_data_queue import MassiveDataQueue
+from kuhl_haus.mdp.integ.massive_data_processor import MassiveDataProcessor
+from kuhl_haus.mdp.integ.utils import get_massive_api_key
+from kuhl_haus.mdp.helpers.process_manager import ProcessManager
+
+
+class Settings(BaseSettings):
+    # Massive/Polygon.io API Key
+    massive_api_key: str = get_massive_api_key()
+
+    # RabbitMQ Settings
+    rabbitmq_url: str = os.environ.get("RABBITMQ_URL", "amqp://crow:crow@localhost:5672/")
+    rabbitmq_host: str = os.environ.get("RABBITMQ_API", "http://crow:crow@localhost:15672/api/")
+    message_ttl_ms: int = os.environ.get("MARKET_DATA_MESSAGE_TTL", 5000)  # 5 seconds in milliseconds
+
+    # Redis Settings
+    redis_url: str = os.environ.get("REDIS_URL", "redis://redis:redis@localhost:6379/0")
+
+    # Server Settings
+    server_ip: str = os.environ.get("SERVER_IP", "0.0.0.0")
+    server_port: int = os.environ.get("SERVER_PORT", 4201)
+    log_level: str = os.environ.get("LOG_LEVEL", "INFO").upper()
+    container_image: str = os.environ.get("CONTAINER_IMAGE", "Unknown")
+    image_version: str = os.environ.get("IMAGE_VERSION", "Unknown")
+
+
+settings = Settings()
+
+logging.basicConfig(
+    level=settings.log_level,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+# Global state
+# market_data_cache: MarketDataCache = None
+market_data_scanners: Dict[str, MarketDataScanner] = {}
+massive_data_processors: Dict[str, MassiveDataProcessor] = {}
+massive_data_queues = [
+    MassiveDataQueue.TRADES.value,
+    MassiveDataQueue.AGGREGATE.value,
+    MassiveDataQueue.QUOTES.value,
+    MassiveDataQueue.HALTS.value,
+    MassiveDataQueue.NEWS.value,
+    MassiveDataQueue.UNKNOWN.value,
+]
+
+# Global process manager
+process_manager: ProcessManager = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Startup and shutdown events"""
+    global process_manager
+
+    logger.info("Starting Market Data Processor...")
+    process_manager = ProcessManager()
+
+    # Start MassiveDataProcessors in separate processes
+    for queue in massive_data_queues:
+        process_manager.start_worker(
+            name=f"mdp_{queue}",
+            worker_class=MassiveDataProcessor,
+            rabbitmq_url=settings.rabbitmq_url,
+            queue_name=queue,
+            redis_url=settings.redis_url,
+        )
+    #     # Market Data Cache
+    #     redis_client = aioredis.from_url(
+    #         settings.redis_url,
+    #         encoding="utf-8",
+    #         decode_responses=True,
+    #         max_connections=1000,
+    #         socket_connect_timeout=10,  # Add timeout
+    #     )
+    #     market_data_cache = MarketDataCache(redis_client=redis_client)
+    #
+    #     # TODO: Create a component to fetch company information from FMP.
+    #
+    # Start MarketDataScanners in separate processes
+    process_manager.start_worker(
+        name=f"scanner_{MarketDataScannerNames.TOP_STOCKS.value}",
+        worker_class=MarketDataScanner,
+        redis_url=settings.redis_url,
+        analyzer=TopStocksAnalyzer(
+            rest_client=RESTClient(api_key=settings.massive_api_key)
+        ),
+        subscriptions=[f"{MarketDataCacheKeys.AGGREGATE.value}:*"]
+    )
+
+    logger.info("Market Data Processor is running.")
+
+    yield
+
+    # Shutdown
+    logger.info("Shutting down Market Data Processor...")
+    process_manager.stop_all(timeout=15.0)
+    logger.info("Market Data Processor is stopped.")
+
+
+app = FastAPI(
+    title="Market Data Processor",
+    description="The MDP is responsible for the heavy lifting which would otherwise constrain the message handling speed of the MDL.",
+    lifespan=lifespan,
+)
+
+
+@app.get("/")
+async def root():
+    # return redirect to health_check
+    return RedirectResponse(url="/health")
+
+
+@app.get("/start")
+async def start_scanners():
+    # Start all massive data processors
+    logger.info("Starting Massive Data Processors...")
+    for processor in massive_data_processors.values():
+        asyncio.create_task(processor.start())
+    logger.info("Massive Data Processors started successfully.")
+
+    logger.info("Starting Market Data Scanners...")
+    for k in market_data_scanners.keys():
+        logger.info(f"Starting {k}...")
+        await market_data_scanners[k].start()
+        logger.info(f"{k} started successfully.")
+    logger.info("Market Data Scanners started successfully.")
+
+
+@app.post("/start_scanner")
+async def start_scanner(scanner_name: str):
+    if scanner_name not in market_data_scanners.keys():
+        return {"status": "error", "message": f"Scanner {scanner_name} not found."}
+    logger.info(f"Starting {scanner_name}...")
+    await market_data_scanners[scanner_name].start()
+    logger.info(f"Started {scanner_name} successfully.")
+    return {"status": "success", "message": f"{scanner_name} started successfully."}
+
+
+@app.get("/stop")
+async def stop_scanners():
+    logger.info("Shutting down Massive Data Processors...")
+    for queue in massive_data_queues:
+        logger.info(f"Stopping {queue}...")
+        await massive_data_processors[queue].stop()
+        logger.info(f"{queue} stopped successfully.")
+    logger.info("Massive Data Processors stopped successfully.")
+    logger.info("Shutting down Market Data Scanners...")
+    for k in market_data_scanners.keys():
+        logger.info(f"Stopping {k}...")
+        await market_data_scanners[k].stop()
+        logger.info(f"{k} stopped successfully.")
+    logger.info("Market Data Scanners stopped successfully.")
+
+
+@app.post("/stop_scanner")
+async def stop_scanner(scanner_name: str):
+    if scanner_name not in market_data_scanners.keys():
+        return {"status": "error", "message": f"Scanner {scanner_name} not found."}
+    logger.info(f"Stopping {scanner_name}...")
+    await market_data_scanners[scanner_name].stop()
+    logger.info(f"Stopped {scanner_name} successfully.")
+    return {"status": "success", "message": f"{scanner_name} stopped successfully."}
+
+
+@app.get("/restart")
+async def restart_scanners():
+    logger.info("Restarting Massive Data Processors...")
+    for queue in massive_data_queues:
+        logger.info(f"Stopping {queue}...")
+        await massive_data_processors[queue].stop()
+        logger.info(f"{queue} stopped successfully.")
+    logger.info("Starting Massive Data Processors...")
+    for processor in massive_data_processors.values():
+        asyncio.create_task(processor.start())
+    logger.info("Massive Data Processors restarted successfully.")
+
+    logger.info("Restarting Market Data Scanners...")
+    for k in market_data_scanners.keys():
+        logger.info(f"Restarting {k}...")
+        await market_data_scanners[k].restart()
+        logger.info(f"{k} restarted successfully.")
+    logger.info("Restarting Market Data Scanners restarted successfully.")
+
+
+@app.post("/restart_scanner")
+async def restart_scanner(scanner_name: str):
+    if scanner_name not in market_data_scanners.keys():
+        return {"status": "error", "message": f"Scanner {scanner_name} not found."}
+    logger.info(f"Restarting {scanner_name}...")
+    await market_data_scanners[scanner_name].restart()
+    logger.info(f"Restarted {scanner_name} successfully.")
+    return {"status": "success", "message": f"{scanner_name} restarted successfully."}
+
+
+@app.get("/health", status_code=200)
+async def health_check(response: Response):
+    """Health check endpoint - always responsive"""
+    try:
+        ret: dict[str, Union[str, dict]] = {
+            "status": "OK",
+            "container_image": settings.container_image,
+            "image_version": settings.image_version,
+        }
+
+        # Non-blocking status collection
+        for queue in massive_data_queues:
+            name = f"mdp_{queue}"
+            ret[name] = process_manager.get_status(name)
+
+        for scanner_name in [MarketDataScannerNames.TOP_STOCKS.value]:
+            name = f"scanner_{scanner_name}"
+            ret[name] = process_manager.get_status(name)
+
+        return ret
+
+    except Exception as e:
+        logger.error(f"Health check error: {e}")
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+        return {"status": "ERROR", "message": str(e)}
+
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=4201)

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus/servers/wds_server.py

@@ -0,0 +1,200 @@
+import asyncio
+import json
+import logging
+import os
+from contextlib import asynccontextmanager
+from typing import Set
+
+import redis.asyncio as redis
+from fastapi import FastAPI, Response, WebSocket, WebSocketDisconnect, status
+from fastapi.responses import JSONResponse, RedirectResponse
+from kuhl_haus.mdp.components.widget_data_service import WidgetDataService
+from pydantic_settings import BaseSettings
+
+
+class UnauthorizedException(Exception):
+    pass
+
+
+class Settings(BaseSettings):
+    # Redis Settings
+    redis_url: str = os.environ.get("REDIS_URL", "redis://redis:redis@localhost:6379/0")
+
+    # Server Settings
+    server_ip: str = os.environ.get("SERVER_IP", "0.0.0.0")
+    server_port: int = os.environ.get("SERVER_PORT", 4202)
+    log_level: str = os.environ.get("LOG_LEVEL", "INFO").upper()
+    container_image: str = os.environ.get("CONTAINER_IMAGE", "Unknown")
+    image_version: str = os.environ.get("IMAGE_VERSION", "Unknown")
+
+    # Auth Settings
+    auth_enabled: bool = os.environ.get("AUTH_ENABLED", False)
+    auth_api_key: str = os.environ.get("AUTH_API_KEY", "secret")
+
+
+settings = Settings()
+logging.basicConfig(
+    level=settings.log_level,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+)
+logger = logging.getLogger(__name__)
+
+
+# Global service instance
+wds_service: WidgetDataService = None
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Manage WDS lifecycle."""
+    global wds_service
+
+    # Startup
+    redis_client = redis.from_url(
+        settings.redis_url,
+        encoding="utf-8",
+        decode_responses=True
+    )
+    pubsub_client = redis_client.pubsub()
+    wds_service = WidgetDataService(redis_client=redis_client, pubsub_client=pubsub_client)
+    await wds_service.start()
+
+    yield
+
+    # Shutdown
+    await wds_service.stop()
+    await pubsub_client.close()
+    await redis_client.close()
+
+
+app = FastAPI(
+    title="Widget Data Service",
+    description="WebSocket interface for client market data subscriptions",
+    lifespan=lifespan,
+)
+
+
+@app.get("/")
+async def root():
+    # return redirect to health_check
+    return RedirectResponse(url="/health")
+
+
+@app.get("/restart")
+async def restart_wds_service():
+    logger.info("Restarting Widget Data Service...")
+    await wds_service.stop()
+    await asyncio.sleep(1)
+    await wds_service.start()
+    logger.info("Widget Data Service restarted successfully.")
+
+
+@app.get("/health", status_code=200)
+async def health_check(response: Response):
+    """Kubernetes health check endpoint."""
+    try:
+        response.status_code = status.HTTP_200_OK
+        return JSONResponse({
+            "status": "OK",
+            "container_image": settings.container_image,
+            "image_version": settings.image_version,
+        })
+    except Exception as e:
+        logger.error(f"Fatal error while processing health check: {e}")
+        response.status_code = status.HTTP_503_SERVICE_UNAVAILABLE
+
+
+@app.websocket("/ws")
+async def websocket_endpoint(websocket: WebSocket):
+    """WebSocket endpoint for feed subscriptions.
+
+    Client protocol:
+        Authenticate:{"action": "auth", "api_key": "secret"}
+        Subscribe:   {"action": "subscribe", "feed": "stocks:luld:*"}
+        Unsubscribe: {"action": "unsubscribe", "feed": "stocks:luld:*"}
+        Snapshot:    {"action": "get", "cache": "stocks:luld:*"}
+    """
+    await websocket.accept()
+    client_info = {
+        "headers": json.dumps(websocket.headers.items()),
+        "host": websocket.client.host,
+        "port": websocket.client.port
+    }
+    logger.info(f"wds.ws.connected client_info:{client_info}")
+
+    active_feeds: Set[str] = set()
+    authenticated: bool = not settings.auth_enabled
+    try:
+        if not authenticated:
+            message = await websocket.receive_text()
+            data = json.loads(message)
+            action = data.get("action")
+
+            if action == "auth":
+                api_key = data.get("api_key")
+                if api_key == settings.auth_api_key:
+                    authenticated = True
+                    logger.info(f"wds.ws.authenticated client_info:{client_info}")
+                    await websocket.send_json({"status": "authorized"})
+                else:
+                    await websocket.send_json({"status": "invalid key"})
+                    await websocket.close()
+                    raise UnauthorizedException("Invalid API key")
+            else:
+                await websocket.send_json({"status": "unauthorized"})
+                await websocket.close()
+                raise UnauthorizedException("Unauthorized")
+        while authenticated:
+            message = await websocket.receive_text()
+            data = json.loads(message)
+            action = data.get("action")
+
+            if action == "subscribe":
+                feed = data.get("feed")
+                if feed:
+                    await wds_service.subscribe(feed, websocket)
+                    active_feeds.add(feed)
+                    await websocket.send_json({"status": "subscribed", "feed": feed})
+
+            elif action == "unsubscribe":
+                feed = data.get("feed")
+                if feed and feed in active_feeds:
+                    await wds_service.unsubscribe(feed, websocket)
+                    active_feeds.remove(feed)
+                    await websocket.send_json({"status": "unsubscribed", "feed": feed})
+
+            elif action == "get":
+                cache_key = data.get("cache")
+                if cache_key:
+                    cached_data = await wds_service.get_cache(cache_key)
+                    await websocket.send_json({
+                        "cache": cache_key,
+                        "data": cached_data
+                    })
+            else:
+                await websocket.send_json({"status": "invalid action"})
+
+    except WebSocketDisconnect:
+        client_info = {
+            "headers": json.dumps(websocket.headers.items()),
+            "host": websocket.client.host,
+            "port": websocket.client.port
+        }
+        logger.info(f"wds.ws.disconnected client_info:{client_info}")
+        await wds_service.disconnect(websocket)
+
+    except UnauthorizedException:
+        client_info = {
+            "headers": json.dumps(websocket.headers.items()),
+            "host": websocket.client.host,
+            "port": websocket.client.port
+        }
+        logger.info(f"wds.ws.unauthorized client_info:{client_info}")
+
+    except Exception as e:
+        logger.exception(f"wds.ws.unhandled_exception {repr(e)}", exc_info=True)
+
+    finally:
+        # Clean up all subscriptions for this client
+        for feed in active_feeds:
+            await wds_service.unsubscribe(feed, websocket)

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus_mdp_servers.egg-info/PKG-INFO

@@ -0,0 +1,36 @@
+Metadata-Version: 2.4
+Name: kuhl-haus-mdp-servers
+Version: 0.0.0
+Summary: Container image build repository for market data processing servers
+Author-email: Tom Pounders <git@oldschool.engineer>
+Project-URL: Homepage, https://github.com/kuhl-haus/kuhl-haus-mdp-servers
+Project-URL: Documentation, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/wiki
+Project-URL: Source, https://github.com/kuhl-haus/kuhl-haus-mdp-servers.git
+Project-URL: Changelog, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/commits
+Project-URL: Tracker, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues
+Classifier: Development Status :: 4 - Beta
+Classifier: Programming Language :: Python
+Requires-Python: <3.13,>=3.9.21
+Description-Content-Type: text/markdown
+Requires-Dist: kuhl-haus-mdp
+Provides-Extra: testing
+Requires-Dist: setuptools; extra == "testing"
+Requires-Dist: pdm-backend; extra == "testing"
+Requires-Dist: pytest; extra == "testing"
+Requires-Dist: pytest-cov; extra == "testing"
+
+
+[![License](https://img.shields.io/github/license/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/blob/mainline/LICENSE.txt)
+[![PyPI](https://img.shields.io/pypi/v/kuhl-haus-mdp-servers.svg)](https://pypi.org/project/kuhl-haus-mdp-servers/) 
+[![release](https://img.shields.io/github/v/release/kuhl-haus/kuhl-haus-mdp-servers?style=flat-square)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/releases)  
+[![Build Status](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml)
+[![CodeQL Advanced](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml/badge.svg)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml) 
+[![Downloads](https://static.pepy.tech/badge/kuhl-haus-mdp-servers/month)](https://pepy.tech/project/kuhl-haus-mdp-servers)
+[![GitHub last commit](https://img.shields.io/github/last-commit/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/branches)
+[![GitHub issues](https://img.shields.io/github/issues/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues) 
+[![GitHub pull requests](https://img.shields.io/github/issues-pr/kuhl-haus/kuhl-haus-mdp-servers)](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/pulls)
+
+
+
+# kuhl-haus-mdp-servers
+Container image build repository for market data processing servers

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus_mdp_servers.egg-info/SOURCES.txt

@@ -0,0 +1,15 @@
+README.md
+pyproject.toml
+setup.cfg
+setup.py
+src/kuhl_haus/servers/__init__.py
+src/kuhl_haus/servers/mdl_server.py
+src/kuhl_haus/servers/mdp_server.py
+src/kuhl_haus/servers/wds_server.py
+src/kuhl_haus_mdp_servers.egg-info/PKG-INFO
+src/kuhl_haus_mdp_servers.egg-info/SOURCES.txt
+src/kuhl_haus_mdp_servers.egg-info/dependency_links.txt
+src/kuhl_haus_mdp_servers.egg-info/entry_points.txt
+src/kuhl_haus_mdp_servers.egg-info/requires.txt
+src/kuhl_haus_mdp_servers.egg-info/top_level.txt
+src/kuhl_haus_mdp_servers.egg-info/zip-safe

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus_mdp_servers.egg-info/dependency_links.txt

@@ -0,0 +1 @@
+

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus_mdp_servers.egg-info/entry_points.txt

@@ -0,0 +1,4 @@
+[console_scripts]
+mdl_server = kuhl_haus.servers.mdl_server:app
+mdp_server = kuhl_haus.servers.mdp_server:app
+wds_server = kuhl_haus.servers.wds_server:app

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus_mdp_servers.egg-info/requires.txt

@@ -0,0 +1,7 @@
+kuhl-haus-mdp
+
+[testing]
+setuptools
+pdm-backend
+pytest
+pytest-cov

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus_mdp_servers.egg-info/top_level.txt

@@ -0,0 +1 @@
+kuhl_haus

kuhl_haus_mdp_servers-0.0.0/src/kuhl_haus_mdp_servers.egg-info/zip-safe

@@ -0,0 +1 @@
+