kuhl-haus-mdp-servers 0.1.2__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kuhl_haus/servers/mdl_server.py +7 -7
- kuhl_haus/servers/wds_server.py +1 -11
- kuhl_haus_mdp_servers-0.1.4.dist-info/METADATA +117 -0
- kuhl_haus_mdp_servers-0.1.4.dist-info/RECORD +9 -0
- kuhl_haus_mdp_servers-0.1.2.dist-info/METADATA +0 -37
- kuhl_haus_mdp_servers-0.1.2.dist-info/RECORD +0 -9
- {kuhl_haus_mdp_servers-0.1.2.dist-info → kuhl_haus_mdp_servers-0.1.4.dist-info}/WHEEL +0 -0
- {kuhl_haus_mdp_servers-0.1.2.dist-info → kuhl_haus_mdp_servers-0.1.4.dist-info}/entry_points.txt +0 -0
- {kuhl_haus_mdp_servers-0.1.2.dist-info → kuhl_haus_mdp_servers-0.1.4.dist-info}/licenses/LICENSE.txt +0 -0
kuhl_haus/servers/mdl_server.py
CHANGED
|
@@ -12,12 +12,6 @@ from kuhl_haus.mdp.integ.massive_data_queues import MassiveDataQueues
|
|
|
12
12
|
from kuhl_haus.mdp.integ.massive_data_listener import MassiveDataListener
|
|
13
13
|
from kuhl_haus.mdp.integ.utils import get_massive_api_key
|
|
14
14
|
|
|
15
|
-
logging.basicConfig(
|
|
16
|
-
level=logging.INFO,
|
|
17
|
-
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
18
|
-
)
|
|
19
|
-
logger = logging.getLogger(__name__)
|
|
20
|
-
|
|
21
15
|
|
|
22
16
|
class Settings(BaseSettings):
|
|
23
17
|
# TODO: Retrieve Massive client settings from Service Control Plane API call
|
|
@@ -28,7 +22,7 @@ class Settings(BaseSettings):
|
|
|
28
22
|
# The default values can be overridden via environment variable; use the API to manage at runtime.
|
|
29
23
|
feed: Union[str, Feed] = os.environ.get("MASSIVE_FEED", Feed.RealTime)
|
|
30
24
|
market: Union[str, Market] = os.environ.get("MASSIVE_MARKET", Market.Stocks)
|
|
31
|
-
subscriptions: Optional[List[str]] = os.environ.get("MASSIVE_SUBSCRIPTIONS", ["
|
|
25
|
+
subscriptions: Optional[List[str]] = os.environ.get("MASSIVE_SUBSCRIPTIONS", ["AM.*"])
|
|
32
26
|
|
|
33
27
|
# Additional Massive/Polygon.io Settings - default values can be overridden via environment variables
|
|
34
28
|
raw: bool = os.environ.get("MASSIVE_RAW", False)
|
|
@@ -55,6 +49,12 @@ class Settings(BaseSettings):
|
|
|
55
49
|
|
|
56
50
|
settings = Settings()
|
|
57
51
|
|
|
52
|
+
logging.basicConfig(
|
|
53
|
+
level=settings.log_level,
|
|
54
|
+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
|
|
55
|
+
)
|
|
56
|
+
logger = logging.getLogger(__name__)
|
|
57
|
+
|
|
58
58
|
# Global state
|
|
59
59
|
massive_data_queues: Optional[MassiveDataQueues] = None
|
|
60
60
|
massive_data_listener: Optional[MassiveDataListener] = None
|
kuhl_haus/servers/wds_server.py
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
import asyncio
|
|
2
1
|
import json
|
|
3
2
|
import logging
|
|
4
3
|
import os
|
|
@@ -18,7 +17,7 @@ class UnauthorizedException(Exception):
|
|
|
18
17
|
|
|
19
18
|
class Settings(BaseSettings):
|
|
20
19
|
# Redis Settings
|
|
21
|
-
redis_url: str = os.environ.get("REDIS_URL", "redis://
|
|
20
|
+
redis_url: str = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
|
|
22
21
|
|
|
23
22
|
# Server Settings
|
|
24
23
|
server_ip: str = os.environ.get("SERVER_IP", "0.0.0.0")
|
|
@@ -80,15 +79,6 @@ async def root():
|
|
|
80
79
|
return RedirectResponse(url="/health")
|
|
81
80
|
|
|
82
81
|
|
|
83
|
-
@app.get("/restart")
|
|
84
|
-
async def restart_wds_service():
|
|
85
|
-
logger.info("Restarting Widget Data Service...")
|
|
86
|
-
await wds_service.stop()
|
|
87
|
-
await asyncio.sleep(1)
|
|
88
|
-
await wds_service.start()
|
|
89
|
-
logger.info("Widget Data Service restarted successfully.")
|
|
90
|
-
|
|
91
|
-
|
|
92
82
|
@app.get("/health", status_code=200)
|
|
93
83
|
async def health_check(response: Response):
|
|
94
84
|
"""Kubernetes health check endpoint."""
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: kuhl-haus-mdp-servers
|
|
3
|
+
Version: 0.1.4
|
|
4
|
+
Summary: Container image build repository for market data processing servers
|
|
5
|
+
Author-Email: Tom Pounders <git@oldschool.engineer>
|
|
6
|
+
License-File: LICENSE.txt
|
|
7
|
+
Classifier: Development Status :: 4 - Beta
|
|
8
|
+
Classifier: Programming Language :: Python
|
|
9
|
+
Project-URL: Homepage, https://github.com/kuhl-haus/kuhl-haus-mdp-servers
|
|
10
|
+
Project-URL: Documentation, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/wiki
|
|
11
|
+
Project-URL: Source, https://github.com/kuhl-haus/kuhl-haus-mdp-servers.git
|
|
12
|
+
Project-URL: Changelog, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/commits
|
|
13
|
+
Project-URL: Tracker, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues
|
|
14
|
+
Requires-Python: <3.13,>=3.9.21
|
|
15
|
+
Requires-Dist: kuhl-haus-mdp
|
|
16
|
+
Requires-Dist: websockets
|
|
17
|
+
Requires-Dist: aio-pika
|
|
18
|
+
Requires-Dist: redis[asyncio]
|
|
19
|
+
Requires-Dist: tenacity
|
|
20
|
+
Requires-Dist: fastapi
|
|
21
|
+
Requires-Dist: uvicorn[standard]
|
|
22
|
+
Requires-Dist: pydantic-settings
|
|
23
|
+
Requires-Dist: python-dotenv
|
|
24
|
+
Requires-Dist: massive
|
|
25
|
+
Provides-Extra: testing
|
|
26
|
+
Requires-Dist: setuptools; extra == "testing"
|
|
27
|
+
Requires-Dist: pdm-backend; extra == "testing"
|
|
28
|
+
Requires-Dist: pytest; extra == "testing"
|
|
29
|
+
Requires-Dist: pytest-cov; extra == "testing"
|
|
30
|
+
Description-Content-Type: text/markdown
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/blob/mainline/LICENSE.txt)
|
|
34
|
+
[](https://pypi.org/project/kuhl-haus-mdp-servers/)
|
|
35
|
+
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/releases)
|
|
36
|
+
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml)
|
|
37
|
+
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml)
|
|
38
|
+
[](https://pepy.tech/project/kuhl-haus-mdp-servers)
|
|
39
|
+
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/branches)
|
|
40
|
+
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues)
|
|
41
|
+
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/pulls)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
# kuhl-haus-mdp-servers
|
|
46
|
+
Container image build repository for market data processing servers
|
|
47
|
+
|
|
48
|
+
## TL;DR
|
|
49
|
+
Non-business Massive (AKA Polygon.IO) accounts are limited to a single WebSocket connection per asset class and it has to be fast enough to handle messages in a non-blocking fashion or it'll get disconnected. The market data processing pipeline consists of loosely-coupled market data processing components so that a single WebSocket connection can handle messages fast enough to maintain a reliable connection with the market data provider.
|
|
50
|
+
|
|
51
|
+
Per, https://massive.com/docs/websocket/quickstart#connecting-to-the-websocket:
|
|
52
|
+
> *By default, one concurrent WebSocket connection per asset class is allowed. If you require multiple simultaneous connections for the same asset class, please [contact support](https://massive.com/contact).*
|
|
53
|
+
|
|
54
|
+
# Components Summary
|
|
55
|
+
|
|
56
|
+
Non-business Massive (AKA Polygon.IO) accounts are limited to a single WebSocket connection per asset class and it has to be fast enough to handle messages in a non-blocking fashion or it'll get disconnected. The Market Data Listener (MDL) connects to the Market Data Source (Massive) and subscribes to unfiltered feeds. MDL inspects the message type for selecting the appropriate serialization method and destination Market Data Queue (MDQ). The Market Data Processors (MDP) subscribe to raw market data in the MDQ and perform the heavy lifting that would otherwise constrain the message handling speed of the MDL. This decoupling allows the MDP and MDL to scale independently. Post-processed market data is stored in the MDC for consumption by the Widget Data Service (WDS). Client-side widgets receive market data from the WDS, which provides a WebSocket interface to MDC pub/sub streams and cached data.
|
|
57
|
+
|
|
58
|
+
[]
|
|
59
|
+
|
|
60
|
+
# Component Descriptions
|
|
61
|
+
|
|
62
|
+
## Market Data Listener (MDL)
|
|
63
|
+
The MDL performs minimal processing on the messages. MDL inspects the message type for selecting the appropriate serialization method and destination queue. MDL implementations may vary as new MDS become available (for example, news).
|
|
64
|
+
|
|
65
|
+
MDL runs as a container and scales independently of other components. The MDL should not be accessible outside the data plane local network.
|
|
66
|
+
|
|
67
|
+
## Market Data Queues (MDQ)
|
|
68
|
+
|
|
69
|
+
**Purpose:** Buffer high-velocity market data stream for server-side processing with aggressive freshness controls
|
|
70
|
+
- **Queue Type:** FIFO with TTL (5-second max message age)
|
|
71
|
+
- **Cleanup Strategy:** Discarded when TTL expires
|
|
72
|
+
- **Message Format:** Timestamped JSON preserving original Massive.com structure
|
|
73
|
+
- **Durability:** Non-persistent messages (speed over reliability for real-time data)
|
|
74
|
+
- **Independence:** Queues operate completely independently - one queue per subscription
|
|
75
|
+
- **Technology**: RabbitMQ
|
|
76
|
+
|
|
77
|
+
The MDQ should not be accessible outside the data plane local network.
|
|
78
|
+
|
|
79
|
+
## Market Data Processors (MDP)
|
|
80
|
+
The purpose of the MDP is to process raw real-time market data and delegate processing to data-specific handlers. This separation of concerns allows MDPs to handle any type of data and simplifies horizontal scaling. The MDP stores its processed results in the Market Data Cache (MDC).
|
|
81
|
+
|
|
82
|
+
The MDP:
|
|
83
|
+
- Hydrates the in-memory cache on MDC
|
|
84
|
+
- Processes market data
|
|
85
|
+
- Publishes messages to pub/sub channels
|
|
86
|
+
- Maintains cache entries in MDC
|
|
87
|
+
|
|
88
|
+
MDPs runs as containers and scale independently of other components. The MDPs should not be accessible outside the data plane local network.
|
|
89
|
+
|
|
90
|
+
## Market Data Cache (MDC)
|
|
91
|
+
|
|
92
|
+
**Purpose:** In-memory data store for serialized processed market data.
|
|
93
|
+
* **Cache Type**: In-memory persistent or with TTL
|
|
94
|
+
- **Queue Type:** pub/sub
|
|
95
|
+
- **Technology**: Redis
|
|
96
|
+
|
|
97
|
+
The MDC should not be accessible outside the data plane local network.
|
|
98
|
+
|
|
99
|
+
## Widget Data Service (WDS)
|
|
100
|
+
**Purpose**:
|
|
101
|
+
1. WebSocket interface provides access to processed market data for client-side code
|
|
102
|
+
2. Is the network-layer boundary between clients and the data that is available on the data plane
|
|
103
|
+
|
|
104
|
+
WDS runs as a container and scales independently of other components. WDS is the only data plane component that should be exposed to client networks.
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
## Service Control Plane (SCP)
|
|
108
|
+
**Purpose**:
|
|
109
|
+
1. Authentication and authorization
|
|
110
|
+
2. Serve static and dynamic content via py4web
|
|
111
|
+
3. Serve SPA to authenticated clients
|
|
112
|
+
4. Injects authentication token and WDS url into SPA environment for authenticated access to WDS
|
|
113
|
+
5. Control plane for managing application components at runtime
|
|
114
|
+
6. API for programmatic access to service controls and instrumentation.
|
|
115
|
+
|
|
116
|
+
The SCP requires access to the data plane network for API access to data plane components.
|
|
117
|
+
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
kuhl_haus/servers/__init__.py,sha256=5dEpAdB3kypH8tCRECoXwbly1WV9kFU5kh8ldGSa0VI,349
|
|
2
|
+
kuhl_haus/servers/mdl_server.py,sha256=iSkP7GANluStDeDqFJhL4IOAQNo94jUiX9HbAnmuazc,10273
|
|
3
|
+
kuhl_haus/servers/mdp_server.py,sha256=sUeZOmSIjzPHIiYmZKlpmIy2Lag4FLZivI9J6XGo-PQ,9017
|
|
4
|
+
kuhl_haus/servers/wds_server.py,sha256=DYLbq8Al67h-p4TzUJYl-RCU61izSUYYNtzvzb5KGyM,6418
|
|
5
|
+
kuhl_haus_mdp_servers-0.1.4.dist-info/METADATA,sha256=GjgadtRvros0_LXVY9k7-byaG7wmlOWTGPvXZ2FlUSE,7347
|
|
6
|
+
kuhl_haus_mdp_servers-0.1.4.dist-info/WHEEL,sha256=tsUv_t7BDeJeRHaSrczbGeuK-TtDpGsWi_JfpzD255I,90
|
|
7
|
+
kuhl_haus_mdp_servers-0.1.4.dist-info/entry_points.txt,sha256=LPD1rLjALMFQF-BcJTcVMcZRqR89QPSqu05BHC4BBWc,172
|
|
8
|
+
kuhl_haus_mdp_servers-0.1.4.dist-info/licenses/LICENSE.txt,sha256=j5GdO9_Y_RH0lr1tsVAY1rtBJxoyC_-MmqaFNDE3dwo,1066
|
|
9
|
+
kuhl_haus_mdp_servers-0.1.4.dist-info/RECORD,,
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
Metadata-Version: 2.4
|
|
2
|
-
Name: kuhl-haus-mdp-servers
|
|
3
|
-
Version: 0.1.2
|
|
4
|
-
Summary: Container image build repository for market data processing servers
|
|
5
|
-
Author-Email: Tom Pounders <git@oldschool.engineer>
|
|
6
|
-
License-File: LICENSE.txt
|
|
7
|
-
Classifier: Development Status :: 4 - Beta
|
|
8
|
-
Classifier: Programming Language :: Python
|
|
9
|
-
Project-URL: Homepage, https://github.com/kuhl-haus/kuhl-haus-mdp-servers
|
|
10
|
-
Project-URL: Documentation, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/wiki
|
|
11
|
-
Project-URL: Source, https://github.com/kuhl-haus/kuhl-haus-mdp-servers.git
|
|
12
|
-
Project-URL: Changelog, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/commits
|
|
13
|
-
Project-URL: Tracker, https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues
|
|
14
|
-
Requires-Python: <3.13,>=3.9.21
|
|
15
|
-
Requires-Dist: kuhl-haus-mdp
|
|
16
|
-
Provides-Extra: testing
|
|
17
|
-
Requires-Dist: setuptools; extra == "testing"
|
|
18
|
-
Requires-Dist: pdm-backend; extra == "testing"
|
|
19
|
-
Requires-Dist: pytest; extra == "testing"
|
|
20
|
-
Requires-Dist: pytest-cov; extra == "testing"
|
|
21
|
-
Description-Content-Type: text/markdown
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/blob/mainline/LICENSE.txt)
|
|
25
|
-
[](https://pypi.org/project/kuhl-haus-mdp-servers/)
|
|
26
|
-
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/releases)
|
|
27
|
-
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/build-images.yml)
|
|
28
|
-
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/actions/workflows/codeql.yml)
|
|
29
|
-
[](https://pepy.tech/project/kuhl-haus-mdp-servers)
|
|
30
|
-
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/branches)
|
|
31
|
-
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/issues)
|
|
32
|
-
[](https://github.com/kuhl-haus/kuhl-haus-mdp-servers/pulls)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
# kuhl-haus-mdp-servers
|
|
37
|
-
Container image build repository for market data processing servers
|
|
@@ -1,9 +0,0 @@
|
|
|
1
|
-
kuhl_haus/servers/__init__.py,sha256=5dEpAdB3kypH8tCRECoXwbly1WV9kFU5kh8ldGSa0VI,349
|
|
2
|
-
kuhl_haus/servers/mdl_server.py,sha256=GObfM0DHBwOSMV6JusW34KmdAwVyrsRimVo0eIvGkcI,10266
|
|
3
|
-
kuhl_haus/servers/mdp_server.py,sha256=sUeZOmSIjzPHIiYmZKlpmIy2Lag4FLZivI9J6XGo-PQ,9017
|
|
4
|
-
kuhl_haus/servers/wds_server.py,sha256=evEyGWsEhs0D4fbQHnyr-eosMlElewNsbdrLKkq3iBY,6703
|
|
5
|
-
kuhl_haus_mdp_servers-0.1.2.dist-info/METADATA,sha256=89qNmV0FV_FQmCOvG8JP7qH5JB-ZB0U_CXIXM_MPL3Y,2538
|
|
6
|
-
kuhl_haus_mdp_servers-0.1.2.dist-info/WHEEL,sha256=tsUv_t7BDeJeRHaSrczbGeuK-TtDpGsWi_JfpzD255I,90
|
|
7
|
-
kuhl_haus_mdp_servers-0.1.2.dist-info/entry_points.txt,sha256=LPD1rLjALMFQF-BcJTcVMcZRqR89QPSqu05BHC4BBWc,172
|
|
8
|
-
kuhl_haus_mdp_servers-0.1.2.dist-info/licenses/LICENSE.txt,sha256=j5GdO9_Y_RH0lr1tsVAY1rtBJxoyC_-MmqaFNDE3dwo,1066
|
|
9
|
-
kuhl_haus_mdp_servers-0.1.2.dist-info/RECORD,,
|
|
File without changes
|
{kuhl_haus_mdp_servers-0.1.2.dist-info → kuhl_haus_mdp_servers-0.1.4.dist-info}/entry_points.txt
RENAMED
|
File without changes
|
{kuhl_haus_mdp_servers-0.1.2.dist-info → kuhl_haus_mdp_servers-0.1.4.dist-info}/licenses/LICENSE.txt
RENAMED
|
File without changes
|