kuhl-haus-mdp 0.1.2__tar.gz → 0.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/PKG-INFO +8 -6
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/pyproject.toml +8 -6
- kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/analyzers/top_stocks.py +220 -0
- kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/components/market_data_cache.py +143 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/massive_data_processor.py +1 -4
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_cache_keys.py +3 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_cache_ttl.py +1 -0
- kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/models/top_stocks_cache_item.py +143 -0
- {kuhl_haus_mdp-0.1.2/tests → kuhl_haus_mdp-0.1.5/tests/analyzers}/test_massive_data_analyzer.py +2 -2
- kuhl_haus_mdp-0.1.5/tests/analyzers/test_top_stocks_rehydrate.py +108 -0
- kuhl_haus_mdp-0.1.5/tests/components/test_market_data_cache.py +708 -0
- kuhl_haus_mdp-0.1.5/tests/helpers/test_utils.py +93 -0
- kuhl_haus_mdp-0.1.5/tests/integ/__init__.py +0 -0
- kuhl_haus_mdp-0.1.5/tests/models/__init__.py +0 -0
- kuhl_haus_mdp-0.1.5/tests/models/test_top_stocks_cache_item.py +109 -0
- kuhl_haus_mdp-0.1.2/src/kuhl_haus/mdp/analyzers/top_stocks.py +0 -408
- kuhl_haus_mdp-0.1.2/src/kuhl_haus/mdp/components/market_data_cache.py +0 -29
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/LICENSE.txt +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/README.md +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/analyzers/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/analyzers/analyzer.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/analyzers/massive_data_analyzer.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/components/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/components/market_data_scanner.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/components/widget_data_service.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/helpers/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/helpers/process_manager.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/helpers/queue_name_resolver.py +0 -0
- {kuhl_haus_mdp-0.1.2/src/kuhl_haus/mdp/integ → kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/helpers}/utils.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/massive_data_listener.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/massive_data_queues.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/web_socket_message_serde.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_analyzer_result.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_pubsub_keys.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_scanner_names.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/massive_data_queue.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2/tests/components → kuhl_haus_mdp-0.1.5/tests/analyzers}/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2/tests/helpers → kuhl_haus_mdp-0.1.5/tests/components}/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/components/test_market_data_scanner.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/components/test_widget_data_service.py +0 -0
- {kuhl_haus_mdp-0.1.2/tests/integ → kuhl_haus_mdp-0.1.5/tests/helpers}/__init__.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/helpers/test_process_manager.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/helpers/test_queue_name_resolver.py +0 -0
- {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/integ/test_web_socket_message_serde.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: kuhl-haus-mdp
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.5
|
|
4
4
|
Summary: Market data processing pipeline for stock market scanner
|
|
5
5
|
Author-Email: Tom Pounders <git@oldschool.engineer>
|
|
6
6
|
License: The MIT License (MIT)
|
|
@@ -33,20 +33,22 @@ Project-URL: Source, https://github.com/kuhl-haus/kuhl-haus-mdp.git
|
|
|
33
33
|
Project-URL: Changelog, https://github.com/kuhl-haus/kuhl-haus-mdp/commits
|
|
34
34
|
Project-URL: Tracker, https://github.com/kuhl-haus/kuhl-haus-mdp/issues
|
|
35
35
|
Requires-Python: <3.13,>=3.9.21
|
|
36
|
-
Requires-Dist:
|
|
36
|
+
Requires-Dist: aiohttp
|
|
37
37
|
Requires-Dist: aio-pika
|
|
38
|
-
Requires-Dist: redis[asyncio]
|
|
39
|
-
Requires-Dist: tenacity
|
|
40
38
|
Requires-Dist: fastapi
|
|
41
|
-
Requires-Dist:
|
|
39
|
+
Requires-Dist: massive
|
|
42
40
|
Requires-Dist: pydantic-settings
|
|
43
41
|
Requires-Dist: python-dotenv
|
|
44
|
-
Requires-Dist:
|
|
42
|
+
Requires-Dist: redis[asyncio]
|
|
43
|
+
Requires-Dist: tenacity
|
|
44
|
+
Requires-Dist: uvicorn[standard]
|
|
45
|
+
Requires-Dist: websockets
|
|
45
46
|
Provides-Extra: testing
|
|
46
47
|
Requires-Dist: setuptools; extra == "testing"
|
|
47
48
|
Requires-Dist: pdm-backend; extra == "testing"
|
|
48
49
|
Requires-Dist: pytest; extra == "testing"
|
|
49
50
|
Requires-Dist: pytest-cov; extra == "testing"
|
|
51
|
+
Requires-Dist: pytest-asyncio; extra == "testing"
|
|
50
52
|
Description-Content-Type: text/markdown
|
|
51
53
|
|
|
52
54
|
<!-- These are examples of badges you might want to add to your README:
|
|
@@ -19,17 +19,18 @@ classifiers = [
|
|
|
19
19
|
"Programming Language :: Python",
|
|
20
20
|
]
|
|
21
21
|
dependencies = [
|
|
22
|
-
"
|
|
22
|
+
"aiohttp",
|
|
23
23
|
"aio-pika",
|
|
24
|
-
"redis[asyncio]",
|
|
25
|
-
"tenacity",
|
|
26
24
|
"fastapi",
|
|
27
|
-
"
|
|
25
|
+
"massive",
|
|
28
26
|
"pydantic-settings",
|
|
29
27
|
"python-dotenv",
|
|
30
|
-
"
|
|
28
|
+
"redis[asyncio]",
|
|
29
|
+
"tenacity",
|
|
30
|
+
"uvicorn[standard]",
|
|
31
|
+
"websockets",
|
|
31
32
|
]
|
|
32
|
-
version = "0.1.
|
|
33
|
+
version = "0.1.5"
|
|
33
34
|
|
|
34
35
|
[project.license]
|
|
35
36
|
file = "LICENSE.txt"
|
|
@@ -47,6 +48,7 @@ testing = [
|
|
|
47
48
|
"pdm-backend",
|
|
48
49
|
"pytest",
|
|
49
50
|
"pytest-cov",
|
|
51
|
+
"pytest-asyncio",
|
|
50
52
|
]
|
|
51
53
|
|
|
52
54
|
[tool.setuptools_scm]
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import time
|
|
3
|
+
from datetime import datetime, timezone, timedelta
|
|
4
|
+
from typing import Optional, List, Iterator
|
|
5
|
+
from zoneinfo import ZoneInfo
|
|
6
|
+
|
|
7
|
+
from massive.exceptions import BadResponse
|
|
8
|
+
from massive.rest import RESTClient
|
|
9
|
+
from massive.rest.models import (
|
|
10
|
+
TickerSnapshot,
|
|
11
|
+
Agg,
|
|
12
|
+
)
|
|
13
|
+
from massive.websocket.models import (
|
|
14
|
+
EquityAgg,
|
|
15
|
+
EventType
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
from kuhl_haus.mdp.analyzers.analyzer import Analyzer
|
|
19
|
+
from kuhl_haus.mdp.components.market_data_cache import MarketDataCache
|
|
20
|
+
from kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
|
|
21
|
+
from kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
|
|
22
|
+
from kuhl_haus.mdp.models.market_data_pubsub_keys import MarketDataPubSubKeys
|
|
23
|
+
from kuhl_haus.mdp.models.top_stocks_cache_item import TopStocksCacheItem
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class TopStocksAnalyzer(Analyzer):
|
|
27
|
+
|
|
28
|
+
def __init__(self, cache: MarketDataCache, **kwargs):
|
|
29
|
+
if "cache_key" not in kwargs:
|
|
30
|
+
kwargs["cache_key"] = MarketDataCacheKeys.TOP_STOCKS_SCANNER.value
|
|
31
|
+
super().__init__(**kwargs)
|
|
32
|
+
self.cache = cache
|
|
33
|
+
self.logger = logging.getLogger(__name__)
|
|
34
|
+
self.cache_item = TopStocksCacheItem()
|
|
35
|
+
self.last_update_time = 0
|
|
36
|
+
self.pre_market_reset = False
|
|
37
|
+
|
|
38
|
+
async def rehydrate(self, data: dict):
|
|
39
|
+
if not data:
|
|
40
|
+
self.cache_item = TopStocksCacheItem()
|
|
41
|
+
self.logger.info("No data to rehydrate TopStocksCacheItem.")
|
|
42
|
+
return
|
|
43
|
+
|
|
44
|
+
# Get current time in UTC, then convert to Eastern Time
|
|
45
|
+
utc_now = datetime.now(timezone.utc)
|
|
46
|
+
et_now = utc_now.astimezone(ZoneInfo("America/New_York"))
|
|
47
|
+
|
|
48
|
+
# Check if within trading hours: Mon-Fri, 04:00-19:59 ET
|
|
49
|
+
is_weekday = et_now.weekday() < 5
|
|
50
|
+
is_trading_hours = 4 <= et_now.hour < 20
|
|
51
|
+
if not is_weekday or not is_trading_hours:
|
|
52
|
+
self.cache_item = TopStocksCacheItem()
|
|
53
|
+
self.logger.info(f"Outside market hours ({et_now.strftime('%H:%M:%S %Z')}), clearing cache.")
|
|
54
|
+
return
|
|
55
|
+
self.cache_item = TopStocksCacheItem(**data)
|
|
56
|
+
self.logger.info("Rehydrated TopStocksCacheItem")
|
|
57
|
+
|
|
58
|
+
async def analyze_data(self, data: dict) -> Optional[List[MarketDataAnalyzerResult]]:
|
|
59
|
+
utc_now = datetime.now(timezone.utc)
|
|
60
|
+
et_now = utc_now.astimezone(ZoneInfo("America/New_York"))
|
|
61
|
+
current_day = et_now.replace(hour=4, minute=0, second=0, microsecond=0).timestamp()
|
|
62
|
+
if current_day != self.cache_item.day_start_time:
|
|
63
|
+
self.logger.info(f"New day: {current_day} - resetting cache.")
|
|
64
|
+
self.cache_item = TopStocksCacheItem()
|
|
65
|
+
self.cache_item.day_start_time = current_day
|
|
66
|
+
elif et_now.hour == 9 and et_now.minute == 30 and not self.pre_market_reset:
|
|
67
|
+
self.logger.info("Market is now open; resetting symbol data cache.")
|
|
68
|
+
self.cache_item.symbol_data_cache = {}
|
|
69
|
+
self.pre_market_reset = True
|
|
70
|
+
|
|
71
|
+
event_type = data.get("event_type")
|
|
72
|
+
symbol = data.get("symbol")
|
|
73
|
+
if not event_type:
|
|
74
|
+
self.logger.info(f"Discarding data: {data}")
|
|
75
|
+
return None
|
|
76
|
+
elif not symbol:
|
|
77
|
+
self.logger.info(f"Discarding data: {data}")
|
|
78
|
+
return None
|
|
79
|
+
elif event_type == EventType.EquityAgg.value:
|
|
80
|
+
self.logger.debug(f"Processing EquityAgg: {data.get('symbol')}")
|
|
81
|
+
await self.handle_equity_agg(EquityAgg(**data))
|
|
82
|
+
elif event_type == EventType.EquityAggMin.value:
|
|
83
|
+
self.logger.debug(f"Processing EquityAggMin: {data.get('symbol')}")
|
|
84
|
+
await self.handle_equity_agg(EquityAgg(**data))
|
|
85
|
+
else:
|
|
86
|
+
self.logger.info(f"Discarding data: {data}")
|
|
87
|
+
return None
|
|
88
|
+
current_time = int(time.time())
|
|
89
|
+
# return results once per second
|
|
90
|
+
if current_time <= self.last_update_time:
|
|
91
|
+
return None
|
|
92
|
+
self.last_update_time = current_time
|
|
93
|
+
|
|
94
|
+
result = [
|
|
95
|
+
MarketDataAnalyzerResult(
|
|
96
|
+
data=self.cache_item.to_dict(),
|
|
97
|
+
cache_key=self.cache_key,
|
|
98
|
+
cache_ttl=28500, # 7 hours, 55 minutes
|
|
99
|
+
),
|
|
100
|
+
MarketDataAnalyzerResult(
|
|
101
|
+
data=self.cache_item.top_volume(100),
|
|
102
|
+
cache_key=MarketDataPubSubKeys.TOP_VOLUME_SCANNER.value,
|
|
103
|
+
cache_ttl=259200, # 3 days
|
|
104
|
+
publish_key=MarketDataPubSubKeys.TOP_VOLUME_SCANNER.value,
|
|
105
|
+
),
|
|
106
|
+
MarketDataAnalyzerResult(
|
|
107
|
+
data=self.cache_item.top_gainers(500),
|
|
108
|
+
cache_key=MarketDataPubSubKeys.TOP_GAINERS_SCANNER.value,
|
|
109
|
+
cache_ttl=259200, # 3 days
|
|
110
|
+
publish_key=MarketDataPubSubKeys.TOP_GAINERS_SCANNER.value,
|
|
111
|
+
),
|
|
112
|
+
MarketDataAnalyzerResult(
|
|
113
|
+
data=self.cache_item.top_gappers(500),
|
|
114
|
+
cache_key=MarketDataPubSubKeys.TOP_GAPPERS_SCANNER.value,
|
|
115
|
+
cache_ttl=259200, # 3 days
|
|
116
|
+
publish_key=MarketDataPubSubKeys.TOP_GAPPERS_SCANNER.value,
|
|
117
|
+
)
|
|
118
|
+
]
|
|
119
|
+
|
|
120
|
+
return result
|
|
121
|
+
|
|
122
|
+
async def handle_equity_agg(self, event: EquityAgg):
|
|
123
|
+
# Get data from symbol data cache or Rest API
|
|
124
|
+
if event.symbol in self.cache_item.symbol_data_cache:
|
|
125
|
+
cached_data = self.cache_item.symbol_data_cache[event.symbol]
|
|
126
|
+
avg_volume = cached_data["avg_volume"]
|
|
127
|
+
prev_day_close = cached_data["prev_day_close"]
|
|
128
|
+
prev_day_volume = cached_data["prev_day_volume"]
|
|
129
|
+
prev_day_vwap = cached_data["prev_day_vwap"]
|
|
130
|
+
else:
|
|
131
|
+
# Get snapshot for previous day's data
|
|
132
|
+
retry_count = 0
|
|
133
|
+
max_tries = 3
|
|
134
|
+
prev_day_close = 0
|
|
135
|
+
prev_day_volume = 0
|
|
136
|
+
prev_day_vwap = 0
|
|
137
|
+
while retry_count < max_tries:
|
|
138
|
+
try:
|
|
139
|
+
snapshot = await self.cache.get_ticker_snapshot(event.symbol)
|
|
140
|
+
prev_day_close = snapshot.prev_day.close
|
|
141
|
+
prev_day_volume = snapshot.prev_day.volume
|
|
142
|
+
prev_day_vwap = snapshot.prev_day.vwap
|
|
143
|
+
break
|
|
144
|
+
except BadResponse as e:
|
|
145
|
+
self.logger.error(f"Error getting snapshot for {event.symbol}: {repr(e)}", exc_info=e, stack_info=True)
|
|
146
|
+
retry_count += 1
|
|
147
|
+
if retry_count == max_tries and prev_day_close == 0:
|
|
148
|
+
self.logger.error(f"Failed to get snapshot for {event.symbol} after {max_tries} tries.")
|
|
149
|
+
return
|
|
150
|
+
|
|
151
|
+
# Get average volume
|
|
152
|
+
retry_count = 0
|
|
153
|
+
max_tries = 3
|
|
154
|
+
avg_volume = 0
|
|
155
|
+
while retry_count < max_tries:
|
|
156
|
+
try:
|
|
157
|
+
avg_volume = await self.cache.get_avg_volume(event.symbol)
|
|
158
|
+
break
|
|
159
|
+
except (BadResponse, ZeroDivisionError) as e:
|
|
160
|
+
self.logger.error(f"Error getting average volume for {event.symbol}: {repr(e)}", exc_info=e, stack_info=True)
|
|
161
|
+
retry_count += 1
|
|
162
|
+
if retry_count == max_tries and avg_volume == 0:
|
|
163
|
+
self.logger.error(f"Failed to get average volume for {event.symbol} after {max_tries} tries.")
|
|
164
|
+
return
|
|
165
|
+
|
|
166
|
+
# Calculate relative volume
|
|
167
|
+
if avg_volume == 0:
|
|
168
|
+
relative_volume = 0
|
|
169
|
+
else:
|
|
170
|
+
relative_volume = event.accumulated_volume / avg_volume
|
|
171
|
+
|
|
172
|
+
# Calculate percentage change since previous close
|
|
173
|
+
if prev_day_close == 0:
|
|
174
|
+
change = 0
|
|
175
|
+
pct_change = 0
|
|
176
|
+
else:
|
|
177
|
+
change = event.close - prev_day_close
|
|
178
|
+
pct_change = change / prev_day_close * 100
|
|
179
|
+
|
|
180
|
+
# Calculate percentage change since opening bell
|
|
181
|
+
change_since_open = 0
|
|
182
|
+
pct_change_since_open = 0
|
|
183
|
+
if event.official_open_price:
|
|
184
|
+
change_since_open = event.close - event.official_open_price
|
|
185
|
+
pct_change_since_open = change_since_open / event.official_open_price * 100
|
|
186
|
+
|
|
187
|
+
# Sort top tickers by accumulated volume
|
|
188
|
+
self.cache_item.top_volume_map[event.symbol] = event.accumulated_volume
|
|
189
|
+
|
|
190
|
+
# Sort top gappers by percentage gain since the previous day's close
|
|
191
|
+
self.cache_item.top_gappers_map[event.symbol] = pct_change
|
|
192
|
+
|
|
193
|
+
# Sort top gainers by percentage gain since the opening bell
|
|
194
|
+
self.cache_item.top_gainers_map[event.symbol] = pct_change_since_open
|
|
195
|
+
|
|
196
|
+
# Update symbol data cache
|
|
197
|
+
self.cache_item.symbol_data_cache[event.symbol] = {
|
|
198
|
+
"symbol": event.symbol,
|
|
199
|
+
"volume": event.volume,
|
|
200
|
+
"accumulated_volume": event.accumulated_volume,
|
|
201
|
+
"relative_volume": relative_volume,
|
|
202
|
+
"official_open_price": event.official_open_price,
|
|
203
|
+
"vwap": event.vwap,
|
|
204
|
+
"open": event.open,
|
|
205
|
+
"close": event.close,
|
|
206
|
+
"high": event.high,
|
|
207
|
+
"low": event.low,
|
|
208
|
+
"aggregate_vwap": event.aggregate_vwap,
|
|
209
|
+
"average_size": event.average_size,
|
|
210
|
+
"avg_volume": avg_volume,
|
|
211
|
+
"prev_day_close": prev_day_close,
|
|
212
|
+
"prev_day_volume": prev_day_volume,
|
|
213
|
+
"prev_day_vwap": prev_day_vwap,
|
|
214
|
+
"change": change,
|
|
215
|
+
"pct_change": pct_change,
|
|
216
|
+
"change_since_open": change_since_open,
|
|
217
|
+
"pct_change_since_open": pct_change_since_open,
|
|
218
|
+
"start_timestamp": event.start_timestamp,
|
|
219
|
+
"end_timestamp": event.end_timestamp,
|
|
220
|
+
}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
import json
|
|
2
|
+
import logging
|
|
3
|
+
from typing import Any, Optional, Iterator, List
|
|
4
|
+
|
|
5
|
+
import aiohttp
|
|
6
|
+
import redis.asyncio as aioredis
|
|
7
|
+
from massive.rest import RESTClient
|
|
8
|
+
from massive.rest.models import (
|
|
9
|
+
TickerSnapshot,
|
|
10
|
+
FinancialRatio,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
from kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
|
|
14
|
+
from kuhl_haus.mdp.models.market_data_cache_ttl import MarketDataCacheTTL
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class MarketDataCache:
|
|
18
|
+
def __init__(self, rest_client: RESTClient, redis_client: aioredis.Redis, massive_api_key: str):
|
|
19
|
+
self.logger = logging.getLogger(__name__)
|
|
20
|
+
self.rest_client = rest_client
|
|
21
|
+
self.massive_api_key = massive_api_key
|
|
22
|
+
self.redis_client = redis_client
|
|
23
|
+
self.http_session = None
|
|
24
|
+
|
|
25
|
+
async def get_cache(self, cache_key: str) -> Optional[dict]:
|
|
26
|
+
"""Fetch current value from Redis cache (for snapshot requests)."""
|
|
27
|
+
value = await self.redis_client.get(cache_key)
|
|
28
|
+
if value:
|
|
29
|
+
return json.loads(value)
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
async def cache_data(self, data: Any, cache_key: str, cache_ttl: int = 0):
|
|
33
|
+
if cache_ttl > 0:
|
|
34
|
+
await self.redis_client.setex(cache_key, cache_ttl, json.dumps(data))
|
|
35
|
+
else:
|
|
36
|
+
await self.redis_client.set(cache_key, json.dumps(data))
|
|
37
|
+
self.logger.debug(f"Cached data for {cache_key}")
|
|
38
|
+
|
|
39
|
+
async def publish_data(self, data: Any, publish_key: str = None):
|
|
40
|
+
await self.redis_client.publish(publish_key, json.dumps(data))
|
|
41
|
+
self.logger.debug(f"Published data for {publish_key}")
|
|
42
|
+
|
|
43
|
+
async def get_ticker_snapshot(self, ticker: str) -> TickerSnapshot:
|
|
44
|
+
self.logger.debug(f"Getting snapshot for {ticker}")
|
|
45
|
+
cache_key = f"{MarketDataCacheKeys.TICKER_SNAPSHOTS.value}:{ticker}"
|
|
46
|
+
result = await self.get_cache(cache_key=cache_key)
|
|
47
|
+
if result:
|
|
48
|
+
snapshot = TickerSnapshot.from_dict(**result)
|
|
49
|
+
else:
|
|
50
|
+
snapshot: TickerSnapshot = self.rest_client.get_snapshot_ticker(
|
|
51
|
+
market_type="stocks",
|
|
52
|
+
ticker=ticker
|
|
53
|
+
)
|
|
54
|
+
self.logger.debug(f"Snapshot result: {snapshot}")
|
|
55
|
+
await self.cache_data(
|
|
56
|
+
data=snapshot,
|
|
57
|
+
cache_key=cache_key,
|
|
58
|
+
cache_ttl=MarketDataCacheTTL.EIGHT_HOURS.value
|
|
59
|
+
)
|
|
60
|
+
return snapshot
|
|
61
|
+
|
|
62
|
+
async def get_avg_volume(self, ticker: str):
|
|
63
|
+
self.logger.debug(f"Getting average volume for {ticker}")
|
|
64
|
+
cache_key = f"{MarketDataCacheKeys.TICKER_AVG_VOLUME.value}:{ticker}"
|
|
65
|
+
avg_volume = await self.get_cache(cache_key=cache_key)
|
|
66
|
+
if avg_volume:
|
|
67
|
+
self.logger.debug(f"Returning cached value for {ticker}: {avg_volume}")
|
|
68
|
+
return avg_volume
|
|
69
|
+
|
|
70
|
+
results: Iterator[FinancialRatio] = self.rest_client.list_financials_ratios(ticker=ticker)
|
|
71
|
+
ratios: List[FinancialRatio] = []
|
|
72
|
+
for financial_ratio in results:
|
|
73
|
+
ratios.append(financial_ratio)
|
|
74
|
+
if len(ratios) == 1:
|
|
75
|
+
avg_volume = ratios[0].average_volume
|
|
76
|
+
else:
|
|
77
|
+
raise Exception(f"Unexpected number of financial ratios for {ticker}: {len(ratios)}")
|
|
78
|
+
|
|
79
|
+
self.logger.debug(f"average volume {ticker}: {avg_volume}")
|
|
80
|
+
await self.cache_data(
|
|
81
|
+
data=avg_volume,
|
|
82
|
+
cache_key=cache_key,
|
|
83
|
+
cache_ttl=MarketDataCacheTTL.TWELVE_HOURS.value
|
|
84
|
+
)
|
|
85
|
+
return avg_volume
|
|
86
|
+
|
|
87
|
+
async def get_free_float(self, ticker: str):
|
|
88
|
+
self.logger.debug(f"Getting free float for {ticker}")
|
|
89
|
+
cache_key = f"{MarketDataCacheKeys.TICKER_FREE_FLOAT.value}:{ticker}"
|
|
90
|
+
free_float = await self.get_cache(cache_key=cache_key)
|
|
91
|
+
if free_float:
|
|
92
|
+
self.logger.debug(f"Returning cached value for {ticker}: {free_float}")
|
|
93
|
+
return free_float
|
|
94
|
+
|
|
95
|
+
# NOTE: This endpoint is experimental and the interface may change.
|
|
96
|
+
# https://massive.com/docs/rest/stocks/fundamentals/float
|
|
97
|
+
url = f"https://api.massive.com/stocks/vX/float"
|
|
98
|
+
params = {
|
|
99
|
+
"ticker": ticker,
|
|
100
|
+
"apiKey": self.massive_api_key
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
session = await self.get_http_session()
|
|
104
|
+
try:
|
|
105
|
+
async with session.get(url, params=params, timeout=aiohttp.ClientTimeout(total=10)) as response:
|
|
106
|
+
response.raise_for_status()
|
|
107
|
+
data = await response.json()
|
|
108
|
+
|
|
109
|
+
# Extract free_float from response
|
|
110
|
+
if data.get("status") == "OK" and data.get("results") is not None:
|
|
111
|
+
results = data["results"]
|
|
112
|
+
if len(results) > 0:
|
|
113
|
+
free_float = results[0].get("free_float")
|
|
114
|
+
else:
|
|
115
|
+
raise Exception(f"No free float data returned for {ticker}")
|
|
116
|
+
else:
|
|
117
|
+
raise Exception(f"Invalid response from Massive API for {ticker}: {data}")
|
|
118
|
+
|
|
119
|
+
except aiohttp.ClientError as e:
|
|
120
|
+
self.logger.error(f"HTTP error fetching free float for {ticker}: {e}")
|
|
121
|
+
raise
|
|
122
|
+
except Exception as e:
|
|
123
|
+
self.logger.error(f"Error fetching free float for {ticker}: {e}")
|
|
124
|
+
raise
|
|
125
|
+
|
|
126
|
+
self.logger.debug(f"free float {ticker}: {free_float}")
|
|
127
|
+
await self.cache_data(
|
|
128
|
+
data=free_float,
|
|
129
|
+
cache_key=cache_key,
|
|
130
|
+
cache_ttl=MarketDataCacheTTL.TWELVE_HOURS.value
|
|
131
|
+
)
|
|
132
|
+
return free_float
|
|
133
|
+
|
|
134
|
+
async def get_http_session(self) -> aiohttp.ClientSession:
|
|
135
|
+
"""Get or create aiohttp session for async HTTP requests."""
|
|
136
|
+
if self.http_session is None or self.http_session.closed:
|
|
137
|
+
self.http_session = aiohttp.ClientSession()
|
|
138
|
+
return self.http_session
|
|
139
|
+
|
|
140
|
+
async def close(self):
|
|
141
|
+
"""Close aiohttp session."""
|
|
142
|
+
if self.http_session and not self.http_session.closed:
|
|
143
|
+
await self.http_session.close()
|
{kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/massive_data_processor.py
RENAMED
|
@@ -1,17 +1,14 @@
|
|
|
1
1
|
import asyncio
|
|
2
2
|
import json
|
|
3
3
|
import logging
|
|
4
|
-
from typing import Dict
|
|
5
4
|
|
|
6
5
|
import aio_pika
|
|
7
|
-
import redis
|
|
8
6
|
import redis.asyncio as aioredis
|
|
9
7
|
from aio_pika.abc import AbstractIncomingMessage
|
|
10
|
-
from aio_pika.exceptions import AMQPConnectionError
|
|
11
8
|
|
|
12
9
|
from kuhl_haus.mdp.analyzers.massive_data_analyzer import MassiveDataAnalyzer
|
|
13
|
-
from kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
|
|
14
10
|
from kuhl_haus.mdp.integ.web_socket_message_serde import WebSocketMessageSerde
|
|
11
|
+
from kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
|
|
15
12
|
|
|
16
13
|
|
|
17
14
|
class MassiveDataProcessor:
|
{kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_cache_keys.py
RENAMED
|
@@ -16,6 +16,9 @@ class MarketDataCacheKeys(Enum):
|
|
|
16
16
|
|
|
17
17
|
# MARKET DATA CACHE
|
|
18
18
|
DAILY_AGGREGATES = 'aggregate:daily'
|
|
19
|
+
TICKER_SNAPSHOTS = 'snapshots'
|
|
20
|
+
TICKER_AVG_VOLUME = 'avg_volume'
|
|
21
|
+
TICKER_FREE_FLOAT = 'free_float'
|
|
19
22
|
|
|
20
23
|
# MARKET DATA PROCESSOR CACHE
|
|
21
24
|
TOP_TRADES_SCANNER = f'cache:{MarketDataScannerNames.TOP_TRADES.value}'
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from typing import Dict, Optional
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
# docs
|
|
7
|
+
# https://massive.com/docs/stocks/ws_stocks_am
|
|
8
|
+
# https://massive.com/docs/websocket/stocks/trades
|
|
9
|
+
|
|
10
|
+
@dataclass()
|
|
11
|
+
class TopStocksCacheItem:
|
|
12
|
+
day_start_time: Optional[float] = 0.0
|
|
13
|
+
|
|
14
|
+
# Cached details for each ticker
|
|
15
|
+
symbol_data_cache: Optional[Dict[str, dict]] = field(default_factory=lambda: defaultdict(dict))
|
|
16
|
+
|
|
17
|
+
# Top Volume map
|
|
18
|
+
top_volume_map: Optional[Dict[str, float]] = field(default_factory=lambda: defaultdict(dict))
|
|
19
|
+
|
|
20
|
+
# Top Gappers map
|
|
21
|
+
top_gappers_map: Optional[Dict[str, float]] = field(default_factory=lambda: defaultdict(dict))
|
|
22
|
+
|
|
23
|
+
# Top Gainers map
|
|
24
|
+
top_gainers_map: Optional[Dict[str, float]] = field(default_factory=lambda: defaultdict(dict))
|
|
25
|
+
|
|
26
|
+
def to_dict(self):
|
|
27
|
+
ret = {
|
|
28
|
+
# Cache start time
|
|
29
|
+
"day_start_time": self.day_start_time,
|
|
30
|
+
|
|
31
|
+
# Maps
|
|
32
|
+
"symbol_data_cache": self.symbol_data_cache,
|
|
33
|
+
"top_volume_map": self.top_volume_map,
|
|
34
|
+
"top_gappers_map": self.top_gappers_map,
|
|
35
|
+
"top_gainers_map": self.top_gainers_map,
|
|
36
|
+
}
|
|
37
|
+
return ret
|
|
38
|
+
|
|
39
|
+
def top_volume(self, limit):
|
|
40
|
+
ret = []
|
|
41
|
+
for ticker, volume in sorted(self.top_volume_map.items(), key=lambda x: x[1], reverse=True)[
|
|
42
|
+
:limit
|
|
43
|
+
]:
|
|
44
|
+
try:
|
|
45
|
+
ret.append({
|
|
46
|
+
"symbol": ticker,
|
|
47
|
+
"volume": self.symbol_data_cache[ticker]["volume"],
|
|
48
|
+
"accumulated_volume": self.symbol_data_cache[ticker]["accumulated_volume"],
|
|
49
|
+
"relative_volume": self.symbol_data_cache[ticker]["relative_volume"],
|
|
50
|
+
"official_open_price": self.symbol_data_cache[ticker]["official_open_price"],
|
|
51
|
+
"vwap": self.symbol_data_cache[ticker]["vwap"],
|
|
52
|
+
"open": self.symbol_data_cache[ticker]["open"],
|
|
53
|
+
"close": self.symbol_data_cache[ticker]["close"],
|
|
54
|
+
"high": self.symbol_data_cache[ticker]["high"],
|
|
55
|
+
"low": self.symbol_data_cache[ticker]["low"],
|
|
56
|
+
"aggregate_vwap": self.symbol_data_cache[ticker]["aggregate_vwap"],
|
|
57
|
+
"average_size": self.symbol_data_cache[ticker]["average_size"],
|
|
58
|
+
"avg_volume": self.symbol_data_cache[ticker]["avg_volume"],
|
|
59
|
+
"prev_day_close": self.symbol_data_cache[ticker]["prev_day_close"],
|
|
60
|
+
"prev_day_volume": self.symbol_data_cache[ticker]["prev_day_volume"],
|
|
61
|
+
"prev_day_vwap": self.symbol_data_cache[ticker]["prev_day_vwap"],
|
|
62
|
+
"change": self.symbol_data_cache[ticker]["change"],
|
|
63
|
+
"pct_change": self.symbol_data_cache[ticker]["pct_change"],
|
|
64
|
+
"change_since_open": self.symbol_data_cache[ticker]["change_since_open"],
|
|
65
|
+
"pct_change_since_open": self.symbol_data_cache[ticker]["pct_change_since_open"],
|
|
66
|
+
"start_timestamp": self.symbol_data_cache[ticker]["start_timestamp"],
|
|
67
|
+
"end_timestamp": self.symbol_data_cache[ticker]["end_timestamp"],
|
|
68
|
+
})
|
|
69
|
+
except KeyError:
|
|
70
|
+
del self.top_volume_map[ticker]
|
|
71
|
+
return ret
|
|
72
|
+
|
|
73
|
+
def top_gappers(self, limit):
|
|
74
|
+
ret = []
|
|
75
|
+
for ticker, pct_change in sorted(self.top_gappers_map.items(), key=lambda x: x[1], reverse=True)[
|
|
76
|
+
:limit
|
|
77
|
+
]:
|
|
78
|
+
try:
|
|
79
|
+
if pct_change <= 0:
|
|
80
|
+
break
|
|
81
|
+
ret.append({
|
|
82
|
+
"symbol": ticker,
|
|
83
|
+
"volume": self.symbol_data_cache[ticker]["volume"],
|
|
84
|
+
"accumulated_volume": self.symbol_data_cache[ticker]["accumulated_volume"],
|
|
85
|
+
"relative_volume": self.symbol_data_cache[ticker]["relative_volume"],
|
|
86
|
+
"official_open_price": self.symbol_data_cache[ticker]["official_open_price"],
|
|
87
|
+
"vwap": self.symbol_data_cache[ticker]["vwap"],
|
|
88
|
+
"open": self.symbol_data_cache[ticker]["open"],
|
|
89
|
+
"close": self.symbol_data_cache[ticker]["close"],
|
|
90
|
+
"high": self.symbol_data_cache[ticker]["high"],
|
|
91
|
+
"low": self.symbol_data_cache[ticker]["low"],
|
|
92
|
+
"aggregate_vwap": self.symbol_data_cache[ticker]["aggregate_vwap"],
|
|
93
|
+
"average_size": self.symbol_data_cache[ticker]["average_size"],
|
|
94
|
+
"avg_volume": self.symbol_data_cache[ticker]["avg_volume"],
|
|
95
|
+
"prev_day_close": self.symbol_data_cache[ticker]["prev_day_close"],
|
|
96
|
+
"prev_day_volume": self.symbol_data_cache[ticker]["prev_day_volume"],
|
|
97
|
+
"prev_day_vwap": self.symbol_data_cache[ticker]["prev_day_vwap"],
|
|
98
|
+
"change": self.symbol_data_cache[ticker]["change"],
|
|
99
|
+
"pct_change": self.symbol_data_cache[ticker]["pct_change"],
|
|
100
|
+
"change_since_open": self.symbol_data_cache[ticker]["change_since_open"],
|
|
101
|
+
"pct_change_since_open": self.symbol_data_cache[ticker]["pct_change_since_open"],
|
|
102
|
+
"start_timestamp": self.symbol_data_cache[ticker]["start_timestamp"],
|
|
103
|
+
"end_timestamp": self.symbol_data_cache[ticker]["end_timestamp"],
|
|
104
|
+
})
|
|
105
|
+
except KeyError:
|
|
106
|
+
del self.top_gappers_map[ticker]
|
|
107
|
+
return ret
|
|
108
|
+
|
|
109
|
+
def top_gainers(self, limit):
|
|
110
|
+
ret = []
|
|
111
|
+
for ticker, pct_change in sorted(self.top_gainers_map.items(), key=lambda x: x[1], reverse=True)[
|
|
112
|
+
:limit
|
|
113
|
+
]:
|
|
114
|
+
try:
|
|
115
|
+
if pct_change <= 0:
|
|
116
|
+
break
|
|
117
|
+
ret.append({
|
|
118
|
+
"symbol": ticker,
|
|
119
|
+
"volume": self.symbol_data_cache[ticker]["volume"],
|
|
120
|
+
"accumulated_volume": self.symbol_data_cache[ticker]["accumulated_volume"],
|
|
121
|
+
"relative_volume": self.symbol_data_cache[ticker]["relative_volume"],
|
|
122
|
+
"official_open_price": self.symbol_data_cache[ticker]["official_open_price"],
|
|
123
|
+
"vwap": self.symbol_data_cache[ticker]["vwap"],
|
|
124
|
+
"open": self.symbol_data_cache[ticker]["open"],
|
|
125
|
+
"close": self.symbol_data_cache[ticker]["close"],
|
|
126
|
+
"high": self.symbol_data_cache[ticker]["high"],
|
|
127
|
+
"low": self.symbol_data_cache[ticker]["low"],
|
|
128
|
+
"aggregate_vwap": self.symbol_data_cache[ticker]["aggregate_vwap"],
|
|
129
|
+
"average_size": self.symbol_data_cache[ticker]["average_size"],
|
|
130
|
+
"avg_volume": self.symbol_data_cache[ticker]["avg_volume"],
|
|
131
|
+
"prev_day_close": self.symbol_data_cache[ticker]["prev_day_close"],
|
|
132
|
+
"prev_day_volume": self.symbol_data_cache[ticker]["prev_day_volume"],
|
|
133
|
+
"prev_day_vwap": self.symbol_data_cache[ticker]["prev_day_vwap"],
|
|
134
|
+
"change": self.symbol_data_cache[ticker]["change"],
|
|
135
|
+
"pct_change": self.symbol_data_cache[ticker]["pct_change"],
|
|
136
|
+
"change_since_open": self.symbol_data_cache[ticker]["change_since_open"],
|
|
137
|
+
"pct_change_since_open": self.symbol_data_cache[ticker]["pct_change_since_open"],
|
|
138
|
+
"start_timestamp": self.symbol_data_cache[ticker]["start_timestamp"],
|
|
139
|
+
"end_timestamp": self.symbol_data_cache[ticker]["end_timestamp"],
|
|
140
|
+
})
|
|
141
|
+
except KeyError:
|
|
142
|
+
del self.top_gainers_map[ticker]
|
|
143
|
+
return ret
|
{kuhl_haus_mdp-0.1.2/tests → kuhl_haus_mdp-0.1.5/tests/analyzers}/test_massive_data_analyzer.py
RENAMED
|
@@ -2,10 +2,10 @@ from unittest.mock import MagicMock
|
|
|
2
2
|
|
|
3
3
|
import pytest
|
|
4
4
|
from massive.websocket.models import EventType
|
|
5
|
+
|
|
6
|
+
from kuhl_haus.mdp.models.market_data_cache_ttl import MarketDataCacheTTL
|
|
5
7
|
from src.kuhl_haus.mdp.analyzers.massive_data_analyzer import MassiveDataAnalyzer
|
|
6
|
-
from src.kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
|
|
7
8
|
from src.kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
|
|
8
|
-
from kuhl_haus.mdp.models.market_data_cache_ttl import MarketDataCacheTTL
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@pytest.fixture
|