kuhl-haus-mdp 0.1.2__tar.gz → 0.1.5__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/PKG-INFO +8 -6
  2. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/pyproject.toml +8 -6
  3. kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/analyzers/top_stocks.py +220 -0
  4. kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/components/market_data_cache.py +143 -0
  5. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/massive_data_processor.py +1 -4
  6. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_cache_keys.py +3 -0
  7. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_cache_ttl.py +1 -0
  8. kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/models/top_stocks_cache_item.py +143 -0
  9. {kuhl_haus_mdp-0.1.2/tests → kuhl_haus_mdp-0.1.5/tests/analyzers}/test_massive_data_analyzer.py +2 -2
  10. kuhl_haus_mdp-0.1.5/tests/analyzers/test_top_stocks_rehydrate.py +108 -0
  11. kuhl_haus_mdp-0.1.5/tests/components/test_market_data_cache.py +708 -0
  12. kuhl_haus_mdp-0.1.5/tests/helpers/test_utils.py +93 -0
  13. kuhl_haus_mdp-0.1.5/tests/integ/__init__.py +0 -0
  14. kuhl_haus_mdp-0.1.5/tests/models/__init__.py +0 -0
  15. kuhl_haus_mdp-0.1.5/tests/models/test_top_stocks_cache_item.py +109 -0
  16. kuhl_haus_mdp-0.1.2/src/kuhl_haus/mdp/analyzers/top_stocks.py +0 -408
  17. kuhl_haus_mdp-0.1.2/src/kuhl_haus/mdp/components/market_data_cache.py +0 -29
  18. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/LICENSE.txt +0 -0
  19. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/README.md +0 -0
  20. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/__init__.py +0 -0
  21. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/analyzers/__init__.py +0 -0
  22. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/analyzers/analyzer.py +0 -0
  23. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/analyzers/massive_data_analyzer.py +0 -0
  24. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/components/__init__.py +0 -0
  25. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/components/market_data_scanner.py +0 -0
  26. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/components/widget_data_service.py +0 -0
  27. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/helpers/__init__.py +0 -0
  28. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/helpers/process_manager.py +0 -0
  29. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/helpers/queue_name_resolver.py +0 -0
  30. {kuhl_haus_mdp-0.1.2/src/kuhl_haus/mdp/integ → kuhl_haus_mdp-0.1.5/src/kuhl_haus/mdp/helpers}/utils.py +0 -0
  31. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/__init__.py +0 -0
  32. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/massive_data_listener.py +0 -0
  33. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/massive_data_queues.py +0 -0
  34. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/integ/web_socket_message_serde.py +0 -0
  35. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/__init__.py +0 -0
  36. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_analyzer_result.py +0 -0
  37. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_pubsub_keys.py +0 -0
  38. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/market_data_scanner_names.py +0 -0
  39. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/src/kuhl_haus/mdp/models/massive_data_queue.py +0 -0
  40. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/__init__.py +0 -0
  41. {kuhl_haus_mdp-0.1.2/tests/components → kuhl_haus_mdp-0.1.5/tests/analyzers}/__init__.py +0 -0
  42. {kuhl_haus_mdp-0.1.2/tests/helpers → kuhl_haus_mdp-0.1.5/tests/components}/__init__.py +0 -0
  43. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/components/test_market_data_scanner.py +0 -0
  44. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/components/test_widget_data_service.py +0 -0
  45. {kuhl_haus_mdp-0.1.2/tests/integ → kuhl_haus_mdp-0.1.5/tests/helpers}/__init__.py +0 -0
  46. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/helpers/test_process_manager.py +0 -0
  47. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/helpers/test_queue_name_resolver.py +0 -0
  48. {kuhl_haus_mdp-0.1.2 → kuhl_haus_mdp-0.1.5}/tests/integ/test_web_socket_message_serde.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: kuhl-haus-mdp
3
- Version: 0.1.2
3
+ Version: 0.1.5
4
4
  Summary: Market data processing pipeline for stock market scanner
5
5
  Author-Email: Tom Pounders <git@oldschool.engineer>
6
6
  License: The MIT License (MIT)
@@ -33,20 +33,22 @@ Project-URL: Source, https://github.com/kuhl-haus/kuhl-haus-mdp.git
33
33
  Project-URL: Changelog, https://github.com/kuhl-haus/kuhl-haus-mdp/commits
34
34
  Project-URL: Tracker, https://github.com/kuhl-haus/kuhl-haus-mdp/issues
35
35
  Requires-Python: <3.13,>=3.9.21
36
- Requires-Dist: websockets
36
+ Requires-Dist: aiohttp
37
37
  Requires-Dist: aio-pika
38
- Requires-Dist: redis[asyncio]
39
- Requires-Dist: tenacity
40
38
  Requires-Dist: fastapi
41
- Requires-Dist: uvicorn[standard]
39
+ Requires-Dist: massive
42
40
  Requires-Dist: pydantic-settings
43
41
  Requires-Dist: python-dotenv
44
- Requires-Dist: massive
42
+ Requires-Dist: redis[asyncio]
43
+ Requires-Dist: tenacity
44
+ Requires-Dist: uvicorn[standard]
45
+ Requires-Dist: websockets
45
46
  Provides-Extra: testing
46
47
  Requires-Dist: setuptools; extra == "testing"
47
48
  Requires-Dist: pdm-backend; extra == "testing"
48
49
  Requires-Dist: pytest; extra == "testing"
49
50
  Requires-Dist: pytest-cov; extra == "testing"
51
+ Requires-Dist: pytest-asyncio; extra == "testing"
50
52
  Description-Content-Type: text/markdown
51
53
 
52
54
  <!-- These are examples of badges you might want to add to your README:
@@ -19,17 +19,18 @@ classifiers = [
19
19
  "Programming Language :: Python",
20
20
  ]
21
21
  dependencies = [
22
- "websockets",
22
+ "aiohttp",
23
23
  "aio-pika",
24
- "redis[asyncio]",
25
- "tenacity",
26
24
  "fastapi",
27
- "uvicorn[standard]",
25
+ "massive",
28
26
  "pydantic-settings",
29
27
  "python-dotenv",
30
- "massive",
28
+ "redis[asyncio]",
29
+ "tenacity",
30
+ "uvicorn[standard]",
31
+ "websockets",
31
32
  ]
32
- version = "0.1.2"
33
+ version = "0.1.5"
33
34
 
34
35
  [project.license]
35
36
  file = "LICENSE.txt"
@@ -47,6 +48,7 @@ testing = [
47
48
  "pdm-backend",
48
49
  "pytest",
49
50
  "pytest-cov",
51
+ "pytest-asyncio",
50
52
  ]
51
53
 
52
54
  [tool.setuptools_scm]
@@ -0,0 +1,220 @@
1
+ import logging
2
+ import time
3
+ from datetime import datetime, timezone, timedelta
4
+ from typing import Optional, List, Iterator
5
+ from zoneinfo import ZoneInfo
6
+
7
+ from massive.exceptions import BadResponse
8
+ from massive.rest import RESTClient
9
+ from massive.rest.models import (
10
+ TickerSnapshot,
11
+ Agg,
12
+ )
13
+ from massive.websocket.models import (
14
+ EquityAgg,
15
+ EventType
16
+ )
17
+
18
+ from kuhl_haus.mdp.analyzers.analyzer import Analyzer
19
+ from kuhl_haus.mdp.components.market_data_cache import MarketDataCache
20
+ from kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
21
+ from kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
22
+ from kuhl_haus.mdp.models.market_data_pubsub_keys import MarketDataPubSubKeys
23
+ from kuhl_haus.mdp.models.top_stocks_cache_item import TopStocksCacheItem
24
+
25
+
26
+ class TopStocksAnalyzer(Analyzer):
27
+
28
+ def __init__(self, cache: MarketDataCache, **kwargs):
29
+ if "cache_key" not in kwargs:
30
+ kwargs["cache_key"] = MarketDataCacheKeys.TOP_STOCKS_SCANNER.value
31
+ super().__init__(**kwargs)
32
+ self.cache = cache
33
+ self.logger = logging.getLogger(__name__)
34
+ self.cache_item = TopStocksCacheItem()
35
+ self.last_update_time = 0
36
+ self.pre_market_reset = False
37
+
38
+ async def rehydrate(self, data: dict):
39
+ if not data:
40
+ self.cache_item = TopStocksCacheItem()
41
+ self.logger.info("No data to rehydrate TopStocksCacheItem.")
42
+ return
43
+
44
+ # Get current time in UTC, then convert to Eastern Time
45
+ utc_now = datetime.now(timezone.utc)
46
+ et_now = utc_now.astimezone(ZoneInfo("America/New_York"))
47
+
48
+ # Check if within trading hours: Mon-Fri, 04:00-19:59 ET
49
+ is_weekday = et_now.weekday() < 5
50
+ is_trading_hours = 4 <= et_now.hour < 20
51
+ if not is_weekday or not is_trading_hours:
52
+ self.cache_item = TopStocksCacheItem()
53
+ self.logger.info(f"Outside market hours ({et_now.strftime('%H:%M:%S %Z')}), clearing cache.")
54
+ return
55
+ self.cache_item = TopStocksCacheItem(**data)
56
+ self.logger.info("Rehydrated TopStocksCacheItem")
57
+
58
+ async def analyze_data(self, data: dict) -> Optional[List[MarketDataAnalyzerResult]]:
59
+ utc_now = datetime.now(timezone.utc)
60
+ et_now = utc_now.astimezone(ZoneInfo("America/New_York"))
61
+ current_day = et_now.replace(hour=4, minute=0, second=0, microsecond=0).timestamp()
62
+ if current_day != self.cache_item.day_start_time:
63
+ self.logger.info(f"New day: {current_day} - resetting cache.")
64
+ self.cache_item = TopStocksCacheItem()
65
+ self.cache_item.day_start_time = current_day
66
+ elif et_now.hour == 9 and et_now.minute == 30 and not self.pre_market_reset:
67
+ self.logger.info("Market is now open; resetting symbol data cache.")
68
+ self.cache_item.symbol_data_cache = {}
69
+ self.pre_market_reset = True
70
+
71
+ event_type = data.get("event_type")
72
+ symbol = data.get("symbol")
73
+ if not event_type:
74
+ self.logger.info(f"Discarding data: {data}")
75
+ return None
76
+ elif not symbol:
77
+ self.logger.info(f"Discarding data: {data}")
78
+ return None
79
+ elif event_type == EventType.EquityAgg.value:
80
+ self.logger.debug(f"Processing EquityAgg: {data.get('symbol')}")
81
+ await self.handle_equity_agg(EquityAgg(**data))
82
+ elif event_type == EventType.EquityAggMin.value:
83
+ self.logger.debug(f"Processing EquityAggMin: {data.get('symbol')}")
84
+ await self.handle_equity_agg(EquityAgg(**data))
85
+ else:
86
+ self.logger.info(f"Discarding data: {data}")
87
+ return None
88
+ current_time = int(time.time())
89
+ # return results once per second
90
+ if current_time <= self.last_update_time:
91
+ return None
92
+ self.last_update_time = current_time
93
+
94
+ result = [
95
+ MarketDataAnalyzerResult(
96
+ data=self.cache_item.to_dict(),
97
+ cache_key=self.cache_key,
98
+ cache_ttl=28500, # 7 hours, 55 minutes
99
+ ),
100
+ MarketDataAnalyzerResult(
101
+ data=self.cache_item.top_volume(100),
102
+ cache_key=MarketDataPubSubKeys.TOP_VOLUME_SCANNER.value,
103
+ cache_ttl=259200, # 3 days
104
+ publish_key=MarketDataPubSubKeys.TOP_VOLUME_SCANNER.value,
105
+ ),
106
+ MarketDataAnalyzerResult(
107
+ data=self.cache_item.top_gainers(500),
108
+ cache_key=MarketDataPubSubKeys.TOP_GAINERS_SCANNER.value,
109
+ cache_ttl=259200, # 3 days
110
+ publish_key=MarketDataPubSubKeys.TOP_GAINERS_SCANNER.value,
111
+ ),
112
+ MarketDataAnalyzerResult(
113
+ data=self.cache_item.top_gappers(500),
114
+ cache_key=MarketDataPubSubKeys.TOP_GAPPERS_SCANNER.value,
115
+ cache_ttl=259200, # 3 days
116
+ publish_key=MarketDataPubSubKeys.TOP_GAPPERS_SCANNER.value,
117
+ )
118
+ ]
119
+
120
+ return result
121
+
122
+ async def handle_equity_agg(self, event: EquityAgg):
123
+ # Get data from symbol data cache or Rest API
124
+ if event.symbol in self.cache_item.symbol_data_cache:
125
+ cached_data = self.cache_item.symbol_data_cache[event.symbol]
126
+ avg_volume = cached_data["avg_volume"]
127
+ prev_day_close = cached_data["prev_day_close"]
128
+ prev_day_volume = cached_data["prev_day_volume"]
129
+ prev_day_vwap = cached_data["prev_day_vwap"]
130
+ else:
131
+ # Get snapshot for previous day's data
132
+ retry_count = 0
133
+ max_tries = 3
134
+ prev_day_close = 0
135
+ prev_day_volume = 0
136
+ prev_day_vwap = 0
137
+ while retry_count < max_tries:
138
+ try:
139
+ snapshot = await self.cache.get_ticker_snapshot(event.symbol)
140
+ prev_day_close = snapshot.prev_day.close
141
+ prev_day_volume = snapshot.prev_day.volume
142
+ prev_day_vwap = snapshot.prev_day.vwap
143
+ break
144
+ except BadResponse as e:
145
+ self.logger.error(f"Error getting snapshot for {event.symbol}: {repr(e)}", exc_info=e, stack_info=True)
146
+ retry_count += 1
147
+ if retry_count == max_tries and prev_day_close == 0:
148
+ self.logger.error(f"Failed to get snapshot for {event.symbol} after {max_tries} tries.")
149
+ return
150
+
151
+ # Get average volume
152
+ retry_count = 0
153
+ max_tries = 3
154
+ avg_volume = 0
155
+ while retry_count < max_tries:
156
+ try:
157
+ avg_volume = await self.cache.get_avg_volume(event.symbol)
158
+ break
159
+ except (BadResponse, ZeroDivisionError) as e:
160
+ self.logger.error(f"Error getting average volume for {event.symbol}: {repr(e)}", exc_info=e, stack_info=True)
161
+ retry_count += 1
162
+ if retry_count == max_tries and avg_volume == 0:
163
+ self.logger.error(f"Failed to get average volume for {event.symbol} after {max_tries} tries.")
164
+ return
165
+
166
+ # Calculate relative volume
167
+ if avg_volume == 0:
168
+ relative_volume = 0
169
+ else:
170
+ relative_volume = event.accumulated_volume / avg_volume
171
+
172
+ # Calculate percentage change since previous close
173
+ if prev_day_close == 0:
174
+ change = 0
175
+ pct_change = 0
176
+ else:
177
+ change = event.close - prev_day_close
178
+ pct_change = change / prev_day_close * 100
179
+
180
+ # Calculate percentage change since opening bell
181
+ change_since_open = 0
182
+ pct_change_since_open = 0
183
+ if event.official_open_price:
184
+ change_since_open = event.close - event.official_open_price
185
+ pct_change_since_open = change_since_open / event.official_open_price * 100
186
+
187
+ # Sort top tickers by accumulated volume
188
+ self.cache_item.top_volume_map[event.symbol] = event.accumulated_volume
189
+
190
+ # Sort top gappers by percentage gain since the previous day's close
191
+ self.cache_item.top_gappers_map[event.symbol] = pct_change
192
+
193
+ # Sort top gainers by percentage gain since the opening bell
194
+ self.cache_item.top_gainers_map[event.symbol] = pct_change_since_open
195
+
196
+ # Update symbol data cache
197
+ self.cache_item.symbol_data_cache[event.symbol] = {
198
+ "symbol": event.symbol,
199
+ "volume": event.volume,
200
+ "accumulated_volume": event.accumulated_volume,
201
+ "relative_volume": relative_volume,
202
+ "official_open_price": event.official_open_price,
203
+ "vwap": event.vwap,
204
+ "open": event.open,
205
+ "close": event.close,
206
+ "high": event.high,
207
+ "low": event.low,
208
+ "aggregate_vwap": event.aggregate_vwap,
209
+ "average_size": event.average_size,
210
+ "avg_volume": avg_volume,
211
+ "prev_day_close": prev_day_close,
212
+ "prev_day_volume": prev_day_volume,
213
+ "prev_day_vwap": prev_day_vwap,
214
+ "change": change,
215
+ "pct_change": pct_change,
216
+ "change_since_open": change_since_open,
217
+ "pct_change_since_open": pct_change_since_open,
218
+ "start_timestamp": event.start_timestamp,
219
+ "end_timestamp": event.end_timestamp,
220
+ }
@@ -0,0 +1,143 @@
1
+ import json
2
+ import logging
3
+ from typing import Any, Optional, Iterator, List
4
+
5
+ import aiohttp
6
+ import redis.asyncio as aioredis
7
+ from massive.rest import RESTClient
8
+ from massive.rest.models import (
9
+ TickerSnapshot,
10
+ FinancialRatio,
11
+ )
12
+
13
+ from kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
14
+ from kuhl_haus.mdp.models.market_data_cache_ttl import MarketDataCacheTTL
15
+
16
+
17
+ class MarketDataCache:
18
+ def __init__(self, rest_client: RESTClient, redis_client: aioredis.Redis, massive_api_key: str):
19
+ self.logger = logging.getLogger(__name__)
20
+ self.rest_client = rest_client
21
+ self.massive_api_key = massive_api_key
22
+ self.redis_client = redis_client
23
+ self.http_session = None
24
+
25
+ async def get_cache(self, cache_key: str) -> Optional[dict]:
26
+ """Fetch current value from Redis cache (for snapshot requests)."""
27
+ value = await self.redis_client.get(cache_key)
28
+ if value:
29
+ return json.loads(value)
30
+ return None
31
+
32
+ async def cache_data(self, data: Any, cache_key: str, cache_ttl: int = 0):
33
+ if cache_ttl > 0:
34
+ await self.redis_client.setex(cache_key, cache_ttl, json.dumps(data))
35
+ else:
36
+ await self.redis_client.set(cache_key, json.dumps(data))
37
+ self.logger.debug(f"Cached data for {cache_key}")
38
+
39
+ async def publish_data(self, data: Any, publish_key: str = None):
40
+ await self.redis_client.publish(publish_key, json.dumps(data))
41
+ self.logger.debug(f"Published data for {publish_key}")
42
+
43
+ async def get_ticker_snapshot(self, ticker: str) -> TickerSnapshot:
44
+ self.logger.debug(f"Getting snapshot for {ticker}")
45
+ cache_key = f"{MarketDataCacheKeys.TICKER_SNAPSHOTS.value}:{ticker}"
46
+ result = await self.get_cache(cache_key=cache_key)
47
+ if result:
48
+ snapshot = TickerSnapshot.from_dict(**result)
49
+ else:
50
+ snapshot: TickerSnapshot = self.rest_client.get_snapshot_ticker(
51
+ market_type="stocks",
52
+ ticker=ticker
53
+ )
54
+ self.logger.debug(f"Snapshot result: {snapshot}")
55
+ await self.cache_data(
56
+ data=snapshot,
57
+ cache_key=cache_key,
58
+ cache_ttl=MarketDataCacheTTL.EIGHT_HOURS.value
59
+ )
60
+ return snapshot
61
+
62
+ async def get_avg_volume(self, ticker: str):
63
+ self.logger.debug(f"Getting average volume for {ticker}")
64
+ cache_key = f"{MarketDataCacheKeys.TICKER_AVG_VOLUME.value}:{ticker}"
65
+ avg_volume = await self.get_cache(cache_key=cache_key)
66
+ if avg_volume:
67
+ self.logger.debug(f"Returning cached value for {ticker}: {avg_volume}")
68
+ return avg_volume
69
+
70
+ results: Iterator[FinancialRatio] = self.rest_client.list_financials_ratios(ticker=ticker)
71
+ ratios: List[FinancialRatio] = []
72
+ for financial_ratio in results:
73
+ ratios.append(financial_ratio)
74
+ if len(ratios) == 1:
75
+ avg_volume = ratios[0].average_volume
76
+ else:
77
+ raise Exception(f"Unexpected number of financial ratios for {ticker}: {len(ratios)}")
78
+
79
+ self.logger.debug(f"average volume {ticker}: {avg_volume}")
80
+ await self.cache_data(
81
+ data=avg_volume,
82
+ cache_key=cache_key,
83
+ cache_ttl=MarketDataCacheTTL.TWELVE_HOURS.value
84
+ )
85
+ return avg_volume
86
+
87
+ async def get_free_float(self, ticker: str):
88
+ self.logger.debug(f"Getting free float for {ticker}")
89
+ cache_key = f"{MarketDataCacheKeys.TICKER_FREE_FLOAT.value}:{ticker}"
90
+ free_float = await self.get_cache(cache_key=cache_key)
91
+ if free_float:
92
+ self.logger.debug(f"Returning cached value for {ticker}: {free_float}")
93
+ return free_float
94
+
95
+ # NOTE: This endpoint is experimental and the interface may change.
96
+ # https://massive.com/docs/rest/stocks/fundamentals/float
97
+ url = f"https://api.massive.com/stocks/vX/float"
98
+ params = {
99
+ "ticker": ticker,
100
+ "apiKey": self.massive_api_key
101
+ }
102
+
103
+ session = await self.get_http_session()
104
+ try:
105
+ async with session.get(url, params=params, timeout=aiohttp.ClientTimeout(total=10)) as response:
106
+ response.raise_for_status()
107
+ data = await response.json()
108
+
109
+ # Extract free_float from response
110
+ if data.get("status") == "OK" and data.get("results") is not None:
111
+ results = data["results"]
112
+ if len(results) > 0:
113
+ free_float = results[0].get("free_float")
114
+ else:
115
+ raise Exception(f"No free float data returned for {ticker}")
116
+ else:
117
+ raise Exception(f"Invalid response from Massive API for {ticker}: {data}")
118
+
119
+ except aiohttp.ClientError as e:
120
+ self.logger.error(f"HTTP error fetching free float for {ticker}: {e}")
121
+ raise
122
+ except Exception as e:
123
+ self.logger.error(f"Error fetching free float for {ticker}: {e}")
124
+ raise
125
+
126
+ self.logger.debug(f"free float {ticker}: {free_float}")
127
+ await self.cache_data(
128
+ data=free_float,
129
+ cache_key=cache_key,
130
+ cache_ttl=MarketDataCacheTTL.TWELVE_HOURS.value
131
+ )
132
+ return free_float
133
+
134
+ async def get_http_session(self) -> aiohttp.ClientSession:
135
+ """Get or create aiohttp session for async HTTP requests."""
136
+ if self.http_session is None or self.http_session.closed:
137
+ self.http_session = aiohttp.ClientSession()
138
+ return self.http_session
139
+
140
+ async def close(self):
141
+ """Close aiohttp session."""
142
+ if self.http_session and not self.http_session.closed:
143
+ await self.http_session.close()
@@ -1,17 +1,14 @@
1
1
  import asyncio
2
2
  import json
3
3
  import logging
4
- from typing import Dict
5
4
 
6
5
  import aio_pika
7
- import redis
8
6
  import redis.asyncio as aioredis
9
7
  from aio_pika.abc import AbstractIncomingMessage
10
- from aio_pika.exceptions import AMQPConnectionError
11
8
 
12
9
  from kuhl_haus.mdp.analyzers.massive_data_analyzer import MassiveDataAnalyzer
13
- from kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
14
10
  from kuhl_haus.mdp.integ.web_socket_message_serde import WebSocketMessageSerde
11
+ from kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
15
12
 
16
13
 
17
14
  class MassiveDataProcessor:
@@ -16,6 +16,9 @@ class MarketDataCacheKeys(Enum):
16
16
 
17
17
  # MARKET DATA CACHE
18
18
  DAILY_AGGREGATES = 'aggregate:daily'
19
+ TICKER_SNAPSHOTS = 'snapshots'
20
+ TICKER_AVG_VOLUME = 'avg_volume'
21
+ TICKER_FREE_FLOAT = 'free_float'
19
22
 
20
23
  # MARKET DATA PROCESSOR CACHE
21
24
  TOP_TRADES_SCANNER = f'cache:{MarketDataScannerNames.TOP_TRADES.value}'
@@ -8,6 +8,7 @@ class MarketDataCacheTTL(Enum):
8
8
  FOUR_HOURS = 14400
9
9
  SIX_HOURS = 21600
10
10
  EIGHT_HOURS = 28800
11
+ TWELVE_HOURS = 43200
11
12
 
12
13
  # Days
13
14
  ONE_DAY = 86400
@@ -0,0 +1,143 @@
1
+ from collections import defaultdict
2
+ from dataclasses import dataclass, field
3
+ from typing import Dict, Optional
4
+
5
+
6
+ # docs
7
+ # https://massive.com/docs/stocks/ws_stocks_am
8
+ # https://massive.com/docs/websocket/stocks/trades
9
+
10
+ @dataclass()
11
+ class TopStocksCacheItem:
12
+ day_start_time: Optional[float] = 0.0
13
+
14
+ # Cached details for each ticker
15
+ symbol_data_cache: Optional[Dict[str, dict]] = field(default_factory=lambda: defaultdict(dict))
16
+
17
+ # Top Volume map
18
+ top_volume_map: Optional[Dict[str, float]] = field(default_factory=lambda: defaultdict(dict))
19
+
20
+ # Top Gappers map
21
+ top_gappers_map: Optional[Dict[str, float]] = field(default_factory=lambda: defaultdict(dict))
22
+
23
+ # Top Gainers map
24
+ top_gainers_map: Optional[Dict[str, float]] = field(default_factory=lambda: defaultdict(dict))
25
+
26
+ def to_dict(self):
27
+ ret = {
28
+ # Cache start time
29
+ "day_start_time": self.day_start_time,
30
+
31
+ # Maps
32
+ "symbol_data_cache": self.symbol_data_cache,
33
+ "top_volume_map": self.top_volume_map,
34
+ "top_gappers_map": self.top_gappers_map,
35
+ "top_gainers_map": self.top_gainers_map,
36
+ }
37
+ return ret
38
+
39
+ def top_volume(self, limit):
40
+ ret = []
41
+ for ticker, volume in sorted(self.top_volume_map.items(), key=lambda x: x[1], reverse=True)[
42
+ :limit
43
+ ]:
44
+ try:
45
+ ret.append({
46
+ "symbol": ticker,
47
+ "volume": self.symbol_data_cache[ticker]["volume"],
48
+ "accumulated_volume": self.symbol_data_cache[ticker]["accumulated_volume"],
49
+ "relative_volume": self.symbol_data_cache[ticker]["relative_volume"],
50
+ "official_open_price": self.symbol_data_cache[ticker]["official_open_price"],
51
+ "vwap": self.symbol_data_cache[ticker]["vwap"],
52
+ "open": self.symbol_data_cache[ticker]["open"],
53
+ "close": self.symbol_data_cache[ticker]["close"],
54
+ "high": self.symbol_data_cache[ticker]["high"],
55
+ "low": self.symbol_data_cache[ticker]["low"],
56
+ "aggregate_vwap": self.symbol_data_cache[ticker]["aggregate_vwap"],
57
+ "average_size": self.symbol_data_cache[ticker]["average_size"],
58
+ "avg_volume": self.symbol_data_cache[ticker]["avg_volume"],
59
+ "prev_day_close": self.symbol_data_cache[ticker]["prev_day_close"],
60
+ "prev_day_volume": self.symbol_data_cache[ticker]["prev_day_volume"],
61
+ "prev_day_vwap": self.symbol_data_cache[ticker]["prev_day_vwap"],
62
+ "change": self.symbol_data_cache[ticker]["change"],
63
+ "pct_change": self.symbol_data_cache[ticker]["pct_change"],
64
+ "change_since_open": self.symbol_data_cache[ticker]["change_since_open"],
65
+ "pct_change_since_open": self.symbol_data_cache[ticker]["pct_change_since_open"],
66
+ "start_timestamp": self.symbol_data_cache[ticker]["start_timestamp"],
67
+ "end_timestamp": self.symbol_data_cache[ticker]["end_timestamp"],
68
+ })
69
+ except KeyError:
70
+ del self.top_volume_map[ticker]
71
+ return ret
72
+
73
+ def top_gappers(self, limit):
74
+ ret = []
75
+ for ticker, pct_change in sorted(self.top_gappers_map.items(), key=lambda x: x[1], reverse=True)[
76
+ :limit
77
+ ]:
78
+ try:
79
+ if pct_change <= 0:
80
+ break
81
+ ret.append({
82
+ "symbol": ticker,
83
+ "volume": self.symbol_data_cache[ticker]["volume"],
84
+ "accumulated_volume": self.symbol_data_cache[ticker]["accumulated_volume"],
85
+ "relative_volume": self.symbol_data_cache[ticker]["relative_volume"],
86
+ "official_open_price": self.symbol_data_cache[ticker]["official_open_price"],
87
+ "vwap": self.symbol_data_cache[ticker]["vwap"],
88
+ "open": self.symbol_data_cache[ticker]["open"],
89
+ "close": self.symbol_data_cache[ticker]["close"],
90
+ "high": self.symbol_data_cache[ticker]["high"],
91
+ "low": self.symbol_data_cache[ticker]["low"],
92
+ "aggregate_vwap": self.symbol_data_cache[ticker]["aggregate_vwap"],
93
+ "average_size": self.symbol_data_cache[ticker]["average_size"],
94
+ "avg_volume": self.symbol_data_cache[ticker]["avg_volume"],
95
+ "prev_day_close": self.symbol_data_cache[ticker]["prev_day_close"],
96
+ "prev_day_volume": self.symbol_data_cache[ticker]["prev_day_volume"],
97
+ "prev_day_vwap": self.symbol_data_cache[ticker]["prev_day_vwap"],
98
+ "change": self.symbol_data_cache[ticker]["change"],
99
+ "pct_change": self.symbol_data_cache[ticker]["pct_change"],
100
+ "change_since_open": self.symbol_data_cache[ticker]["change_since_open"],
101
+ "pct_change_since_open": self.symbol_data_cache[ticker]["pct_change_since_open"],
102
+ "start_timestamp": self.symbol_data_cache[ticker]["start_timestamp"],
103
+ "end_timestamp": self.symbol_data_cache[ticker]["end_timestamp"],
104
+ })
105
+ except KeyError:
106
+ del self.top_gappers_map[ticker]
107
+ return ret
108
+
109
+ def top_gainers(self, limit):
110
+ ret = []
111
+ for ticker, pct_change in sorted(self.top_gainers_map.items(), key=lambda x: x[1], reverse=True)[
112
+ :limit
113
+ ]:
114
+ try:
115
+ if pct_change <= 0:
116
+ break
117
+ ret.append({
118
+ "symbol": ticker,
119
+ "volume": self.symbol_data_cache[ticker]["volume"],
120
+ "accumulated_volume": self.symbol_data_cache[ticker]["accumulated_volume"],
121
+ "relative_volume": self.symbol_data_cache[ticker]["relative_volume"],
122
+ "official_open_price": self.symbol_data_cache[ticker]["official_open_price"],
123
+ "vwap": self.symbol_data_cache[ticker]["vwap"],
124
+ "open": self.symbol_data_cache[ticker]["open"],
125
+ "close": self.symbol_data_cache[ticker]["close"],
126
+ "high": self.symbol_data_cache[ticker]["high"],
127
+ "low": self.symbol_data_cache[ticker]["low"],
128
+ "aggregate_vwap": self.symbol_data_cache[ticker]["aggregate_vwap"],
129
+ "average_size": self.symbol_data_cache[ticker]["average_size"],
130
+ "avg_volume": self.symbol_data_cache[ticker]["avg_volume"],
131
+ "prev_day_close": self.symbol_data_cache[ticker]["prev_day_close"],
132
+ "prev_day_volume": self.symbol_data_cache[ticker]["prev_day_volume"],
133
+ "prev_day_vwap": self.symbol_data_cache[ticker]["prev_day_vwap"],
134
+ "change": self.symbol_data_cache[ticker]["change"],
135
+ "pct_change": self.symbol_data_cache[ticker]["pct_change"],
136
+ "change_since_open": self.symbol_data_cache[ticker]["change_since_open"],
137
+ "pct_change_since_open": self.symbol_data_cache[ticker]["pct_change_since_open"],
138
+ "start_timestamp": self.symbol_data_cache[ticker]["start_timestamp"],
139
+ "end_timestamp": self.symbol_data_cache[ticker]["end_timestamp"],
140
+ })
141
+ except KeyError:
142
+ del self.top_gainers_map[ticker]
143
+ return ret
@@ -2,10 +2,10 @@ from unittest.mock import MagicMock
2
2
 
3
3
  import pytest
4
4
  from massive.websocket.models import EventType
5
+
6
+ from kuhl_haus.mdp.models.market_data_cache_ttl import MarketDataCacheTTL
5
7
  from src.kuhl_haus.mdp.analyzers.massive_data_analyzer import MassiveDataAnalyzer
6
- from src.kuhl_haus.mdp.models.market_data_analyzer_result import MarketDataAnalyzerResult
7
8
  from src.kuhl_haus.mdp.models.market_data_cache_keys import MarketDataCacheKeys
8
- from kuhl_haus.mdp.models.market_data_cache_ttl import MarketDataCacheTTL
9
9
 
10
10
 
11
11
  @pytest.fixture