equity-aggregator 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- equity_aggregator/README.md +40 -36
- equity_aggregator/adapters/__init__.py +13 -7
- equity_aggregator/adapters/data_sources/__init__.py +4 -6
- equity_aggregator/adapters/data_sources/_utils/_client.py +1 -1
- equity_aggregator/adapters/data_sources/{authoritative_feeds → _utils}/_record_types.py +1 -1
- equity_aggregator/adapters/data_sources/discovery_feeds/__init__.py +17 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/__init__.py +10 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/backoff.py +33 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/parser.py +107 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/intrinio.py +305 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/session.py +197 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/__init__.py +9 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/backoff.py +33 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/parser.py +120 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/lseg.py +239 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/session.py +162 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/sec/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/sec}/sec.py +4 -5
- equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/stock_analysis.py +150 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/__init__.py +5 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/tradingview.py +275 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/xetra/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/xetra}/xetra.py +9 -12
- equity_aggregator/adapters/data_sources/enrichment_feeds/__init__.py +6 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/__init__.py +5 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/api.py +71 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/download.py +109 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/gleif.py +195 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/parser.py +75 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/__init__.py +1 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/_utils/__init__.py +11 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/{utils → _utils}/backoff.py +1 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/{utils → _utils}/fuzzy.py +28 -26
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/_utils/json.py +36 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/__init__.py +1 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/{summary.py → quote_summary.py} +44 -30
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/search.py +10 -5
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/auth.py +130 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/config.py +3 -3
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/ranking.py +97 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/session.py +85 -218
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/transport.py +191 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/yfinance.py +413 -0
- equity_aggregator/adapters/data_sources/reference_lookup/exchange_rate_api.py +6 -13
- equity_aggregator/adapters/data_sources/reference_lookup/openfigi.py +23 -7
- equity_aggregator/cli/dispatcher.py +11 -8
- equity_aggregator/cli/main.py +14 -5
- equity_aggregator/cli/parser.py +1 -1
- equity_aggregator/cli/signals.py +32 -0
- equity_aggregator/domain/_utils/__init__.py +2 -2
- equity_aggregator/domain/_utils/_load_converter.py +30 -21
- equity_aggregator/domain/_utils/_merge.py +221 -368
- equity_aggregator/domain/_utils/_merge_config.py +205 -0
- equity_aggregator/domain/_utils/_strategies.py +180 -0
- equity_aggregator/domain/pipeline/resolve.py +17 -11
- equity_aggregator/domain/pipeline/runner.py +4 -4
- equity_aggregator/domain/pipeline/seed.py +5 -1
- equity_aggregator/domain/pipeline/transforms/__init__.py +2 -2
- equity_aggregator/domain/pipeline/transforms/canonicalise.py +1 -1
- equity_aggregator/domain/pipeline/transforms/enrich.py +328 -285
- equity_aggregator/domain/pipeline/transforms/group.py +48 -0
- equity_aggregator/logging_config.py +4 -1
- equity_aggregator/schemas/__init__.py +11 -5
- equity_aggregator/schemas/canonical.py +11 -6
- equity_aggregator/schemas/feeds/__init__.py +11 -5
- equity_aggregator/schemas/feeds/gleif_feed_data.py +35 -0
- equity_aggregator/schemas/feeds/intrinio_feed_data.py +142 -0
- equity_aggregator/schemas/feeds/{lse_feed_data.py → lseg_feed_data.py} +85 -52
- equity_aggregator/schemas/feeds/sec_feed_data.py +36 -6
- equity_aggregator/schemas/feeds/stock_analysis_feed_data.py +107 -0
- equity_aggregator/schemas/feeds/tradingview_feed_data.py +144 -0
- equity_aggregator/schemas/feeds/xetra_feed_data.py +1 -1
- equity_aggregator/schemas/feeds/yfinance_feed_data.py +47 -35
- equity_aggregator/schemas/raw.py +5 -3
- equity_aggregator/schemas/types.py +7 -0
- equity_aggregator/schemas/validators.py +81 -27
- equity_aggregator/storage/data_store.py +5 -3
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/METADATA +205 -115
- equity_aggregator-0.1.4.dist-info/RECORD +103 -0
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/WHEEL +1 -1
- equity_aggregator/adapters/data_sources/authoritative_feeds/__init__.py +0 -13
- equity_aggregator/adapters/data_sources/authoritative_feeds/euronext.py +0 -420
- equity_aggregator/adapters/data_sources/authoritative_feeds/lse.py +0 -352
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/feed.py +0 -350
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/utils/__init__.py +0 -9
- equity_aggregator/domain/pipeline/transforms/deduplicate.py +0 -54
- equity_aggregator/schemas/feeds/euronext_feed_data.py +0 -59
- equity_aggregator-0.1.1.dist-info/RECORD +0 -72
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/entry_points.txt +0 -0
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/licenses/LICENCE.txt +0 -0
|
@@ -3,422 +3,465 @@
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import logging
|
|
5
5
|
from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable
|
|
6
|
-
from contextlib import AsyncExitStack
|
|
6
|
+
from contextlib import AsyncExitStack, asynccontextmanager
|
|
7
7
|
from typing import NamedTuple
|
|
8
8
|
|
|
9
|
-
from equity_aggregator.adapters import open_yfinance_feed
|
|
10
|
-
from equity_aggregator.domain._utils import
|
|
9
|
+
from equity_aggregator.adapters import open_gleif_feed, open_yfinance_feed
|
|
10
|
+
from equity_aggregator.domain._utils import (
|
|
11
|
+
EquityIdentifiers,
|
|
12
|
+
extract_identifiers,
|
|
13
|
+
get_usd_converter,
|
|
14
|
+
merge,
|
|
15
|
+
)
|
|
16
|
+
from equity_aggregator.schemas import GleifFeedData, YFinanceFeedData
|
|
11
17
|
from equity_aggregator.schemas.raw import RawEquity
|
|
12
|
-
from equity_aggregator.schemas import YFinanceFeedData
|
|
13
18
|
|
|
14
19
|
logger = logging.getLogger(__name__)
|
|
15
20
|
|
|
16
21
|
# Type alias for an async function that fetches enrichment data for an equity
|
|
17
|
-
type FetchFunc = Callable[..., Awaitable[dict[str, object]]]
|
|
22
|
+
type FetchFunc = Callable[..., Awaitable[dict[str, object] | None]]
|
|
18
23
|
|
|
19
|
-
# Type alias for a factory that creates an async feed context manager
|
|
20
|
-
type FeedFactory = Callable[[], AsyncIterator[object]]
|
|
21
24
|
|
|
22
|
-
|
|
23
|
-
|
|
25
|
+
class FeedSpec(NamedTuple):
|
|
26
|
+
"""
|
|
27
|
+
Static specification for an enrichment feed.
|
|
24
28
|
|
|
25
|
-
|
|
26
|
-
|
|
29
|
+
Attributes:
|
|
30
|
+
factory: Async context manager factory that yields a feed instance.
|
|
31
|
+
model: Pydantic model for validating feed data.
|
|
32
|
+
limit: Maximum number of concurrent requests to this feed.
|
|
33
|
+
"""
|
|
27
34
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
open_yfinance_feed, # factory for creating YFinance feed context
|
|
32
|
-
YFinanceFeedData, # data model for YFinance feed data
|
|
33
|
-
10, # concurrency limit (max simultaneous YFinance requests)
|
|
34
|
-
),
|
|
35
|
-
]
|
|
35
|
+
factory: Callable
|
|
36
|
+
model: type
|
|
37
|
+
limit: int
|
|
36
38
|
|
|
37
39
|
|
|
38
40
|
class EnrichmentFeed(NamedTuple):
|
|
39
41
|
"""
|
|
40
|
-
|
|
42
|
+
Runtime instance of an enrichment feed with rate limiting applied.
|
|
41
43
|
|
|
42
|
-
|
|
43
|
-
fetch
|
|
44
|
-
model
|
|
45
|
-
semaphore (asyncio.Semaphore): Semaphore to control concurrency for fetch.
|
|
46
|
-
|
|
47
|
-
Returns:
|
|
48
|
-
EnrichmentFeed: A named tuple containing fetch, model, and semaphore.
|
|
44
|
+
Attributes:
|
|
45
|
+
fetch: Rate-limited async function to fetch enrichment data.
|
|
46
|
+
model: Pydantic model for validating feed data.
|
|
49
47
|
"""
|
|
50
48
|
|
|
51
49
|
fetch: FetchFunc
|
|
52
50
|
model: type
|
|
53
|
-
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# Specification for all enrichment feeds
|
|
54
|
+
enrichment_feed_specs: tuple[FeedSpec, ...] = (
|
|
55
|
+
FeedSpec(open_yfinance_feed, YFinanceFeedData, 20),
|
|
56
|
+
FeedSpec(open_gleif_feed, GleifFeedData, 100),
|
|
57
|
+
)
|
|
54
58
|
|
|
55
59
|
|
|
56
60
|
async def enrich(
|
|
57
|
-
|
|
61
|
+
equity_groups: AsyncIterable[list[RawEquity]],
|
|
58
62
|
) -> AsyncIterable[RawEquity]:
|
|
59
63
|
"""
|
|
60
|
-
Enrich
|
|
64
|
+
Enrich equity groups and merge all sources (discovery + enrichment).
|
|
61
65
|
|
|
62
|
-
|
|
63
|
-
enrichment
|
|
64
|
-
|
|
66
|
+
For each group of discovery feed equities, computes median identifiers,
|
|
67
|
+
queries enrichment feeds, then performs a single merge of all sources
|
|
68
|
+
for optimal data quality.
|
|
65
69
|
|
|
66
70
|
Args:
|
|
67
|
-
|
|
68
|
-
Async iterable stream of RawEquity objects to enrich.
|
|
71
|
+
equity_groups: Stream of equity groups (discovery feed sources).
|
|
69
72
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
enrichment finishes.
|
|
73
|
+
Yields:
|
|
74
|
+
RawEquity: Fully merged and enriched equities.
|
|
73
75
|
"""
|
|
74
|
-
async with
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
model=model,
|
|
79
|
-
semaphore=asyncio.Semaphore(limit),
|
|
80
|
-
)
|
|
81
|
-
for factory, model, limit in feed_specs
|
|
82
|
-
]
|
|
76
|
+
async with _open_feeds(enrichment_feed_specs) as feeds:
|
|
77
|
+
async for enriched in _process_stream(equity_groups, feeds):
|
|
78
|
+
yield enriched
|
|
79
|
+
|
|
83
80
|
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
81
|
+
@asynccontextmanager
|
|
82
|
+
async def _open_feeds(
|
|
83
|
+
specs: tuple[FeedSpec, ...],
|
|
84
|
+
) -> AsyncIterator[tuple[EnrichmentFeed, ...]]:
|
|
85
|
+
"""
|
|
86
|
+
Open and initialise all enrichment feeds with lifecycle management.
|
|
87
|
+
|
|
88
|
+
Creates an async context that initialises each feed with rate-limited fetch
|
|
89
|
+
functions, manages their lifecycle through AsyncExitStack, and logs completion
|
|
90
|
+
when the context exits. All feeds are initialised sequentially to ensure
|
|
91
|
+
proper resource allocation.
|
|
91
92
|
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
93
|
+
Args:
|
|
94
|
+
specs: Tuple of feed specifications to initialise.
|
|
95
|
+
|
|
96
|
+
Yields:
|
|
97
|
+
tuple[EnrichmentFeed, ...]: Initialised feeds with rate limiting applied,
|
|
98
|
+
ready for concurrent enrichment operations.
|
|
99
|
+
"""
|
|
100
|
+
async with AsyncExitStack() as stack:
|
|
101
|
+
feeds = tuple([await _init_feed(spec, stack) for spec in specs])
|
|
102
|
+
yield feeds
|
|
95
103
|
|
|
96
104
|
logger.info(
|
|
97
|
-
"Enrichment finished
|
|
98
|
-
|
|
99
|
-
", ".join(feed.model.__name__.removesuffix("FeedData") for feed in feeds),
|
|
105
|
+
"Enrichment finished using feeds: %s",
|
|
106
|
+
", ".join(_feed_name(f.model) for f in feeds),
|
|
100
107
|
)
|
|
101
108
|
|
|
102
109
|
|
|
103
|
-
async def
|
|
104
|
-
source: RawEquity,
|
|
105
|
-
feeds: list[EnrichmentFeed],
|
|
106
|
-
) -> RawEquity:
|
|
110
|
+
async def _init_feed(spec: FeedSpec, stack: AsyncExitStack) -> EnrichmentFeed:
|
|
107
111
|
"""
|
|
108
|
-
|
|
112
|
+
Initialise a single enrichment feed with rate limiting and lifecycle management.
|
|
109
113
|
|
|
110
|
-
|
|
111
|
-
|
|
114
|
+
Opens the feed using its factory, registers it with the provided AsyncExitStack
|
|
115
|
+
for automatic cleanup, wraps the fetch function with semaphore-based rate
|
|
116
|
+
limiting, and returns a ready-to-use EnrichmentFeed instance.
|
|
112
117
|
|
|
113
118
|
Args:
|
|
114
|
-
|
|
115
|
-
|
|
119
|
+
spec: Feed specification containing factory, model, and concurrency limit.
|
|
120
|
+
stack: AsyncExitStack to manage the feed's async context lifecycle.
|
|
116
121
|
|
|
117
122
|
Returns:
|
|
118
|
-
|
|
123
|
+
EnrichmentFeed: Initialised feed with rate-limited fetch function and
|
|
124
|
+
validation model.
|
|
119
125
|
"""
|
|
126
|
+
feed_instance = await stack.enter_async_context(spec.factory())
|
|
120
127
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
128
|
+
return EnrichmentFeed(
|
|
129
|
+
fetch=_rate_limited(feed_instance.fetch_equity, asyncio.Semaphore(spec.limit)),
|
|
130
|
+
model=spec.model,
|
|
131
|
+
)
|
|
125
132
|
|
|
126
|
-
Args:
|
|
127
|
-
feed (EnrichmentFeed): The enrichment feed containing the fetch function,
|
|
128
|
-
model, and semaphore for concurrency control.
|
|
129
133
|
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
134
|
+
def _rate_limited(
|
|
135
|
+
fn: FetchFunc,
|
|
136
|
+
semaphore: asyncio.Semaphore,
|
|
137
|
+
*,
|
|
138
|
+
timeout: float = 300.0,
|
|
139
|
+
) -> FetchFunc:
|
|
140
|
+
"""
|
|
141
|
+
Wrap an async fetch function with semaphore-based rate limiting and timeout.
|
|
136
142
|
|
|
137
|
-
|
|
143
|
+
The timeout applies only to the actual fetch operation, not the semaphore
|
|
144
|
+
wait time. This ensures tasks waiting in the queue don't timeout before
|
|
145
|
+
they get their turn to execute.
|
|
138
146
|
|
|
139
|
-
|
|
140
|
-
|
|
147
|
+
Args:
|
|
148
|
+
fn: Async function to wrap.
|
|
149
|
+
semaphore: Semaphore to control concurrent calls.
|
|
150
|
+
timeout: Maximum time in seconds for the fetch operation (default: 300s).
|
|
141
151
|
|
|
142
|
-
|
|
143
|
-
|
|
152
|
+
Returns:
|
|
153
|
+
FetchFunc: Wrapped function that acquires semaphore before calling fn
|
|
154
|
+
with timeout protection.
|
|
155
|
+
"""
|
|
144
156
|
|
|
157
|
+
async def wrapper(*args: object, **kwargs: object) -> object:
|
|
158
|
+
async with semaphore:
|
|
159
|
+
return await asyncio.wait_for(fn(*args, **kwargs), timeout=timeout)
|
|
145
160
|
|
|
146
|
-
|
|
147
|
-
source: RawEquity,
|
|
148
|
-
fetch_func: FetchFunc,
|
|
149
|
-
feed_model: type,
|
|
150
|
-
) -> RawEquity:
|
|
151
|
-
"""
|
|
152
|
-
Enrich a RawEquity using a feed: fetch, validate, and convert to USD.
|
|
161
|
+
return wrapper
|
|
153
162
|
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
163
|
+
|
|
164
|
+
def _feed_name(model: type) -> str:
|
|
165
|
+
"""
|
|
166
|
+
Extract a concise feed name from a model class.
|
|
157
167
|
|
|
158
168
|
Args:
|
|
159
|
-
|
|
160
|
-
fetch_func (FetchFunc): Async function to fetch enrichment data.
|
|
161
|
-
feed_model (type): Pydantic model class for validating feed data.
|
|
169
|
+
model: The Pydantic model class (e.g., YFinanceFeedData).
|
|
162
170
|
|
|
163
171
|
Returns:
|
|
164
|
-
|
|
165
|
-
enrichment fails or is unnecessary.
|
|
172
|
+
str: The feed name (e.g., "YFinance").
|
|
166
173
|
"""
|
|
167
|
-
|
|
168
|
-
if not _has_missing_fields(source):
|
|
169
|
-
return source
|
|
174
|
+
return model.__name__.removesuffix("FeedData")
|
|
170
175
|
|
|
171
|
-
# derive a concise feed name for logging (e.g. "YFinance" from "YFinanceFeedData")
|
|
172
|
-
feed_name = feed_model.__name__.removesuffix("FeedData")
|
|
173
176
|
|
|
174
|
-
|
|
175
|
-
|
|
177
|
+
async def _process_stream(
|
|
178
|
+
equity_groups: AsyncIterable[list[RawEquity]],
|
|
179
|
+
feeds: tuple[EnrichmentFeed, ...],
|
|
180
|
+
) -> AsyncIterable[RawEquity]:
|
|
181
|
+
"""
|
|
182
|
+
Schedule enrichment for each equity group and yield merged results.
|
|
176
183
|
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
184
|
+
Creates an enrichment task for each group in the input stream, then
|
|
185
|
+
yields enriched equities as their tasks complete (potentially out of
|
|
186
|
+
original order).
|
|
180
187
|
|
|
181
|
-
|
|
182
|
-
|
|
188
|
+
Args:
|
|
189
|
+
equity_groups: Stream of equity groups to enrich.
|
|
190
|
+
feeds: Active feeds to use for enrichment.
|
|
183
191
|
|
|
184
|
-
|
|
185
|
-
|
|
192
|
+
Yields:
|
|
193
|
+
RawEquity: Merged equities from all sources (discovery + enrichment).
|
|
194
|
+
"""
|
|
195
|
+
async with asyncio.TaskGroup() as tg:
|
|
196
|
+
tasks = [
|
|
197
|
+
tg.create_task(_enrich_equity_group(group, feeds))
|
|
198
|
+
async for group in equity_groups
|
|
199
|
+
]
|
|
200
|
+
for coro in asyncio.as_completed(tasks):
|
|
201
|
+
yield await coro
|
|
186
202
|
|
|
187
203
|
|
|
188
|
-
async def
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
*,
|
|
193
|
-
wait_timeout: float = 180.0,
|
|
194
|
-
) -> dict[str, object] | None:
|
|
204
|
+
async def _enrich_equity_group(
|
|
205
|
+
discovery_sources: list[RawEquity],
|
|
206
|
+
feeds: tuple[EnrichmentFeed, ...],
|
|
207
|
+
) -> RawEquity:
|
|
195
208
|
"""
|
|
196
|
-
|
|
197
|
-
timeouts and errors.
|
|
209
|
+
Enrich an equity group and merge all sources.
|
|
198
210
|
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
211
|
+
Extracts representative identifiers from discovery sources, queries
|
|
212
|
+
enrichment feeds with those identifiers, then performs a single merge
|
|
213
|
+
of all data points (discovery + enrichment).
|
|
202
214
|
|
|
203
215
|
Args:
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
feed_name (str): The name of the enrichment feed for logging context.
|
|
207
|
-
wait_timeout (float, optional): Maximum time to wait for the fetch, in
|
|
208
|
-
seconds.
|
|
216
|
+
discovery_sources: Discovery feed equities for this group.
|
|
217
|
+
feeds: Active enrichment feeds.
|
|
209
218
|
|
|
210
219
|
Returns:
|
|
211
|
-
|
|
212
|
-
exception occurs or the data is empty.
|
|
220
|
+
RawEquity: Merged equity from all available sources.
|
|
213
221
|
"""
|
|
214
|
-
|
|
222
|
+
# Extract representative identifiers for enrichment queries
|
|
223
|
+
identifiers = extract_identifiers(discovery_sources)
|
|
215
224
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
name=source.name,
|
|
221
|
-
isin=source.isin,
|
|
222
|
-
cusip=source.cusip,
|
|
223
|
-
),
|
|
224
|
-
timeout=wait_timeout,
|
|
225
|
-
)
|
|
225
|
+
# Fetch enrichment data using identifiers
|
|
226
|
+
enrichment_results = await asyncio.gather(
|
|
227
|
+
*(_enrich_from_feed(identifiers, feed) for feed in feeds),
|
|
228
|
+
)
|
|
226
229
|
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
feed_name,
|
|
230
|
-
source,
|
|
231
|
-
error,
|
|
232
|
-
)
|
|
230
|
+
# Filter out None results (failed enrichment attempts)
|
|
231
|
+
enrichment_sources = [r for r in enrichment_results if r is not None]
|
|
233
232
|
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
wait_timeout,
|
|
238
|
-
feed_name,
|
|
239
|
-
)
|
|
233
|
+
# Single merge of all sources: discovery + enrichment
|
|
234
|
+
all_sources = discovery_sources + enrichment_sources
|
|
235
|
+
return merge(all_sources)
|
|
240
236
|
|
|
241
|
-
except Exception as error:
|
|
242
|
-
logger.error(
|
|
243
|
-
"Error fetching from %s: %s",
|
|
244
|
-
feed_name,
|
|
245
|
-
error,
|
|
246
|
-
)
|
|
247
|
-
|
|
248
|
-
return data
|
|
249
237
|
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
) ->
|
|
238
|
+
async def _enrich_from_feed(
|
|
239
|
+
identifiers: EquityIdentifiers,
|
|
240
|
+
feed: EnrichmentFeed,
|
|
241
|
+
) -> RawEquity | None:
|
|
254
242
|
"""
|
|
255
|
-
|
|
243
|
+
Fetch, validate, and convert enrichment data from a single feed.
|
|
244
|
+
|
|
245
|
+
Uses representative identifiers to query the feed, validates the response,
|
|
246
|
+
converts to USD (if monetary data present), and returns the enriched equity.
|
|
247
|
+
Returns None on any failure.
|
|
256
248
|
|
|
257
249
|
Args:
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
data.
|
|
250
|
+
identifiers: Representative identifiers for querying the feed.
|
|
251
|
+
feed: Active feed to use.
|
|
261
252
|
|
|
262
253
|
Returns:
|
|
263
|
-
|
|
264
|
-
source, validates and coerces the record using the feed model, and
|
|
265
|
-
returns a RawEquity instance if successful. If validation fails, log
|
|
266
|
-
and returns the original source.
|
|
254
|
+
RawEquity | None: Enriched equity in USD, or None if enrichment fails.
|
|
267
255
|
"""
|
|
268
|
-
feed_name =
|
|
256
|
+
feed_name = _feed_name(feed.model)
|
|
269
257
|
|
|
270
|
-
|
|
271
|
-
"""
|
|
272
|
-
Validate and coerce a record using the feed model, returning a RawEquity.
|
|
258
|
+
fetched = await _safe_fetch(identifiers, feed.fetch, feed_name)
|
|
273
259
|
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
Returns:
|
|
279
|
-
RawEquity: The validated RawEquity, or the original source if
|
|
280
|
-
validation fails.
|
|
281
|
-
"""
|
|
282
|
-
try:
|
|
283
|
-
# validate the record against the feed model, coercing types as needed
|
|
284
|
-
coerced = feed_model.model_validate(record).model_dump()
|
|
285
|
-
|
|
286
|
-
# convert the coerced data to a RawEquity instance
|
|
287
|
-
return RawEquity.model_validate(coerced)
|
|
260
|
+
validated = (
|
|
261
|
+
_validate(fetched, feed.model, feed_name, identifiers) if fetched else None
|
|
262
|
+
)
|
|
288
263
|
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
fields = {err["loc"][0] for err in error.errors()}
|
|
292
|
-
summary = f"invalid {', '.join(sorted(fields))}"
|
|
293
|
-
else:
|
|
294
|
-
summary = str(error)
|
|
264
|
+
if validated is None:
|
|
265
|
+
return None
|
|
295
266
|
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
)
|
|
301
|
-
return source
|
|
267
|
+
# Non-monetary feeds (e.g. GLEIF) have no currency - skip conversion
|
|
268
|
+
if validated.currency is None:
|
|
269
|
+
_log_success(feed_name, identifiers, validated)
|
|
270
|
+
return validated
|
|
302
271
|
|
|
303
|
-
return
|
|
272
|
+
return await _to_usd(validated, feed_name, identifiers)
|
|
304
273
|
|
|
305
274
|
|
|
306
|
-
def
|
|
275
|
+
async def _safe_fetch(
|
|
276
|
+
identifiers: EquityIdentifiers,
|
|
277
|
+
fetch: FetchFunc,
|
|
278
|
+
feed_name: str,
|
|
279
|
+
) -> dict[str, object] | None:
|
|
307
280
|
"""
|
|
308
|
-
|
|
281
|
+
Safely fetch raw data using identifiers from an enrichment feed.
|
|
282
|
+
|
|
283
|
+
Handles errors, returning None on failure. Logs all errors with
|
|
284
|
+
appropriate context. Timeout is handled by the _rate_limited wrapper.
|
|
309
285
|
|
|
310
286
|
Args:
|
|
311
|
-
|
|
287
|
+
identifiers: Representative identifiers for the equity.
|
|
288
|
+
fetch: Async fetch function for the enrichment feed (already
|
|
289
|
+
wrapped with timeout protection via _rate_limited).
|
|
290
|
+
feed_name: Feed name for logging context.
|
|
312
291
|
|
|
313
292
|
Returns:
|
|
314
|
-
|
|
293
|
+
dict[str, object] | None: Fetched data as dictionary, or None on failure.
|
|
315
294
|
"""
|
|
316
|
-
|
|
295
|
+
try:
|
|
296
|
+
return await fetch(
|
|
297
|
+
symbol=identifiers.symbol,
|
|
298
|
+
name=identifiers.name,
|
|
299
|
+
isin=identifiers.isin,
|
|
300
|
+
cusip=identifiers.cusip,
|
|
301
|
+
cik=identifiers.cik,
|
|
302
|
+
lei=identifiers.lei,
|
|
303
|
+
share_class_figi=identifiers.share_class_figi,
|
|
304
|
+
)
|
|
317
305
|
|
|
306
|
+
except LookupError as e:
|
|
307
|
+
_log_failure(feed_name, identifiers, e)
|
|
318
308
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
309
|
+
except TimeoutError:
|
|
310
|
+
logger.error(
|
|
311
|
+
"Timed out fetching from %s for symbol=%s (isin=%s, cusip=%s, lei=%s). "
|
|
312
|
+
"Request exceeded timeout waiting for response.",
|
|
313
|
+
feed_name,
|
|
314
|
+
identifiers.symbol,
|
|
315
|
+
identifiers.isin or "<none>",
|
|
316
|
+
identifiers.cusip or "<none>",
|
|
317
|
+
identifiers.lei or "<none>",
|
|
318
|
+
)
|
|
319
|
+
|
|
320
|
+
except Exception as e:
|
|
321
|
+
logger.error(
|
|
322
|
+
"Error fetching from %s for symbol=%s: %s: %s",
|
|
323
|
+
feed_name,
|
|
324
|
+
identifiers.symbol,
|
|
325
|
+
type(e).__name__,
|
|
326
|
+
e or "<empty>",
|
|
327
|
+
)
|
|
328
|
+
|
|
329
|
+
return None
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _validate(
|
|
333
|
+
record: dict[str, object],
|
|
334
|
+
model: type,
|
|
335
|
+
feed_name: str,
|
|
336
|
+
identifiers: EquityIdentifiers,
|
|
337
|
+
) -> RawEquity | None:
|
|
323
338
|
"""
|
|
324
|
-
|
|
325
|
-
`enriched`.
|
|
339
|
+
Validate record against model schema and convert to RawEquity.
|
|
326
340
|
|
|
327
|
-
|
|
328
|
-
the
|
|
329
|
-
|
|
341
|
+
Validates the fetched record using the feed's Pydantic model, then
|
|
342
|
+
converts the validated data to a RawEquity instance. Only injects
|
|
343
|
+
share_class_figi from discovery sources if the enrichment feed didn't
|
|
344
|
+
provide one. Returns None on validation failure.
|
|
330
345
|
|
|
331
346
|
Args:
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
347
|
+
record: Raw record to validate.
|
|
348
|
+
model: Pydantic model class for validating feed data.
|
|
349
|
+
feed_name: Feed name for logging context.
|
|
350
|
+
identifiers: Representative ids for logging context and share_class_figi.
|
|
335
351
|
|
|
336
352
|
Returns:
|
|
337
|
-
RawEquity
|
|
353
|
+
RawEquity | None: Validated equity, with share_class_figi from discovery
|
|
354
|
+
sources if enrichment feed didn't provide one, or None on failure.
|
|
338
355
|
"""
|
|
339
|
-
|
|
340
|
-
|
|
356
|
+
try:
|
|
357
|
+
coerced = model.model_validate(record).model_dump()
|
|
341
358
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
for field, value in enriched_data.items()
|
|
346
|
-
if getattr(source, field) is None
|
|
347
|
-
}
|
|
359
|
+
# Only inject share_class_figi if enrichment feed didn't provide one
|
|
360
|
+
if "share_class_figi" not in coerced or coerced["share_class_figi"] is None:
|
|
361
|
+
coerced["share_class_figi"] = identifiers.share_class_figi
|
|
348
362
|
|
|
349
|
-
|
|
350
|
-
|
|
363
|
+
return RawEquity.model_validate(coerced)
|
|
364
|
+
|
|
365
|
+
except Exception as e:
|
|
366
|
+
summary = (
|
|
367
|
+
f"invalid {', '.join(sorted(err['loc'][0] for err in e.errors()))}"
|
|
368
|
+
if hasattr(e, "errors")
|
|
369
|
+
else str(e)
|
|
370
|
+
)
|
|
371
|
+
_log_failure(feed_name, identifiers, summary)
|
|
372
|
+
return None
|
|
351
373
|
|
|
352
374
|
|
|
353
|
-
async def
|
|
375
|
+
async def _to_usd(
|
|
354
376
|
validated: RawEquity,
|
|
355
|
-
source: RawEquity,
|
|
356
377
|
feed_name: str,
|
|
357
|
-
|
|
378
|
+
identifiers: EquityIdentifiers,
|
|
379
|
+
) -> RawEquity | None:
|
|
358
380
|
"""
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
381
|
+
Convert a validated RawEquity instance to USD.
|
|
382
|
+
|
|
383
|
+
Applies currency conversion using the global USD converter. Returns
|
|
384
|
+
None on conversion failure. Only called for feeds that provide
|
|
385
|
+
monetary data (currency is not None).
|
|
362
386
|
|
|
363
387
|
Args:
|
|
364
|
-
validated
|
|
365
|
-
|
|
366
|
-
|
|
388
|
+
validated: RawEquity instance to convert to USD.
|
|
389
|
+
feed_name: Feed name for logging context.
|
|
390
|
+
identifiers: Representative identifiers for logging context.
|
|
367
391
|
|
|
368
392
|
Returns:
|
|
369
|
-
RawEquity:
|
|
370
|
-
source RawEquity.
|
|
393
|
+
RawEquity | None: USD-converted equity or None on failure.
|
|
371
394
|
"""
|
|
372
395
|
converter = await get_usd_converter()
|
|
373
396
|
|
|
374
397
|
try:
|
|
375
398
|
converted = converter(validated)
|
|
376
399
|
|
|
377
|
-
if converted is None:
|
|
378
|
-
raise ValueError(
|
|
400
|
+
if converted is None or converted.currency != "USD":
|
|
401
|
+
raise ValueError(
|
|
402
|
+
f"USD conversion failed: {converted.currency if converted else None}",
|
|
403
|
+
)
|
|
404
|
+
|
|
405
|
+
_log_success(feed_name, identifiers, converted)
|
|
379
406
|
return converted
|
|
380
407
|
|
|
381
|
-
except Exception as
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
source,
|
|
385
|
-
error,
|
|
386
|
-
)
|
|
387
|
-
return source
|
|
408
|
+
except Exception as e:
|
|
409
|
+
_log_failure(feed_name, identifiers, e)
|
|
410
|
+
return None
|
|
388
411
|
|
|
389
412
|
|
|
390
|
-
def
|
|
413
|
+
def _log_success(
|
|
391
414
|
feed_name: str,
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
*,
|
|
395
|
-
level: int = logging.DEBUG,
|
|
415
|
+
identifiers: EquityIdentifiers,
|
|
416
|
+
result: RawEquity,
|
|
396
417
|
) -> None:
|
|
397
418
|
"""
|
|
398
|
-
Log
|
|
399
|
-
|
|
400
|
-
This logs details about the equity and the error context when no data is
|
|
401
|
-
available from a feed, or when enrichment fails.
|
|
419
|
+
Log successful enrichment with representative identifiers.
|
|
402
420
|
|
|
403
421
|
Args:
|
|
404
|
-
feed_name
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
422
|
+
feed_name: Name of the enrichment feed.
|
|
423
|
+
identifiers: Representative identifiers from discovery sources.
|
|
424
|
+
result: The enriched RawEquity returned by the feed.
|
|
425
|
+
"""
|
|
426
|
+
prefix = f"[{feed_name}:{identifiers.symbol}]"
|
|
427
|
+
|
|
428
|
+
msg = (
|
|
429
|
+
f"{prefix:<24} SUCCESS: {feed_name} feed for symbol={identifiers.symbol}, "
|
|
430
|
+
f"name={identifiers.name} "
|
|
431
|
+
f"(share_class_figi={identifiers.share_class_figi or '<none>'}, "
|
|
432
|
+
f"isin={identifiers.isin or result.isin or '<none>'}, "
|
|
433
|
+
f"cusip={identifiers.cusip or result.cusip or '<none>'}, "
|
|
434
|
+
f"cik={identifiers.cik or result.cik or '<none>'}, "
|
|
435
|
+
f"lei={identifiers.lei or result.lei or '<none>'})"
|
|
436
|
+
)
|
|
408
437
|
|
|
409
|
-
|
|
410
|
-
|
|
438
|
+
logger.debug(msg)
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _log_failure(
|
|
442
|
+
feed_name: str,
|
|
443
|
+
identifiers: EquityIdentifiers,
|
|
444
|
+
error: object,
|
|
445
|
+
) -> None:
|
|
446
|
+
"""
|
|
447
|
+
Log failed enrichment with the input identifiers that were attempted.
|
|
448
|
+
|
|
449
|
+
Args:
|
|
450
|
+
feed_name: Name of the enrichment feed.
|
|
451
|
+
identifiers: Representative identifiers that were used for the lookup.
|
|
452
|
+
error: Error or context describing why the lookup failed.
|
|
411
453
|
"""
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
"
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
error,
|
|
454
|
+
prefix = f"[{feed_name}:{identifiers.symbol}]"
|
|
455
|
+
|
|
456
|
+
msg = (
|
|
457
|
+
f"{prefix:<24} FAILURE: {feed_name} feed for symbol={identifiers.symbol}, "
|
|
458
|
+
f"name={identifiers.name} "
|
|
459
|
+
f"(share_class_figi={identifiers.share_class_figi or '<none>'}, "
|
|
460
|
+
f"isin={identifiers.isin or '<none>'}, "
|
|
461
|
+
f"cusip={identifiers.cusip or '<none>'}, "
|
|
462
|
+
f"cik={identifiers.cik or '<none>'}, "
|
|
463
|
+
f"lei={identifiers.lei or '<none>'}). "
|
|
464
|
+
f"{error}"
|
|
424
465
|
)
|
|
466
|
+
|
|
467
|
+
logger.debug(msg)
|