equity-aggregator 0.1.1__py3-none-any.whl → 0.1.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. equity_aggregator/README.md +40 -36
  2. equity_aggregator/adapters/__init__.py +13 -7
  3. equity_aggregator/adapters/data_sources/__init__.py +4 -6
  4. equity_aggregator/adapters/data_sources/_utils/_client.py +1 -1
  5. equity_aggregator/adapters/data_sources/{authoritative_feeds → _utils}/_record_types.py +1 -1
  6. equity_aggregator/adapters/data_sources/discovery_feeds/__init__.py +17 -0
  7. equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/__init__.py +7 -0
  8. equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/__init__.py +10 -0
  9. equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/backoff.py +33 -0
  10. equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/parser.py +107 -0
  11. equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/intrinio.py +305 -0
  12. equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/session.py +197 -0
  13. equity_aggregator/adapters/data_sources/discovery_feeds/lseg/__init__.py +7 -0
  14. equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/__init__.py +9 -0
  15. equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/backoff.py +33 -0
  16. equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/parser.py +120 -0
  17. equity_aggregator/adapters/data_sources/discovery_feeds/lseg/lseg.py +239 -0
  18. equity_aggregator/adapters/data_sources/discovery_feeds/lseg/session.py +162 -0
  19. equity_aggregator/adapters/data_sources/discovery_feeds/sec/__init__.py +7 -0
  20. equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/sec}/sec.py +4 -5
  21. equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/__init__.py +7 -0
  22. equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/stock_analysis.py +150 -0
  23. equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/__init__.py +5 -0
  24. equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/tradingview.py +275 -0
  25. equity_aggregator/adapters/data_sources/discovery_feeds/xetra/__init__.py +7 -0
  26. equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/xetra}/xetra.py +9 -12
  27. equity_aggregator/adapters/data_sources/enrichment_feeds/__init__.py +6 -1
  28. equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/__init__.py +5 -0
  29. equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/api.py +71 -0
  30. equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/download.py +109 -0
  31. equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/gleif.py +195 -0
  32. equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/parser.py +75 -0
  33. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/__init__.py +1 -1
  34. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/_utils/__init__.py +11 -0
  35. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/{utils → _utils}/backoff.py +1 -1
  36. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/{utils → _utils}/fuzzy.py +28 -26
  37. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/_utils/json.py +36 -0
  38. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/__init__.py +1 -1
  39. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/{summary.py → quote_summary.py} +44 -30
  40. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/search.py +10 -5
  41. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/auth.py +130 -0
  42. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/config.py +3 -3
  43. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/ranking.py +97 -0
  44. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/session.py +85 -218
  45. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/transport.py +191 -0
  46. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/yfinance.py +413 -0
  47. equity_aggregator/adapters/data_sources/reference_lookup/exchange_rate_api.py +6 -13
  48. equity_aggregator/adapters/data_sources/reference_lookup/openfigi.py +23 -7
  49. equity_aggregator/cli/dispatcher.py +11 -8
  50. equity_aggregator/cli/main.py +14 -5
  51. equity_aggregator/cli/parser.py +1 -1
  52. equity_aggregator/cli/signals.py +32 -0
  53. equity_aggregator/domain/_utils/__init__.py +2 -2
  54. equity_aggregator/domain/_utils/_load_converter.py +30 -21
  55. equity_aggregator/domain/_utils/_merge.py +221 -368
  56. equity_aggregator/domain/_utils/_merge_config.py +205 -0
  57. equity_aggregator/domain/_utils/_strategies.py +180 -0
  58. equity_aggregator/domain/pipeline/resolve.py +17 -11
  59. equity_aggregator/domain/pipeline/runner.py +4 -4
  60. equity_aggregator/domain/pipeline/seed.py +5 -1
  61. equity_aggregator/domain/pipeline/transforms/__init__.py +2 -2
  62. equity_aggregator/domain/pipeline/transforms/canonicalise.py +1 -1
  63. equity_aggregator/domain/pipeline/transforms/enrich.py +328 -285
  64. equity_aggregator/domain/pipeline/transforms/group.py +48 -0
  65. equity_aggregator/logging_config.py +4 -1
  66. equity_aggregator/schemas/__init__.py +11 -5
  67. equity_aggregator/schemas/canonical.py +11 -6
  68. equity_aggregator/schemas/feeds/__init__.py +11 -5
  69. equity_aggregator/schemas/feeds/gleif_feed_data.py +35 -0
  70. equity_aggregator/schemas/feeds/intrinio_feed_data.py +142 -0
  71. equity_aggregator/schemas/feeds/{lse_feed_data.py → lseg_feed_data.py} +85 -52
  72. equity_aggregator/schemas/feeds/sec_feed_data.py +36 -6
  73. equity_aggregator/schemas/feeds/stock_analysis_feed_data.py +107 -0
  74. equity_aggregator/schemas/feeds/tradingview_feed_data.py +144 -0
  75. equity_aggregator/schemas/feeds/xetra_feed_data.py +1 -1
  76. equity_aggregator/schemas/feeds/yfinance_feed_data.py +47 -35
  77. equity_aggregator/schemas/raw.py +5 -3
  78. equity_aggregator/schemas/types.py +7 -0
  79. equity_aggregator/schemas/validators.py +81 -27
  80. equity_aggregator/storage/data_store.py +5 -3
  81. {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/METADATA +205 -115
  82. equity_aggregator-0.1.4.dist-info/RECORD +103 -0
  83. {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/WHEEL +1 -1
  84. equity_aggregator/adapters/data_sources/authoritative_feeds/__init__.py +0 -13
  85. equity_aggregator/adapters/data_sources/authoritative_feeds/euronext.py +0 -420
  86. equity_aggregator/adapters/data_sources/authoritative_feeds/lse.py +0 -352
  87. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/feed.py +0 -350
  88. equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/utils/__init__.py +0 -9
  89. equity_aggregator/domain/pipeline/transforms/deduplicate.py +0 -54
  90. equity_aggregator/schemas/feeds/euronext_feed_data.py +0 -59
  91. equity_aggregator-0.1.1.dist-info/RECORD +0 -72
  92. {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/entry_points.txt +0 -0
  93. {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.4.dist-info}/licenses/LICENCE.txt +0 -0
@@ -3,422 +3,465 @@
3
3
  import asyncio
4
4
  import logging
5
5
  from collections.abc import AsyncIterable, AsyncIterator, Awaitable, Callable
6
- from contextlib import AsyncExitStack
6
+ from contextlib import AsyncExitStack, asynccontextmanager
7
7
  from typing import NamedTuple
8
8
 
9
- from equity_aggregator.adapters import open_yfinance_feed
10
- from equity_aggregator.domain._utils import get_usd_converter, merge
9
+ from equity_aggregator.adapters import open_gleif_feed, open_yfinance_feed
10
+ from equity_aggregator.domain._utils import (
11
+ EquityIdentifiers,
12
+ extract_identifiers,
13
+ get_usd_converter,
14
+ merge,
15
+ )
16
+ from equity_aggregator.schemas import GleifFeedData, YFinanceFeedData
11
17
  from equity_aggregator.schemas.raw import RawEquity
12
- from equity_aggregator.schemas import YFinanceFeedData
13
18
 
14
19
  logger = logging.getLogger(__name__)
15
20
 
16
21
  # Type alias for an async function that fetches enrichment data for an equity
17
- type FetchFunc = Callable[..., Awaitable[dict[str, object]]]
22
+ type FetchFunc = Callable[..., Awaitable[dict[str, object] | None]]
18
23
 
19
- # Type alias for a factory that creates an async feed context manager
20
- type FeedFactory = Callable[[], AsyncIterator[object]]
21
24
 
22
- # Type alias for a tuple describing a feed: (factory, model, concurrency limit)
23
- type FeedSpec = tuple[FeedFactory, type, int]
25
+ class FeedSpec(NamedTuple):
26
+ """
27
+ Static specification for an enrichment feed.
24
28
 
25
- # Type alias for a function that validates and converts feed data to RawEquity
26
- type ValidatorFunc = Callable[[dict[str, object], RawEquity], RawEquity]
29
+ Attributes:
30
+ factory: Async context manager factory that yields a feed instance.
31
+ model: Pydantic model for validating feed data.
32
+ limit: Maximum number of concurrent requests to this feed.
33
+ """
27
34
 
28
- # List of enrichment feeds to use, each with its factory, model, and concurrency limit
29
- feed_specs: list[FeedSpec] = [
30
- (
31
- open_yfinance_feed, # factory for creating YFinance feed context
32
- YFinanceFeedData, # data model for YFinance feed data
33
- 10, # concurrency limit (max simultaneous YFinance requests)
34
- ),
35
- ]
35
+ factory: Callable
36
+ model: type
37
+ limit: int
36
38
 
37
39
 
38
40
  class EnrichmentFeed(NamedTuple):
39
41
  """
40
- Represents a feed for enrichment in a data pipeline.
42
+ Runtime instance of an enrichment feed with rate limiting applied.
41
43
 
42
- Args:
43
- fetch (FetchFunc): A callable responsible for fetching enrichment data.
44
- model (type): The type of model to use for enrichment.
45
- semaphore (asyncio.Semaphore): Semaphore to control concurrency for fetch.
46
-
47
- Returns:
48
- EnrichmentFeed: A named tuple containing fetch, model, and semaphore.
44
+ Attributes:
45
+ fetch: Rate-limited async function to fetch enrichment data.
46
+ model: Pydantic model for validating feed data.
49
47
  """
50
48
 
51
49
  fetch: FetchFunc
52
50
  model: type
53
- semaphore: asyncio.Semaphore
51
+
52
+
53
+ # Specification for all enrichment feeds
54
+ enrichment_feed_specs: tuple[FeedSpec, ...] = (
55
+ FeedSpec(open_yfinance_feed, YFinanceFeedData, 20),
56
+ FeedSpec(open_gleif_feed, GleifFeedData, 100),
57
+ )
54
58
 
55
59
 
56
60
  async def enrich(
57
- raw_equities: AsyncIterable[RawEquity],
61
+ equity_groups: AsyncIterable[list[RawEquity]],
58
62
  ) -> AsyncIterable[RawEquity]:
59
63
  """
60
- Enrich a stream of RawEquity objects concurrently using configured feeds.
64
+ Enrich equity groups and merge all sources (discovery + enrichment).
61
65
 
62
- Each RawEquity is scheduled for enrichment and yielded as soon as its
63
- enrichment completes. Enrichment is performed concurrently, respecting
64
- per-feed concurrency limits.
66
+ For each group of discovery feed equities, computes median identifiers,
67
+ queries enrichment feeds, then performs a single merge of all sources
68
+ for optimal data quality.
65
69
 
66
70
  Args:
67
- raw_equities (AsyncIterable[RawEquity]):
68
- Async iterable stream of RawEquity objects to enrich.
71
+ equity_groups: Stream of equity groups (discovery feed sources).
69
72
 
70
- Returns:
71
- AsyncIterable[RawEquity]: Yields each enriched RawEquity as soon as
72
- enrichment finishes.
73
+ Yields:
74
+ RawEquity: Fully merged and enriched equities.
73
75
  """
74
- async with AsyncExitStack() as stack:
75
- feeds: list[EnrichmentFeed] = [
76
- EnrichmentFeed(
77
- fetch=(await stack.enter_async_context(factory())).fetch_equity,
78
- model=model,
79
- semaphore=asyncio.Semaphore(limit),
80
- )
81
- for factory, model, limit in feed_specs
82
- ]
76
+ async with _open_feeds(enrichment_feed_specs) as feeds:
77
+ async for enriched in _process_stream(equity_groups, feeds):
78
+ yield enriched
79
+
83
80
 
84
- # launch enrichment tasks and yield results as they complete
85
- async with asyncio.TaskGroup() as enrich_tasks:
86
- tasks: list[asyncio.Task[RawEquity]] = []
87
- async for equity in raw_equities:
88
- tasks.append(
89
- enrich_tasks.create_task(_enrich_equity(equity, feeds)),
90
- )
81
+ @asynccontextmanager
82
+ async def _open_feeds(
83
+ specs: tuple[FeedSpec, ...],
84
+ ) -> AsyncIterator[tuple[EnrichmentFeed, ...]]:
85
+ """
86
+ Open and initialise all enrichment feeds with lifecycle management.
87
+
88
+ Creates an async context that initialises each feed with rate-limited fetch
89
+ functions, manages their lifecycle through AsyncExitStack, and logs completion
90
+ when the context exits. All feeds are initialised sequentially to ensure
91
+ proper resource allocation.
91
92
 
92
- for completed in asyncio.as_completed(tasks):
93
- enriched = await completed
94
- yield enriched
93
+ Args:
94
+ specs: Tuple of feed specifications to initialise.
95
+
96
+ Yields:
97
+ tuple[EnrichmentFeed, ...]: Initialised feeds with rate limiting applied,
98
+ ready for concurrent enrichment operations.
99
+ """
100
+ async with AsyncExitStack() as stack:
101
+ feeds = tuple([await _init_feed(spec, stack) for spec in specs])
102
+ yield feeds
95
103
 
96
104
  logger.info(
97
- "Enrichment finished for %d equities using enrichment feeds: %s",
98
- len(tasks),
99
- ", ".join(feed.model.__name__.removesuffix("FeedData") for feed in feeds),
105
+ "Enrichment finished using feeds: %s",
106
+ ", ".join(_feed_name(f.model) for f in feeds),
100
107
  )
101
108
 
102
109
 
103
- async def _enrich_equity(
104
- source: RawEquity,
105
- feeds: list[EnrichmentFeed],
106
- ) -> RawEquity:
110
+ async def _init_feed(spec: FeedSpec, stack: AsyncExitStack) -> EnrichmentFeed:
107
111
  """
108
- Enrich a RawEquity instance concurrently using all configured enrichment feeds.
112
+ Initialise a single enrichment feed with rate limiting and lifecycle management.
109
113
 
110
- For each feed, fetch and validate data for the given equity. Merge results with
111
- the source, preferring non-None fields from the source.
114
+ Opens the feed using its factory, registers it with the provided AsyncExitStack
115
+ for automatic cleanup, wraps the fetch function with semaphore-based rate
116
+ limiting, and returns a ready-to-use EnrichmentFeed instance.
112
117
 
113
118
  Args:
114
- source (RawEquity): The RawEquity object to enrich (assumed USD-denominated).
115
- feeds (list[EnrichmentFeed]): List of enrichment feeds to use.
119
+ spec: Feed specification containing factory, model, and concurrency limit.
120
+ stack: AsyncExitStack to manage the feed's async context lifecycle.
116
121
 
117
122
  Returns:
118
- RawEquity: The enriched RawEquity with missing fields filled where possible.
123
+ EnrichmentFeed: Initialised feed with rate-limited fetch function and
124
+ validation model.
119
125
  """
126
+ feed_instance = await stack.enter_async_context(spec.factory())
120
127
 
121
- async def run(feed: EnrichmentFeed) -> RawEquity:
122
- """
123
- Enrich a RawEquity using a single feed, respecting the feed's concurrency
124
- limit.
128
+ return EnrichmentFeed(
129
+ fetch=_rate_limited(feed_instance.fetch_equity, asyncio.Semaphore(spec.limit)),
130
+ model=spec.model,
131
+ )
125
132
 
126
- Args:
127
- feed (EnrichmentFeed): The enrichment feed containing the fetch function,
128
- model, and semaphore for concurrency control.
129
133
 
130
- Returns:
131
- RawEquity: The enriched RawEquity instance, or the original if enrichment
132
- fails.
133
- """
134
- async with feed.semaphore:
135
- return await _enrich_with_feed(source, feed.fetch, feed.model)
134
+ def _rate_limited(
135
+ fn: FetchFunc,
136
+ semaphore: asyncio.Semaphore,
137
+ *,
138
+ timeout: float = 300.0,
139
+ ) -> FetchFunc:
140
+ """
141
+ Wrap an async fetch function with semaphore-based rate limiting and timeout.
136
142
 
137
- enriched_equities = await asyncio.gather(*(run(feed) for feed in feeds))
143
+ The timeout applies only to the actual fetch operation, not the semaphore
144
+ wait time. This ensures tasks waiting in the queue don't timeout before
145
+ they get their turn to execute.
138
146
 
139
- # merge all feed‐enriched RawEquity instances into one
140
- merged_from_feeds = merge(enriched_equities)
147
+ Args:
148
+ fn: Async function to wrap.
149
+ semaphore: Semaphore to control concurrent calls.
150
+ timeout: Maximum time in seconds for the fetch operation (default: 300s).
141
151
 
142
- # replace only the none‐fields in source with values from merged_from_feeds
143
- return _replace_none_with_enriched(source, merged_from_feeds)
152
+ Returns:
153
+ FetchFunc: Wrapped function that acquires semaphore before calling fn
154
+ with timeout protection.
155
+ """
144
156
 
157
+ async def wrapper(*args: object, **kwargs: object) -> object:
158
+ async with semaphore:
159
+ return await asyncio.wait_for(fn(*args, **kwargs), timeout=timeout)
145
160
 
146
- async def _enrich_with_feed(
147
- source: RawEquity,
148
- fetch_func: FetchFunc,
149
- feed_model: type,
150
- ) -> RawEquity:
151
- """
152
- Enrich a RawEquity using a feed: fetch, validate, and convert to USD.
161
+ return wrapper
153
162
 
154
- If the source has no missing fields, returns it unchanged. Otherwise, fetches
155
- data from the feed, validates it, and converts to USD. If any step fails,
156
- returns the original source.
163
+
164
+ def _feed_name(model: type) -> str:
165
+ """
166
+ Extract a concise feed name from a model class.
157
167
 
158
168
  Args:
159
- source (RawEquity): The equity to enrich, possibly with missing fields.
160
- fetch_func (FetchFunc): Async function to fetch enrichment data.
161
- feed_model (type): Pydantic model class for validating feed data.
169
+ model: The Pydantic model class (e.g., YFinanceFeedData).
162
170
 
163
171
  Returns:
164
- RawEquity: The enriched RawEquity in USD, or the original source if
165
- enrichment fails or is unnecessary.
172
+ str: The feed name (e.g., "YFinance").
166
173
  """
167
- # if source has no missing fields, skip enrichment
168
- if not _has_missing_fields(source):
169
- return source
174
+ return model.__name__.removesuffix("FeedData")
170
175
 
171
- # derive a concise feed name for logging (e.g. "YFinance" from "YFinanceFeedData")
172
- feed_name = feed_model.__name__.removesuffix("FeedData")
173
176
 
174
- # fetch the raw data, with timeout and exception handling
175
- fetched_raw_data = await _safe_fetch(source, fetch_func, feed_name)
177
+ async def _process_stream(
178
+ equity_groups: AsyncIterable[list[RawEquity]],
179
+ feeds: tuple[EnrichmentFeed, ...],
180
+ ) -> AsyncIterable[RawEquity]:
181
+ """
182
+ Schedule enrichment for each equity group and yield merged results.
176
183
 
177
- # if no data was fetched, fall back to source
178
- if not fetched_raw_data:
179
- return source
184
+ Creates an enrichment task for each group in the input stream, then
185
+ yields enriched equities as their tasks complete (potentially out of
186
+ original order).
180
187
 
181
- # validate the fetched data against the feed model
182
- validated = _make_validator(feed_model)(fetched_raw_data, source)
188
+ Args:
189
+ equity_groups: Stream of equity groups to enrich.
190
+ feeds: Active feeds to use for enrichment.
183
191
 
184
- # always convert the validated feed‐record to USD or else fall back to source
185
- return await _convert_to_usd_or_fallback(validated, source, feed_name)
192
+ Yields:
193
+ RawEquity: Merged equities from all sources (discovery + enrichment).
194
+ """
195
+ async with asyncio.TaskGroup() as tg:
196
+ tasks = [
197
+ tg.create_task(_enrich_equity_group(group, feeds))
198
+ async for group in equity_groups
199
+ ]
200
+ for coro in asyncio.as_completed(tasks):
201
+ yield await coro
186
202
 
187
203
 
188
- async def _safe_fetch(
189
- source: RawEquity,
190
- fetcher: FetchFunc,
191
- feed_name: str,
192
- *,
193
- wait_timeout: float = 180.0,
194
- ) -> dict[str, object] | None:
204
+ async def _enrich_equity_group(
205
+ discovery_sources: list[RawEquity],
206
+ feeds: tuple[EnrichmentFeed, ...],
207
+ ) -> RawEquity:
195
208
  """
196
- Safely fetch raw data for a RawEquity from an enrichment feed, handling
197
- timeouts and errors.
209
+ Enrich an equity group and merge all sources.
198
210
 
199
- Note:
200
- The CIK (Central Index Key) is intentionally omitted as an identifier
201
- for enrichment feeds, as it lacks broad support.
211
+ Extracts representative identifiers from discovery sources, queries
212
+ enrichment feeds with those identifiers, then performs a single merge
213
+ of all data points (discovery + enrichment).
202
214
 
203
215
  Args:
204
- source (RawEquity): The RawEquity instance to fetch data for.
205
- fetcher (FetchFunc): The async fetch function for the enrichment feed.
206
- feed_name (str): The name of the enrichment feed for logging context.
207
- wait_timeout (float, optional): Maximum time to wait for the fetch, in
208
- seconds.
216
+ discovery_sources: Discovery feed equities for this group.
217
+ feeds: Active enrichment feeds.
209
218
 
210
219
  Returns:
211
- dict[str, object] | None: The fetched data as a dictionary, or None if an
212
- exception occurs or the data is empty.
220
+ RawEquity: Merged equity from all available sources.
213
221
  """
214
- data: dict[str, object] | None = None
222
+ # Extract representative identifiers for enrichment queries
223
+ identifiers = extract_identifiers(discovery_sources)
215
224
 
216
- try:
217
- data = await asyncio.wait_for(
218
- fetcher(
219
- symbol=source.symbol,
220
- name=source.name,
221
- isin=source.isin,
222
- cusip=source.cusip,
223
- ),
224
- timeout=wait_timeout,
225
- )
225
+ # Fetch enrichment data using identifiers
226
+ enrichment_results = await asyncio.gather(
227
+ *(_enrich_from_feed(identifiers, feed) for feed in feeds),
228
+ )
226
229
 
227
- except LookupError as error:
228
- _log_no_feed_data(
229
- feed_name,
230
- source,
231
- error,
232
- )
230
+ # Filter out None results (failed enrichment attempts)
231
+ enrichment_sources = [r for r in enrichment_results if r is not None]
233
232
 
234
- except TimeoutError:
235
- logger.error(
236
- "Timed out after %.0f s while fetching from %s.",
237
- wait_timeout,
238
- feed_name,
239
- )
233
+ # Single merge of all sources: discovery + enrichment
234
+ all_sources = discovery_sources + enrichment_sources
235
+ return merge(all_sources)
240
236
 
241
- except Exception as error:
242
- logger.error(
243
- "Error fetching from %s: %s",
244
- feed_name,
245
- error,
246
- )
247
-
248
- return data
249
237
 
250
-
251
- def _make_validator(
252
- feed_model: type,
253
- ) -> ValidatorFunc:
238
+ async def _enrich_from_feed(
239
+ identifiers: EquityIdentifiers,
240
+ feed: EnrichmentFeed,
241
+ ) -> RawEquity | None:
254
242
  """
255
- Create a validator function for a given feed model to validate and coerce records.
243
+ Fetch, validate, and convert enrichment data from a single feed.
244
+
245
+ Uses representative identifiers to query the feed, validates the response,
246
+ converts to USD (if monetary data present), and returns the enriched equity.
247
+ Returns None on any failure.
256
248
 
257
249
  Args:
258
- feed_model (type): The Pydantic model class used to validate and coerce
259
- input records. The model should define the expected schema for the feed
260
- data.
250
+ identifiers: Representative identifiers for querying the feed.
251
+ feed: Active feed to use.
261
252
 
262
253
  Returns:
263
- ValidatorFunc: A function that takes a record dictionary and a RawEquity
264
- source, validates and coerces the record using the feed model, and
265
- returns a RawEquity instance if successful. If validation fails, log
266
- and returns the original source.
254
+ RawEquity | None: Enriched equity in USD, or None if enrichment fails.
267
255
  """
268
- feed_name = feed_model.__name__.removesuffix("FeedData")
256
+ feed_name = _feed_name(feed.model)
269
257
 
270
- def validate(record: dict[str, object], source: RawEquity) -> RawEquity:
271
- """
272
- Validate and coerce a record using the feed model, returning a RawEquity.
258
+ fetched = await _safe_fetch(identifiers, feed.fetch, feed_name)
273
259
 
274
- Args:
275
- record (dict[str, object]): The raw record to validate and coerce.
276
- source (RawEquity): The original RawEquity to return on failure.
277
-
278
- Returns:
279
- RawEquity: The validated RawEquity, or the original source if
280
- validation fails.
281
- """
282
- try:
283
- # validate the record against the feed model, coercing types as needed
284
- coerced = feed_model.model_validate(record).model_dump()
285
-
286
- # convert the coerced data to a RawEquity instance
287
- return RawEquity.model_validate(coerced)
260
+ validated = (
261
+ _validate(fetched, feed.model, feed_name, identifiers) if fetched else None
262
+ )
288
263
 
289
- except Exception as error:
290
- if hasattr(error, "errors"):
291
- fields = {err["loc"][0] for err in error.errors()}
292
- summary = f"invalid {', '.join(sorted(fields))}"
293
- else:
294
- summary = str(error)
264
+ if validated is None:
265
+ return None
295
266
 
296
- _log_no_feed_data(
297
- feed_name,
298
- source,
299
- summary,
300
- )
301
- return source
267
+ # Non-monetary feeds (e.g. GLEIF) have no currency - skip conversion
268
+ if validated.currency is None:
269
+ _log_success(feed_name, identifiers, validated)
270
+ return validated
302
271
 
303
- return validate
272
+ return await _to_usd(validated, feed_name, identifiers)
304
273
 
305
274
 
306
- def _has_missing_fields(equity: RawEquity) -> bool:
275
+ async def _safe_fetch(
276
+ identifiers: EquityIdentifiers,
277
+ fetch: FetchFunc,
278
+ feed_name: str,
279
+ ) -> dict[str, object] | None:
307
280
  """
308
- Checks if any field in a RawEquity instance is missing (i.e., set to None).
281
+ Safely fetch raw data using identifiers from an enrichment feed.
282
+
283
+ Handles errors, returning None on failure. Logs all errors with
284
+ appropriate context. Timeout is handled by the _rate_limited wrapper.
309
285
 
310
286
  Args:
311
- equity (RawEquity): The RawEquity instance to check for missing fields.
287
+ identifiers: Representative identifiers for the equity.
288
+ fetch: Async fetch function for the enrichment feed (already
289
+ wrapped with timeout protection via _rate_limited).
290
+ feed_name: Feed name for logging context.
312
291
 
313
292
  Returns:
314
- bool: True if any field is None, indicating a missing value; False otherwise.
293
+ dict[str, object] | None: Fetched data as dictionary, or None on failure.
315
294
  """
316
- return any(value is None for value in equity.model_dump().values())
295
+ try:
296
+ return await fetch(
297
+ symbol=identifiers.symbol,
298
+ name=identifiers.name,
299
+ isin=identifiers.isin,
300
+ cusip=identifiers.cusip,
301
+ cik=identifiers.cik,
302
+ lei=identifiers.lei,
303
+ share_class_figi=identifiers.share_class_figi,
304
+ )
317
305
 
306
+ except LookupError as e:
307
+ _log_failure(feed_name, identifiers, e)
318
308
 
319
- def _replace_none_with_enriched(
320
- source: RawEquity,
321
- enriched: RawEquity,
322
- ) -> RawEquity:
309
+ except TimeoutError:
310
+ logger.error(
311
+ "Timed out fetching from %s for symbol=%s (isin=%s, cusip=%s, lei=%s). "
312
+ "Request exceeded timeout waiting for response.",
313
+ feed_name,
314
+ identifiers.symbol,
315
+ identifiers.isin or "<none>",
316
+ identifiers.cusip or "<none>",
317
+ identifiers.lei or "<none>",
318
+ )
319
+
320
+ except Exception as e:
321
+ logger.error(
322
+ "Error fetching from %s for symbol=%s: %s: %s",
323
+ feed_name,
324
+ identifiers.symbol,
325
+ type(e).__name__,
326
+ e or "<empty>",
327
+ )
328
+
329
+ return None
330
+
331
+
332
+ def _validate(
333
+ record: dict[str, object],
334
+ model: type,
335
+ feed_name: str,
336
+ identifiers: EquityIdentifiers,
337
+ ) -> RawEquity | None:
323
338
  """
324
- Return new RawEquity instance with missing fields from `source` filled in from
325
- `enriched`.
339
+ Validate record against model schema and convert to RawEquity.
326
340
 
327
- For each field, if `source` has a non-None value, it is kept. If `source` has None,
328
- the value from `enriched` is used, but only if it is not None. None values in
329
- `enriched` never overwrite any value in `source`.
341
+ Validates the fetched record using the feed's Pydantic model, then
342
+ converts the validated data to a RawEquity instance. Only injects
343
+ share_class_figi from discovery sources if the enrichment feed didn't
344
+ provide one. Returns None on validation failure.
330
345
 
331
346
  Args:
332
- source (RawEquity): The original RawEquity instance, possibly with missing
333
- fields.
334
- enriched (RawEquity): The RawEquity instance to use for filling missing fields.
347
+ record: Raw record to validate.
348
+ model: Pydantic model class for validating feed data.
349
+ feed_name: Feed name for logging context.
350
+ identifiers: Representative ids for logging context and share_class_figi.
335
351
 
336
352
  Returns:
337
- RawEquity: A new RawEquity instance with missing fields filled from `enriched`.
353
+ RawEquity | None: Validated equity, with share_class_figi from discovery
354
+ sources if enrichment feed didn't provide one, or None on failure.
338
355
  """
339
- # dump enriched, don’t include any None values
340
- enriched_data = enriched.model_dump(exclude_none=True)
356
+ try:
357
+ coerced = model.model_validate(record).model_dump()
341
358
 
342
- # pick only the keys where source is None
343
- to_update = {
344
- field: value
345
- for field, value in enriched_data.items()
346
- if getattr(source, field) is None
347
- }
359
+ # Only inject share_class_figi if enrichment feed didn't provide one
360
+ if "share_class_figi" not in coerced or coerced["share_class_figi"] is None:
361
+ coerced["share_class_figi"] = identifiers.share_class_figi
348
362
 
349
- # return a copy of source with just those missing fields filled in
350
- return source.model_copy(update=to_update)
363
+ return RawEquity.model_validate(coerced)
364
+
365
+ except Exception as e:
366
+ summary = (
367
+ f"invalid {', '.join(sorted(err['loc'][0] for err in e.errors()))}"
368
+ if hasattr(e, "errors")
369
+ else str(e)
370
+ )
371
+ _log_failure(feed_name, identifiers, summary)
372
+ return None
351
373
 
352
374
 
353
- async def _convert_to_usd_or_fallback(
375
+ async def _to_usd(
354
376
  validated: RawEquity,
355
- source: RawEquity,
356
377
  feed_name: str,
357
- ) -> RawEquity:
378
+ identifiers: EquityIdentifiers,
379
+ ) -> RawEquity | None:
358
380
  """
359
- Attempt to convert a validated RawEquity instance to USD. If conversion fails
360
- due to a missing FX rate (ValueError), log and return the original
361
- source RawEquity.
381
+ Convert a validated RawEquity instance to USD.
382
+
383
+ Applies currency conversion using the global USD converter. Returns
384
+ None on conversion failure. Only called for feeds that provide
385
+ monetary data (currency is not None).
362
386
 
363
387
  Args:
364
- validated (RawEquity): The RawEquity instance to convert to USD.
365
- source (RawEquity): The original RawEquity to return on conversion failure.
366
- feed_name (str): The name of the enrichment feed for logging context.
388
+ validated: RawEquity instance to convert to USD.
389
+ feed_name: Feed name for logging context.
390
+ identifiers: Representative identifiers for logging context.
367
391
 
368
392
  Returns:
369
- RawEquity: The USD-converted RawEquity if successful, otherwise the original
370
- source RawEquity.
393
+ RawEquity | None: USD-converted equity or None on failure.
371
394
  """
372
395
  converter = await get_usd_converter()
373
396
 
374
397
  try:
375
398
  converted = converter(validated)
376
399
 
377
- if converted is None:
378
- raise ValueError("USD conversion failed")
400
+ if converted is None or converted.currency != "USD":
401
+ raise ValueError(
402
+ f"USD conversion failed: {converted.currency if converted else None}",
403
+ )
404
+
405
+ _log_success(feed_name, identifiers, converted)
379
406
  return converted
380
407
 
381
- except Exception as error:
382
- _log_no_feed_data(
383
- feed_name,
384
- source,
385
- error,
386
- )
387
- return source
408
+ except Exception as e:
409
+ _log_failure(feed_name, identifiers, e)
410
+ return None
388
411
 
389
412
 
390
- def _log_no_feed_data(
413
+ def _log_success(
391
414
  feed_name: str,
392
- source: RawEquity,
393
- error: object,
394
- *,
395
- level: int = logging.DEBUG,
415
+ identifiers: EquityIdentifiers,
416
+ result: RawEquity,
396
417
  ) -> None:
397
418
  """
398
- Log a standardised message for missing or failed enrichment feed data.
399
-
400
- This logs details about the equity and the error context when no data is
401
- available from a feed, or when enrichment fails.
419
+ Log successful enrichment with representative identifiers.
402
420
 
403
421
  Args:
404
- feed_name (str): Name of the enrichment feed.
405
- source (RawEquity): Equity instance with identifying fields.
406
- error (object): Error or context for the missing data.
407
- level (int, optional): Logging level (default: logging.DEBUG).
422
+ feed_name: Name of the enrichment feed.
423
+ identifiers: Representative identifiers from discovery sources.
424
+ result: The enriched RawEquity returned by the feed.
425
+ """
426
+ prefix = f"[{feed_name}:{identifiers.symbol}]"
427
+
428
+ msg = (
429
+ f"{prefix:<24} SUCCESS: {feed_name} feed for symbol={identifiers.symbol}, "
430
+ f"name={identifiers.name} "
431
+ f"(share_class_figi={identifiers.share_class_figi or '<none>'}, "
432
+ f"isin={identifiers.isin or result.isin or '<none>'}, "
433
+ f"cusip={identifiers.cusip or result.cusip or '<none>'}, "
434
+ f"cik={identifiers.cik or result.cik or '<none>'}, "
435
+ f"lei={identifiers.lei or result.lei or '<none>'})"
436
+ )
408
437
 
409
- Returns:
410
- None
438
+ logger.debug(msg)
439
+
440
+
441
+ def _log_failure(
442
+ feed_name: str,
443
+ identifiers: EquityIdentifiers,
444
+ error: object,
445
+ ) -> None:
446
+ """
447
+ Log failed enrichment with the input identifiers that were attempted.
448
+
449
+ Args:
450
+ feed_name: Name of the enrichment feed.
451
+ identifiers: Representative identifiers that were used for the lookup.
452
+ error: Error or context describing why the lookup failed.
411
453
  """
412
- logger.log(
413
- level,
414
- "No %s feed data for symbol=%s, name=%s "
415
- "(isin=%s, cusip=%s, cik=%s, share_class_figi=%s). %s",
416
- feed_name,
417
- source.symbol,
418
- source.name,
419
- source.isin or "<none>",
420
- source.cusip or "<none>",
421
- source.cik or "<none>",
422
- source.share_class_figi or "<none>",
423
- error,
454
+ prefix = f"[{feed_name}:{identifiers.symbol}]"
455
+
456
+ msg = (
457
+ f"{prefix:<24} FAILURE: {feed_name} feed for symbol={identifiers.symbol}, "
458
+ f"name={identifiers.name} "
459
+ f"(share_class_figi={identifiers.share_class_figi or '<none>'}, "
460
+ f"isin={identifiers.isin or '<none>'}, "
461
+ f"cusip={identifiers.cusip or '<none>'}, "
462
+ f"cik={identifiers.cik or '<none>'}, "
463
+ f"lei={identifiers.lei or '<none>'}). "
464
+ f"{error}"
424
465
  )
466
+
467
+ logger.debug(msg)