equity-aggregator 0.1.1__py3-none-any.whl → 0.1.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- equity_aggregator/README.md +49 -39
- equity_aggregator/adapters/__init__.py +13 -7
- equity_aggregator/adapters/data_sources/__init__.py +4 -6
- equity_aggregator/adapters/data_sources/_utils/_client.py +1 -1
- equity_aggregator/adapters/data_sources/{authoritative_feeds → _utils}/_record_types.py +1 -1
- equity_aggregator/adapters/data_sources/discovery_feeds/__init__.py +17 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/__init__.py +10 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/backoff.py +33 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/_utils/parser.py +107 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/intrinio.py +305 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/session.py +197 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/__init__.py +9 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/backoff.py +33 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/parser.py +120 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/lseg.py +239 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/lseg/session.py +162 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/sec/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/sec}/sec.py +4 -5
- equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/stock_analysis.py +150 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/__init__.py +5 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/tradingview.py +275 -0
- equity_aggregator/adapters/data_sources/discovery_feeds/xetra/__init__.py +7 -0
- equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/xetra}/xetra.py +9 -12
- equity_aggregator/adapters/data_sources/enrichment_feeds/__init__.py +6 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/__init__.py +5 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/api.py +71 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/download.py +109 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/gleif.py +195 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/parser.py +75 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/__init__.py +1 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/_utils/__init__.py +11 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/{utils → _utils}/backoff.py +1 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/{utils → _utils}/fuzzy.py +28 -26
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/_utils/json.py +36 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/__init__.py +1 -1
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/{summary.py → quote_summary.py} +44 -30
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/api/search.py +10 -5
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/auth.py +130 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/config.py +3 -3
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/ranking.py +97 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/session.py +85 -218
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/transport.py +191 -0
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/yfinance.py +413 -0
- equity_aggregator/adapters/data_sources/reference_lookup/exchange_rate_api.py +6 -13
- equity_aggregator/adapters/data_sources/reference_lookup/openfigi.py +23 -7
- equity_aggregator/cli/dispatcher.py +11 -8
- equity_aggregator/cli/main.py +14 -5
- equity_aggregator/cli/parser.py +1 -1
- equity_aggregator/cli/signals.py +32 -0
- equity_aggregator/domain/_utils/__init__.py +2 -2
- equity_aggregator/domain/_utils/_load_converter.py +30 -21
- equity_aggregator/domain/_utils/_merge.py +221 -368
- equity_aggregator/domain/_utils/_merge_config.py +205 -0
- equity_aggregator/domain/_utils/_strategies.py +180 -0
- equity_aggregator/domain/pipeline/resolve.py +17 -11
- equity_aggregator/domain/pipeline/runner.py +4 -4
- equity_aggregator/domain/pipeline/seed.py +5 -1
- equity_aggregator/domain/pipeline/transforms/__init__.py +2 -2
- equity_aggregator/domain/pipeline/transforms/canonicalise.py +1 -1
- equity_aggregator/domain/pipeline/transforms/enrich.py +328 -285
- equity_aggregator/domain/pipeline/transforms/group.py +48 -0
- equity_aggregator/logging_config.py +4 -1
- equity_aggregator/schemas/__init__.py +11 -5
- equity_aggregator/schemas/canonical.py +11 -6
- equity_aggregator/schemas/feeds/__init__.py +11 -5
- equity_aggregator/schemas/feeds/gleif_feed_data.py +35 -0
- equity_aggregator/schemas/feeds/intrinio_feed_data.py +142 -0
- equity_aggregator/schemas/feeds/{lse_feed_data.py → lseg_feed_data.py} +85 -52
- equity_aggregator/schemas/feeds/sec_feed_data.py +36 -6
- equity_aggregator/schemas/feeds/stock_analysis_feed_data.py +107 -0
- equity_aggregator/schemas/feeds/tradingview_feed_data.py +144 -0
- equity_aggregator/schemas/feeds/xetra_feed_data.py +1 -1
- equity_aggregator/schemas/feeds/yfinance_feed_data.py +47 -35
- equity_aggregator/schemas/raw.py +5 -3
- equity_aggregator/schemas/types.py +7 -0
- equity_aggregator/schemas/validators.py +81 -27
- equity_aggregator/storage/data_store.py +5 -3
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.5.dist-info}/METADATA +205 -115
- equity_aggregator-0.1.5.dist-info/RECORD +103 -0
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.5.dist-info}/WHEEL +1 -1
- equity_aggregator/adapters/data_sources/authoritative_feeds/__init__.py +0 -13
- equity_aggregator/adapters/data_sources/authoritative_feeds/euronext.py +0 -420
- equity_aggregator/adapters/data_sources/authoritative_feeds/lse.py +0 -352
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/feed.py +0 -350
- equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/utils/__init__.py +0 -9
- equity_aggregator/domain/pipeline/transforms/deduplicate.py +0 -54
- equity_aggregator/schemas/feeds/euronext_feed_data.py +0 -59
- equity_aggregator-0.1.1.dist-info/RECORD +0 -72
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.5.dist-info}/entry_points.txt +0 -0
- {equity_aggregator-0.1.1.dist-info → equity_aggregator-0.1.5.dist-info}/licenses/LICENCE.txt +0 -0
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
# transforms/group.py
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
from collections.abc import AsyncIterable
|
|
5
|
+
from itertools import groupby
|
|
6
|
+
|
|
7
|
+
from equity_aggregator.schemas.raw import RawEquity
|
|
8
|
+
|
|
9
|
+
logger = logging.getLogger(__name__)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
async def group(
|
|
13
|
+
raw_equities: AsyncIterable[RawEquity],
|
|
14
|
+
) -> AsyncIterable[list[RawEquity]]:
|
|
15
|
+
"""
|
|
16
|
+
Group equities by share_class_figi, preserving all source data points.
|
|
17
|
+
|
|
18
|
+
Groups equities sharing the same share_class_figi identifier into lists,
|
|
19
|
+
preserving all discovery feed sources. This allows the enrichment stage
|
|
20
|
+
to compute median identifiers and perform a single merge of all sources
|
|
21
|
+
(discovery + enrichment) for optimal data quality.
|
|
22
|
+
|
|
23
|
+
Args:
|
|
24
|
+
raw_equities: Async iterable of RawEquity records to group.
|
|
25
|
+
|
|
26
|
+
Yields:
|
|
27
|
+
list[RawEquity]: Groups of equities with the same share_class_figi.
|
|
28
|
+
"""
|
|
29
|
+
aggregated_raw_equities = [raw_equity async for raw_equity in raw_equities]
|
|
30
|
+
|
|
31
|
+
total = len(aggregated_raw_equities)
|
|
32
|
+
unique = len({equity.share_class_figi for equity in aggregated_raw_equities})
|
|
33
|
+
duplicates = total - unique
|
|
34
|
+
|
|
35
|
+
logger.info(
|
|
36
|
+
"Grouped %d raw equities with %d duplicates into %d unique groups",
|
|
37
|
+
total,
|
|
38
|
+
duplicates,
|
|
39
|
+
unique,
|
|
40
|
+
)
|
|
41
|
+
|
|
42
|
+
aggregated_raw_equities.sort(key=lambda equity: equity.share_class_figi)
|
|
43
|
+
|
|
44
|
+
for _, equity_group in groupby(
|
|
45
|
+
aggregated_raw_equities,
|
|
46
|
+
key=lambda equity: equity.share_class_figi,
|
|
47
|
+
):
|
|
48
|
+
yield list(equity_group)
|
|
@@ -27,7 +27,10 @@ LOGGING = {
|
|
|
27
27
|
},
|
|
28
28
|
"formatters": {
|
|
29
29
|
"standard": {
|
|
30
|
-
"format":
|
|
30
|
+
"format": (
|
|
31
|
+
"%(asctime)s | %(module)-20s | %(levelname)-5s | "
|
|
32
|
+
"%(taskName)-12s | %(message)s"
|
|
33
|
+
),
|
|
31
34
|
"datefmt": "%Y-%m-%dT%H:%M:%S",
|
|
32
35
|
},
|
|
33
36
|
},
|
|
@@ -2,9 +2,12 @@
|
|
|
2
2
|
|
|
3
3
|
from .canonical import CanonicalEquity, EquityFinancials, EquityIdentity
|
|
4
4
|
from .feeds import (
|
|
5
|
-
|
|
6
|
-
|
|
5
|
+
GleifFeedData,
|
|
6
|
+
IntrinioFeedData,
|
|
7
|
+
LsegFeedData,
|
|
7
8
|
SecFeedData,
|
|
9
|
+
StockAnalysisFeedData,
|
|
10
|
+
TradingViewFeedData,
|
|
8
11
|
XetraFeedData,
|
|
9
12
|
YFinanceFeedData,
|
|
10
13
|
)
|
|
@@ -14,11 +17,14 @@ __all__ = [
|
|
|
14
17
|
"EquityFinancials",
|
|
15
18
|
"EquityIdentity",
|
|
16
19
|
"CanonicalEquity",
|
|
17
|
-
#
|
|
18
|
-
"
|
|
19
|
-
"
|
|
20
|
+
# discovery feeds
|
|
21
|
+
"IntrinioFeedData",
|
|
22
|
+
"LsegFeedData",
|
|
20
23
|
"SecFeedData",
|
|
24
|
+
"StockAnalysisFeedData",
|
|
25
|
+
"TradingViewFeedData",
|
|
21
26
|
"XetraFeedData",
|
|
22
27
|
# enrichment feeds
|
|
28
|
+
"GleifFeedData",
|
|
23
29
|
"YFinanceFeedData",
|
|
24
30
|
]
|
|
@@ -14,6 +14,7 @@ from .types import (
|
|
|
14
14
|
CUSIPStrOpt,
|
|
15
15
|
FIGIStrReq,
|
|
16
16
|
ISINStrOpt,
|
|
17
|
+
LEIStrOpt,
|
|
17
18
|
MICListOpt,
|
|
18
19
|
SignedDecOpt,
|
|
19
20
|
UnsignedDecOpt,
|
|
@@ -27,8 +28,9 @@ class EquityIdentity(BaseModel):
|
|
|
27
28
|
"""
|
|
28
29
|
Globally unique identity metadata for a single equity record.
|
|
29
30
|
|
|
30
|
-
The
|
|
31
|
-
the equity. Other local identifiers such as ISIN, CUSIP or
|
|
31
|
+
The definitive identifier is `share_class_figi`, which uniquely distinguishes
|
|
32
|
+
the equity. Other local identifiers such as ISIN, CUSIP, CIK or LEI may also
|
|
33
|
+
be present.
|
|
32
34
|
|
|
33
35
|
Attributes:
|
|
34
36
|
name (UpperStrReq): Full name of the equity.
|
|
@@ -37,14 +39,16 @@ class EquityIdentity(BaseModel):
|
|
|
37
39
|
isin (ISINStrOpt): Optional International Securities Identification Number.
|
|
38
40
|
cusip (CUSIPStrOpt): Optional CUSIP identifier.
|
|
39
41
|
cik (CIKStrOpt): Optional Central Index Key for SEC filings.
|
|
42
|
+
lei (LEIStrOpt): Optional Legal Entity Identifier (ISO 17442).
|
|
40
43
|
|
|
41
44
|
Args:
|
|
42
45
|
name (UpperStrReq): Full name of the equity, required.
|
|
43
46
|
symbol (UpperStrReq): Trading symbol, required.
|
|
44
|
-
share_class_figi (FIGIStrReq):
|
|
47
|
+
share_class_figi (FIGIStrReq): Definitive OpenFIGI identifier, required.
|
|
45
48
|
isin (ISINStrOpt): ISIN code, if available.
|
|
46
49
|
cusip (CUSIPStrOpt): CUSIP code, if available.
|
|
47
50
|
cik (CIKStrOpt): CIK code, if available.
|
|
51
|
+
lei (LEIStrOpt): LEI code, if available.
|
|
48
52
|
|
|
49
53
|
Returns:
|
|
50
54
|
EquityIdentity: Immutable identity record for the equity.
|
|
@@ -56,7 +60,7 @@ class EquityIdentity(BaseModel):
|
|
|
56
60
|
name: UpperStrReq = Field(..., description="Equity name, required.")
|
|
57
61
|
symbol: UpperStrReq = Field(..., description="Equity symbol, required.")
|
|
58
62
|
|
|
59
|
-
# required share_class_figi is the
|
|
63
|
+
# required share_class_figi is the definitive id uniquely identifying an equity.
|
|
60
64
|
share_class_figi: FIGIStrReq = Field(
|
|
61
65
|
...,
|
|
62
66
|
description="Equity share class FIGI, required.",
|
|
@@ -66,6 +70,7 @@ class EquityIdentity(BaseModel):
|
|
|
66
70
|
isin: ISINStrOpt = None
|
|
67
71
|
cusip: CUSIPStrOpt = None
|
|
68
72
|
cik: CIKStrOpt = None
|
|
73
|
+
lei: LEIStrOpt = None
|
|
69
74
|
|
|
70
75
|
|
|
71
76
|
# ─────────────────────── Supplementary financial data ──────────────────────
|
|
@@ -104,7 +109,7 @@ class EquityFinancials(BaseModel):
|
|
|
104
109
|
short_interest: UnsignedDecOpt = None
|
|
105
110
|
share_float: UnsignedDecOpt = None
|
|
106
111
|
shares_outstanding: UnsignedDecOpt = None
|
|
107
|
-
revenue_per_share:
|
|
112
|
+
revenue_per_share: SignedDecOpt = None
|
|
108
113
|
profit_margin: SignedDecOpt = None
|
|
109
114
|
gross_margin: SignedDecOpt = None
|
|
110
115
|
operating_margin: SignedDecOpt = None
|
|
@@ -114,7 +119,7 @@ class EquityFinancials(BaseModel):
|
|
|
114
119
|
return_on_assets: SignedDecOpt = None
|
|
115
120
|
performance_1_year: SignedDecOpt = None
|
|
116
121
|
total_debt: UnsignedDecOpt = None
|
|
117
|
-
revenue:
|
|
122
|
+
revenue: SignedDecOpt = None
|
|
118
123
|
ebitda: SignedDecOpt = None
|
|
119
124
|
trailing_pe: SignedDecOpt = None
|
|
120
125
|
price_to_book: SignedDecOpt = None
|
|
@@ -1,17 +1,23 @@
|
|
|
1
1
|
# feeds/__init__.py
|
|
2
2
|
|
|
3
|
-
from .
|
|
4
|
-
from .
|
|
3
|
+
from .gleif_feed_data import GleifFeedData
|
|
4
|
+
from .intrinio_feed_data import IntrinioFeedData
|
|
5
|
+
from .lseg_feed_data import LsegFeedData
|
|
5
6
|
from .sec_feed_data import SecFeedData
|
|
7
|
+
from .stock_analysis_feed_data import StockAnalysisFeedData
|
|
8
|
+
from .tradingview_feed_data import TradingViewFeedData
|
|
6
9
|
from .xetra_feed_data import XetraFeedData
|
|
7
10
|
from .yfinance_feed_data import YFinanceFeedData
|
|
8
11
|
|
|
9
12
|
__all__ = [
|
|
10
|
-
#
|
|
11
|
-
"
|
|
12
|
-
"
|
|
13
|
+
# discovery feeds
|
|
14
|
+
"IntrinioFeedData",
|
|
15
|
+
"LsegFeedData",
|
|
13
16
|
"SecFeedData",
|
|
17
|
+
"StockAnalysisFeedData",
|
|
18
|
+
"TradingViewFeedData",
|
|
14
19
|
"XetraFeedData",
|
|
15
20
|
# enrichment feeds
|
|
21
|
+
"GleifFeedData",
|
|
16
22
|
"YFinanceFeedData",
|
|
17
23
|
]
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# feeds/gleif_feed_data.py
|
|
2
|
+
|
|
3
|
+
from pydantic import BaseModel, ConfigDict
|
|
4
|
+
|
|
5
|
+
from .feed_validators import required
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@required("name", "symbol")
|
|
9
|
+
class GleifFeedData(BaseModel):
|
|
10
|
+
"""
|
|
11
|
+
GleifFeedData represents a single record from the GLEIF ISIN->LEI mapping feed.
|
|
12
|
+
|
|
13
|
+
This is a minimal enrichment feed that provides LEI (Legal Entity Identifier)
|
|
14
|
+
data based on ISIN lookups. The name and symbol fields are passed through from
|
|
15
|
+
the source equity to satisfy RawEquity requirements.
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
name (str): The equity name (passed through from source).
|
|
19
|
+
symbol (str): The equity symbol (passed through from source).
|
|
20
|
+
isin (str | None): The ISIN used for the LEI lookup.
|
|
21
|
+
lei (str | None): The Legal Entity Identifier from GLEIF mapping.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
# Fields exactly match RawEquity's signature
|
|
25
|
+
name: str
|
|
26
|
+
symbol: str
|
|
27
|
+
isin: str | None = None
|
|
28
|
+
lei: str | None = None
|
|
29
|
+
|
|
30
|
+
model_config = ConfigDict(
|
|
31
|
+
# ignore extra fields in incoming GLEIF raw data feed
|
|
32
|
+
extra="ignore",
|
|
33
|
+
# defer strict type validation to RawEquity
|
|
34
|
+
strict=False,
|
|
35
|
+
)
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
# feeds/intrinio_feed_data.py
|
|
2
|
+
|
|
3
|
+
from decimal import Decimal
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, model_validator
|
|
6
|
+
|
|
7
|
+
from .feed_validators import required
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@required("name", "symbol")
|
|
11
|
+
class IntrinioFeedData(BaseModel):
|
|
12
|
+
"""
|
|
13
|
+
Represents a single Intrinio feed record, transforming and normalising
|
|
14
|
+
incoming fields to match the RawEquity model's expected attributes.
|
|
15
|
+
|
|
16
|
+
Combines company metadata from the /companies endpoint, security identifiers from
|
|
17
|
+
the /companies/{ticker}/securities endpoint, and quote data from the
|
|
18
|
+
/securities/{share_class_figi}/quote endpoint.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
name (str): Company name.
|
|
22
|
+
symbol (str): Equity symbol, mapped from security "ticker".
|
|
23
|
+
cik (str | None): Central Index Key (10-digit zero-padded string from API).
|
|
24
|
+
lei (str | None): Legal Entity Identifier from company data.
|
|
25
|
+
share_class_figi (str | None): Share class FIGI identifier.
|
|
26
|
+
mics (list[str] | None): Market Identifier Codes from exchange_mic.
|
|
27
|
+
currency (str | None): The trading currency.
|
|
28
|
+
last_price (str | float | Decimal | None): Last traded price.
|
|
29
|
+
fifty_two_week_min (str | float | Decimal | None): 52-week low price.
|
|
30
|
+
fifty_two_week_max (str | float | Decimal | None): 52-week high price.
|
|
31
|
+
market_volume (str | float | Decimal | None): Latest trading volume.
|
|
32
|
+
dividend_yield (str | float | Decimal | None): Annual dividend yield.
|
|
33
|
+
market_cap (str | float | Decimal | None): Market capitalisation.
|
|
34
|
+
performance_1_year (str | float | Decimal | None): 1-year performance.
|
|
35
|
+
|
|
36
|
+
Returns:
|
|
37
|
+
IntrinioFeedData: An instance with fields normalised for RawEquity
|
|
38
|
+
validation.
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
# Fields match RawEquity's signature
|
|
42
|
+
name: str
|
|
43
|
+
symbol: str
|
|
44
|
+
cik: str | None = None
|
|
45
|
+
lei: str | None = None
|
|
46
|
+
share_class_figi: str | None = None
|
|
47
|
+
mics: list[str] | None = None
|
|
48
|
+
currency: str | None = None
|
|
49
|
+
last_price: str | float | Decimal | None = None
|
|
50
|
+
fifty_two_week_min: str | float | Decimal | None = None
|
|
51
|
+
fifty_two_week_max: str | float | Decimal | None = None
|
|
52
|
+
market_volume: str | float | Decimal | None = None
|
|
53
|
+
dividend_yield: str | float | Decimal | None = None
|
|
54
|
+
market_cap: str | float | Decimal | None = None
|
|
55
|
+
performance_1_year: str | float | Decimal | None = None
|
|
56
|
+
|
|
57
|
+
@model_validator(mode="before")
|
|
58
|
+
def _normalise_fields(self: dict[str, object]) -> dict[str, object]:
|
|
59
|
+
"""
|
|
60
|
+
Normalise a raw Intrinio feed record into the flat schema expected
|
|
61
|
+
by RawEquity.
|
|
62
|
+
|
|
63
|
+
Combines company data, security data, and quote data, mapping fields to
|
|
64
|
+
RawEquity attributes.
|
|
65
|
+
|
|
66
|
+
Args:
|
|
67
|
+
self (dict[str, object]): Raw payload containing Intrinio feed
|
|
68
|
+
data with quote data.
|
|
69
|
+
|
|
70
|
+
Returns:
|
|
71
|
+
dict[str, object]: A new dictionary with renamed keys suitable for the
|
|
72
|
+
RawEquity schema.
|
|
73
|
+
"""
|
|
74
|
+
# Extract quote data if present
|
|
75
|
+
quote = self.get("quote", {}) or {}
|
|
76
|
+
|
|
77
|
+
# Build MICs list from exchange_mic if present
|
|
78
|
+
exchange_mic = self.get("exchange_mic")
|
|
79
|
+
mics = [exchange_mic] if exchange_mic else None
|
|
80
|
+
|
|
81
|
+
return {
|
|
82
|
+
# name → RawEquity.name
|
|
83
|
+
"name": self.get("name"),
|
|
84
|
+
# cik → RawEquity.cik
|
|
85
|
+
"cik": self.get("cik"),
|
|
86
|
+
# lei → RawEquity.lei
|
|
87
|
+
"lei": self.get("lei"),
|
|
88
|
+
# ticker → RawEquity.symbol
|
|
89
|
+
"symbol": self.get("ticker"),
|
|
90
|
+
# share_class_figi → RawEquity.share_class_figi
|
|
91
|
+
"share_class_figi": self.get("share_class_figi"),
|
|
92
|
+
# exchange_mic → RawEquity.mics
|
|
93
|
+
"mics": mics,
|
|
94
|
+
# security.currency → RawEquity.currency
|
|
95
|
+
"currency": self.get("currency"),
|
|
96
|
+
# last → RawEquity.last_price
|
|
97
|
+
"last_price": quote.get("last"),
|
|
98
|
+
# eod_fifty_two_week_low → RawEquity.fifty_two_week_min
|
|
99
|
+
"fifty_two_week_min": quote.get("eod_fifty_two_week_low"),
|
|
100
|
+
# eod_fifty_two_week_high → RawEquity.fifty_two_week_max
|
|
101
|
+
"fifty_two_week_max": quote.get("eod_fifty_two_week_high"),
|
|
102
|
+
# market_volume → RawEquity.market_volume
|
|
103
|
+
"market_volume": quote.get("market_volume"),
|
|
104
|
+
# dividendyield → RawEquity.dividend_yield
|
|
105
|
+
"dividend_yield": quote.get("dividendyield"),
|
|
106
|
+
# marketcap → RawEquity.market_cap
|
|
107
|
+
"market_cap": quote.get("marketcap"),
|
|
108
|
+
# change_percent_365_days → RawEquity.performance_1_year
|
|
109
|
+
# Convert from percentage (e.g., 14.6572) to decimal (0.146572)
|
|
110
|
+
"performance_1_year": _percent_to_decimal(
|
|
111
|
+
quote.get("change_percent_365_days"),
|
|
112
|
+
),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
model_config = ConfigDict(
|
|
116
|
+
# ignore extra fields in incoming Intrinio raw data feed
|
|
117
|
+
extra="ignore",
|
|
118
|
+
# defer strict type validation to RawEquity
|
|
119
|
+
strict=False,
|
|
120
|
+
)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _percent_to_decimal(percent: str | float | None) -> Decimal | None:
|
|
124
|
+
"""
|
|
125
|
+
Convert a percentage value to decimal format.
|
|
126
|
+
|
|
127
|
+
Converts percentage values (e.g., 14.6572 representing 14.6572%) to decimal
|
|
128
|
+
format (0.146572) for consistency with RawEquity's performance_1_year field.
|
|
129
|
+
|
|
130
|
+
Args:
|
|
131
|
+
percent (str | float | None): The percentage value to convert.
|
|
132
|
+
|
|
133
|
+
Returns:
|
|
134
|
+
Decimal | None: The decimal value, or None if input is None or invalid.
|
|
135
|
+
"""
|
|
136
|
+
if percent is None:
|
|
137
|
+
return None
|
|
138
|
+
|
|
139
|
+
try:
|
|
140
|
+
return Decimal(str(percent)) / Decimal("100")
|
|
141
|
+
except (ValueError, TypeError):
|
|
142
|
+
return None
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
# feeds/
|
|
1
|
+
# feeds/lseg_feed_data.py
|
|
2
2
|
|
|
3
3
|
import re
|
|
4
4
|
from decimal import Decimal
|
|
@@ -9,27 +9,30 @@ from .feed_validators import required
|
|
|
9
9
|
|
|
10
10
|
|
|
11
11
|
@required("name", "symbol")
|
|
12
|
-
class
|
|
12
|
+
class LsegFeedData(BaseModel):
|
|
13
13
|
"""
|
|
14
|
-
Represents
|
|
15
|
-
to match the RawEquity model's expected attributes. If the currency is "GBX",
|
|
14
|
+
Represents single LSEG feed record, transforming and normalising incoming
|
|
15
|
+
fields to match the RawEquity model's expected attributes. If the currency is "GBX",
|
|
16
16
|
price fields such as "last_price" are automatically converted from pence to
|
|
17
17
|
pounds (GBP) for consistency.
|
|
18
18
|
|
|
19
19
|
Args:
|
|
20
20
|
name (str): The issuer's full name, mapped from "issuername".
|
|
21
|
-
symbol (str): The tradable instrument
|
|
21
|
+
symbol (str): The tradable instrument symbol, mapped from "tidm".
|
|
22
22
|
isin (str | None): The ISIN identifier, if available.
|
|
23
|
-
mics (list[str]): List of MIC codes for trading venues; defaults to ["XLON"].
|
|
24
23
|
currency (str | None): The trading currency code, with "GBX" converted to
|
|
25
24
|
"GBP" if applicable.
|
|
26
25
|
last_price (str | float | int | Decimal | None): Last traded price, mapped
|
|
27
26
|
from "lastprice" and converted from pence to pounds if currency is "GBX".
|
|
28
|
-
market_cap (str | float | int | Decimal | None): Market capitalisation,
|
|
29
|
-
from "marketcapitalization".
|
|
27
|
+
market_cap (str | float | int | Decimal | None): Market capitalisation,
|
|
28
|
+
mapped from "marketcapitalization".
|
|
29
|
+
fifty_two_week_min (str | float | int | Decimal | None): 52-week low,
|
|
30
|
+
mapped from "fiftyTwoWeeksMin" (converted from pence to GBP if needed).
|
|
31
|
+
fifty_two_week_max (str | float | int | Decimal | None): 52-week high,
|
|
32
|
+
mapped from "fiftyTwoWeeksMax" (converted from pence to GBP if needed).
|
|
30
33
|
|
|
31
34
|
Returns:
|
|
32
|
-
|
|
35
|
+
LsegFeedData: An instance with fields normalised for RawEquity validation,
|
|
33
36
|
including automatic GBX to GBP conversion where relevant.
|
|
34
37
|
"""
|
|
35
38
|
|
|
@@ -37,61 +40,97 @@ class LseFeedData(BaseModel):
|
|
|
37
40
|
name: str
|
|
38
41
|
symbol: str
|
|
39
42
|
isin: str | None
|
|
40
|
-
mics: list[str]
|
|
41
43
|
currency: str | None
|
|
42
44
|
last_price: str | float | int | Decimal | None
|
|
43
45
|
market_cap: str | float | int | Decimal | None
|
|
44
|
-
fifty_two_week_min: str | float | int | Decimal | None
|
|
45
|
-
fifty_two_week_max: str | float | int | Decimal | None
|
|
46
|
+
fifty_two_week_min: str | float | int | Decimal | None
|
|
47
|
+
fifty_two_week_max: str | float | int | Decimal | None
|
|
46
48
|
|
|
47
49
|
@model_validator(mode="before")
|
|
48
50
|
def _normalise_fields(self: dict[str, object]) -> dict[str, object]:
|
|
49
51
|
"""
|
|
50
|
-
Normalise
|
|
52
|
+
Normalise raw LSEG feed record into the flat schema expected by RawEquity.
|
|
51
53
|
|
|
52
54
|
Extracts and renames nested fields to match the RawEquity signature. If the
|
|
53
55
|
currency is "GBX", automatically converts price fields from pence to pounds
|
|
54
|
-
(GBP) using the convert_gbx_to_gbp helper.
|
|
56
|
+
(GBP) using the convert_gbx_to_gbp helper. Treats 0 as None for monetary
|
|
57
|
+
fields since LSEG API uses 0 to represent missing data for certain fields.
|
|
55
58
|
|
|
56
59
|
Args:
|
|
57
|
-
self (dict[str, object]): Raw payload containing raw
|
|
60
|
+
self (dict[str, object]): Raw payload containing raw LSEG feed data.
|
|
58
61
|
|
|
59
62
|
Returns:
|
|
60
63
|
dict[str, object]: A new dictionary with renamed keys and, if applicable,
|
|
61
64
|
price and currency fields converted from GBX to GBP, suitable for the
|
|
62
65
|
RawEquity schema.
|
|
63
66
|
"""
|
|
64
|
-
|
|
65
|
-
|
|
67
|
+
|
|
68
|
+
# convert GBX to GBP and sanitise zero monetary values
|
|
69
|
+
raw = _convert_gbx_to_gbp(self)
|
|
70
|
+
raw = _sanitise_zero_monetary_values(raw)
|
|
71
|
+
|
|
66
72
|
return {
|
|
67
|
-
# issuername → maps to RawEquity.name
|
|
68
73
|
"name": raw.get("issuername"),
|
|
69
|
-
# tidm → maps to RawEquity.symbol
|
|
70
74
|
"symbol": raw.get("tidm"),
|
|
71
75
|
"isin": raw.get("isin"),
|
|
72
|
-
# no CUSIP, CIK or FIGI in
|
|
73
|
-
# default to XLON if mic not provided
|
|
74
|
-
"mics": raw.get("mics") or ["XLON"],
|
|
76
|
+
# no CUSIP, CIK or FIGI in LSEG feed, so omitting from model
|
|
75
77
|
"currency": raw.get("currency"),
|
|
76
|
-
# lastprice → maps to RawEquity.last_price
|
|
77
78
|
"last_price": raw.get("lastprice"),
|
|
78
|
-
# marketcapitalization → maps to RawEquity.market_cap
|
|
79
79
|
"market_cap": raw.get("marketcapitalization"),
|
|
80
|
-
# fiftyTwoWeeksMin → maps to RawEquity.fifty_two_week_min
|
|
81
80
|
"fifty_two_week_min": raw.get("fiftyTwoWeeksMin"),
|
|
82
|
-
# fiftyTwoWeeksMax → maps to RawEquity.fifty_two_week_max
|
|
83
81
|
"fifty_two_week_max": raw.get("fiftyTwoWeeksMax"),
|
|
84
|
-
# no additional fields in
|
|
82
|
+
# no additional fields in LSEG feed, so omitting from model
|
|
85
83
|
}
|
|
86
84
|
|
|
87
85
|
model_config = ConfigDict(
|
|
88
|
-
# ignore extra fields in incoming
|
|
86
|
+
# ignore extra fields in incoming LSEG raw data feed
|
|
89
87
|
extra="ignore",
|
|
90
88
|
# defer strict type validation to RawEquity
|
|
91
89
|
strict=False,
|
|
92
90
|
)
|
|
93
91
|
|
|
94
92
|
|
|
93
|
+
def _convert_gbx_to_gbp(raw: dict) -> dict:
|
|
94
|
+
"""
|
|
95
|
+
Convert price and currency fields from GBX (pence) to GBP (pounds).
|
|
96
|
+
|
|
97
|
+
If the input dictionary has a "currency" field set to "GBX", this function
|
|
98
|
+
divides all price fields (lastprice, fiftyTwoWeeksMin, fiftyTwoWeeksMax)
|
|
99
|
+
by 100 to convert from pence to pounds, sets the "currency" field to "GBP",
|
|
100
|
+
and returns a new dictionary with these updates. All other fields remain
|
|
101
|
+
unchanged. If the currency is not "GBX", the original dictionary is returned
|
|
102
|
+
unmodified.
|
|
103
|
+
|
|
104
|
+
Args:
|
|
105
|
+
raw (dict): A dictionary containing at least a "currency" field, and
|
|
106
|
+
optionally price fields representing values in pence.
|
|
107
|
+
|
|
108
|
+
Returns:
|
|
109
|
+
dict: A new dictionary with price fields converted to pounds and
|
|
110
|
+
"currency" set to "GBP" if original currency was "GBX". Otherwise,
|
|
111
|
+
returns original dict.
|
|
112
|
+
"""
|
|
113
|
+
if raw.get("currency") != "GBX":
|
|
114
|
+
return raw
|
|
115
|
+
|
|
116
|
+
updates = {"currency": "GBP"}
|
|
117
|
+
|
|
118
|
+
# Convert lastprice
|
|
119
|
+
lastprice = _gbx_to_decimal(raw.get("lastprice"))
|
|
120
|
+
updates["lastprice"] = lastprice / Decimal("100") if lastprice else None
|
|
121
|
+
|
|
122
|
+
# Convert fiftyTwoWeeksMin
|
|
123
|
+
min_price = _gbx_to_decimal(raw.get("fiftyTwoWeeksMin"))
|
|
124
|
+
updates["fiftyTwoWeeksMin"] = min_price / Decimal("100") if min_price else None
|
|
125
|
+
|
|
126
|
+
# Convert fiftyTwoWeeksMax
|
|
127
|
+
max_price = _gbx_to_decimal(raw.get("fiftyTwoWeeksMax"))
|
|
128
|
+
updates["fiftyTwoWeeksMax"] = max_price / Decimal("100") if max_price else None
|
|
129
|
+
|
|
130
|
+
# return a new dict rather than mutating in place
|
|
131
|
+
return {**raw, **updates}
|
|
132
|
+
|
|
133
|
+
|
|
95
134
|
def _gbx_to_decimal(pence: str | None) -> Decimal | None:
|
|
96
135
|
"""
|
|
97
136
|
Convert a pence string (e.g., "150", "1,50") to a Decimal value.
|
|
@@ -122,36 +161,30 @@ def _gbx_to_decimal(pence: str | None) -> Decimal | None:
|
|
|
122
161
|
return Decimal(s)
|
|
123
162
|
|
|
124
163
|
|
|
125
|
-
def
|
|
164
|
+
def _sanitise_zero_monetary_values(raw: dict) -> dict:
|
|
126
165
|
"""
|
|
127
|
-
|
|
166
|
+
Treat 0 as None for LSEG monetary fields.
|
|
128
167
|
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
remain unchanged. If the currency is not "GBX", the original dictionary is returned
|
|
133
|
-
unmodified.
|
|
168
|
+
LSEG API returns 0 for missing monetary data. Since 0 is not a valid
|
|
169
|
+
price or market cap, we treat it as None to allow enrichment feeds
|
|
170
|
+
to provide valid data downstream.
|
|
134
171
|
|
|
135
172
|
Args:
|
|
136
|
-
raw
|
|
137
|
-
a "lastprice" field representing the price in pence.
|
|
173
|
+
raw: Dictionary containing LSEG API fields.
|
|
138
174
|
|
|
139
175
|
Returns:
|
|
140
|
-
|
|
141
|
-
to "GBP" if original currency was "GBX". Otherwise, returns original dict.
|
|
176
|
+
A new dictionary with 0 values converted to None for monetary fields.
|
|
142
177
|
"""
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
178
|
+
monetary_fields = [
|
|
179
|
+
"lastprice",
|
|
180
|
+
"marketcapitalization",
|
|
181
|
+
"fiftyTwoWeeksMin",
|
|
182
|
+
"fiftyTwoWeeksMax",
|
|
183
|
+
]
|
|
184
|
+
|
|
185
|
+
updates = {
|
|
186
|
+
field: None if raw.get(field) == 0 else raw.get(field)
|
|
187
|
+
for field in monetary_fields
|
|
188
|
+
}
|
|
148
189
|
|
|
149
|
-
updates = {"currency": "GBP"}
|
|
150
|
-
if amount is None:
|
|
151
|
-
updates["lastprice"] = None
|
|
152
|
-
else:
|
|
153
|
-
# convert pence to pounds
|
|
154
|
-
updates["lastprice"] = amount / Decimal("100")
|
|
155
|
-
|
|
156
|
-
# return a new dict rather than mutating in place
|
|
157
190
|
return {**raw, **updates}
|
|
@@ -14,13 +14,15 @@ class SecFeedData(BaseModel):
|
|
|
14
14
|
Args:
|
|
15
15
|
name (str): Company name, mapped from "name".
|
|
16
16
|
symbol (str): Equity symbol, mapped from "symbol".
|
|
17
|
+
cik (str): Central Index Key, converted from int to 10-digit zero-padded string.
|
|
17
18
|
mics (list[str]): List of MIC codes; defaults to an empty list if missing.
|
|
18
19
|
|
|
19
20
|
Returns:
|
|
20
|
-
|
|
21
|
+
SecFeedData: An instance with fields normalised for RawEquity validation.
|
|
21
22
|
"""
|
|
22
23
|
|
|
23
|
-
# Fields exactly match RawEquity
|
|
24
|
+
# Fields exactly match RawEquity's signature
|
|
25
|
+
cik: str
|
|
24
26
|
name: str
|
|
25
27
|
symbol: str
|
|
26
28
|
mics: list[str]
|
|
@@ -37,12 +39,15 @@ class SecFeedData(BaseModel):
|
|
|
37
39
|
dict[str, object]: A new dictionary with renamed keys suitable for the
|
|
38
40
|
RawEquity schema.
|
|
39
41
|
"""
|
|
42
|
+
# convert int CIK to string
|
|
43
|
+
raw = convert_cik_to_str(self)
|
|
44
|
+
|
|
40
45
|
return {
|
|
41
|
-
"
|
|
42
|
-
"
|
|
43
|
-
"
|
|
46
|
+
"cik": raw.get("cik"),
|
|
47
|
+
"name": raw.get("name"),
|
|
48
|
+
"symbol": raw.get("symbol"),
|
|
44
49
|
# no CUSIP, ISIN or FIGI in SEC feed, so omitting from model
|
|
45
|
-
"mics":
|
|
50
|
+
"mics": raw.get("mics"),
|
|
46
51
|
# no currency or last_price in SEC feed, so omitting from model
|
|
47
52
|
# no more additional fields in SEC feed, so omitting from model
|
|
48
53
|
}
|
|
@@ -53,3 +58,28 @@ class SecFeedData(BaseModel):
|
|
|
53
58
|
# defer strict type validation to RawEquity
|
|
54
59
|
strict=False,
|
|
55
60
|
)
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def convert_cik_to_str(raw: dict) -> dict:
|
|
64
|
+
"""
|
|
65
|
+
Normalise SEC CIK integer value to ensure compatibility with RawEquity schema.
|
|
66
|
+
|
|
67
|
+
The SEC API returns CIK values as integers, but the RawEquity schema expects
|
|
68
|
+
10-digit zero-padded string values for all CIK fields. This function converts
|
|
69
|
+
integer CIK values to properly formatted strings while preserving all other
|
|
70
|
+
fields unchanged.
|
|
71
|
+
|
|
72
|
+
Args:
|
|
73
|
+
raw (dict): A dictionary containing raw SEC feed data with potentially
|
|
74
|
+
integer CIK values.
|
|
75
|
+
|
|
76
|
+
Returns:
|
|
77
|
+
dict: A new dictionary with CIK converted to 10-digit zero-padded string
|
|
78
|
+
if present and not None. All other fields remain unchanged.
|
|
79
|
+
"""
|
|
80
|
+
# Convert integer CIK to 10-digit zero-padded string
|
|
81
|
+
cik_value = raw.get("cik")
|
|
82
|
+
updates = {"cik": str(cik_value).zfill(10)}
|
|
83
|
+
|
|
84
|
+
# Return new dict rather than mutating in place
|
|
85
|
+
return {**raw, **updates}
|