equity-aggregator 0.1.1__py3-none-any.whl → 0.1.5__py3-none-any.whl

equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/tradingview.py

@@ -0,0 +1,275 @@
+# tradingview/tradingview.py
+
+import logging
+import math
+
+from httpx import AsyncClient
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+from equity_aggregator.adapters.data_sources._utils._record_types import (
+    EquityRecord,
+    RecordStream,
+)
+from equity_aggregator.storage import load_cache, save_cache
+
+logger = logging.getLogger(__name__)
+
+_TRADINGVIEW_SCAN_URL = "https://scanner.tradingview.com/america/scan"
+_PAGE_SIZE = 1000
+_EXPECTED_ARRAY_LENGTH = 19
+
+_REQUEST_BODY_TEMPLATE = {
+    "markets": ["america"],
+    "symbols": {
+        "query": {"types": ["stock"]},
+        "tickers": [],
+    },
+    "options": {"lang": "en"},
+    "filter": [],  # Empty to fetch all stocks
+    "columns": [
+        "name",
+        "description",
+        "exchange",
+        "currency",
+        "close",
+        "market_cap_basic",
+        "volume",
+        "dividends_yield_current",
+        "float_shares_outstanding",
+        "total_shares_outstanding_fundamental",
+        "total_revenue_ttm",
+        "ebitda_ttm",
+        "price_earnings_ttm",
+        "price_book_fq",
+        "earnings_per_share_basic_ttm",
+        "return_on_equity_fq",
+        "return_on_assets_fq",
+        "sector",
+        "industry",
+    ],
+    "sort": {"sortBy": "name", "sortOrder": "asc"},
+    # range is set per request for pagination
+}
+
+
+async def fetch_equity_records(
+    client: AsyncClient | None = None,
+    *,
+    cache_key: str = "tradingview_records",
+) -> RecordStream:
+    """
+    Yield each TradingView equity record exactly once, using cache if available.
+
+    If a cache is present, loads and yields records from cache. Otherwise, fetches
+    all equity records from the TradingView scanner endpoint by paginating through
+    all available pages, deduplicates by symbol, yields records, and caches results.
+
+    Args:
+        client (AsyncClient | None): Optional HTTP client to use for requests.
+        cache_key (str): The key under which to cache the records.
+
+    Yields:
+        EquityRecord: Parsed TradingView equity record.
+    """
+    cached = load_cache(cache_key)
+    if cached:
+        logger.info("Loaded %d TradingView records from cache.", len(cached))
+        for record in cached:
+            yield record
+        return
+
+    client = client or make_client()
+
+    async with client:
+        async for record in _stream_and_cache(client, cache_key=cache_key):
+            yield record
+
+
+async def _stream_and_cache(
+    client: AsyncClient,
+    *,
+    cache_key: str,
+) -> RecordStream:
+    """
+    Stream TradingView equity records, deduplicate by symbol, cache them, and yield.
+
+    Fetches all records using pagination, deduplicates by symbol to ensure uniqueness,
+    then yields each record and caches the complete set.
+
+    Args:
+        client (AsyncClient): The HTTP client used for requests.
+        cache_key (str): The key under which to cache the records.
+
+    Yields:
+        EquityRecord: Each unique TradingView equity record as retrieved.
+
+    Side Effects:
+        Saves all streamed records to cache after streaming completes.
+    """
+    all_records = await _fetch_all_records(client)
+    unique_records = _deduplicate_by_symbol(all_records)
+
+    for record in unique_records:
+        yield record
+
+    save_cache(cache_key, unique_records)
+    logger.info("Saved %d TradingView records to cache.", len(unique_records))
+
+
+async def _fetch_all_records(client: AsyncClient) -> list[EquityRecord]:
+    """
+    Fetch all equity records from TradingView scanner, handling pagination.
+
+    Retrieves the first page to determine total count, then fetches remaining
+    pages sequentially. Stops on first error to avoid cascade failures.
+
+    Args:
+        client (AsyncClient): The HTTP client used for requests.
+
+    Returns:
+        list[EquityRecord]: All fetched equity records across all pages.
+    """
+    # Fetch first page to get total count
+    first_page_records, total_count = await _fetch_page(client, 0, _PAGE_SIZE)
+
+    if total_count <= _PAGE_SIZE:
+        return first_page_records
+
+    # Calculate total pages needed
+    total_pages = math.ceil(total_count / _PAGE_SIZE)
+    all_records = first_page_records
+
+    # Fetch remaining pages sequentially
+    for page in range(1, total_pages):
+        start = page * _PAGE_SIZE
+        end = start + _PAGE_SIZE
+
+        try:
+            page_records, _ = await _fetch_page(client, start, end)
+            all_records.extend(page_records)
+        except Exception as error:
+            logger.warning(
+                "Failed to fetch page range [%d, %d]: %s. Returning partial results.",
+                start,
+                end,
+                error,
+            )
+            break
+
+    return all_records
+
+
+async def _fetch_page(
+    client: AsyncClient,
+    start: int,
+    end: int,
+) -> tuple[list[EquityRecord], int]:
+    """
+    Fetch a single page of results from TradingView scanner.
+
+    Args:
+        client (AsyncClient): The HTTP client used for requests.
+        start (int): Starting index for pagination range.
+        end (int): Ending index for pagination range.
+
+    Returns:
+        tuple[list[EquityRecord], int]: Tuple of (parsed records, total count from API).
+    """
+    request_body = {**_REQUEST_BODY_TEMPLATE, "range": [start, end]}
+
+    response = await client.post(_TRADINGVIEW_SCAN_URL, json=request_body)
+    response.raise_for_status()
+
+    payload = response.json()
+    return _parse_response(payload)
+
+
+def _parse_response(payload: dict) -> tuple[list[EquityRecord], int]:
+    """
+    Parse TradingView API response into equity records.
+
+    Extracts the data array and total count from the response payload,
+    then parses each item into an EquityRecord.
+
+    Args:
+        payload (dict): The JSON response from TradingView API.
+
+    Returns:
+        tuple[list[EquityRecord], int]: Tuple of (parsed records, total count).
+    """
+    data = payload.get("data", [])
+    total_count = payload.get("totalCount", 0)
+
+    records = []
+    for row in data:
+        record = _parse_row(row)
+        if record:
+            records.append(record)
+
+    return records, total_count
+
+
+def _parse_row(row: dict | None) -> EquityRecord | None:
+    """
+    Parse a single TradingView API response row into an EquityRecord.
+
+    Args:
+        row (dict | None): A single row from the TradingView API response.
+
+    Returns:
+        EquityRecord | None: The parsed equity record, or None if invalid.
+    """
+    # Validate row exists and has data array
+    if not row:
+        return None
+
+    d = row.get("d", [])
+    if not d or len(d) < _EXPECTED_ARRAY_LENGTH:
+        if d is not None:
+            logger.warning(
+                "Invalid data array length: expected %d, got %d",
+                _EXPECTED_ARRAY_LENGTH,
+                len(d),
+            )
+        return None
+
+    # Extract and validate required fields, then build the equity record
+    symbol = d[0]
+    name = d[1]
+
+    return (
+        {
+            "s": row.get("s"),  # Preserve original exchange:symbol format
+            "d": d,  # Pass the full data array to schema for processing
+        }
+        if symbol and name
+        else None
+    )
+
+
+def _deduplicate_by_symbol(records: list[EquityRecord]) -> list[EquityRecord]:
+    """
+    Deduplicate records by symbol, maintaining insertion order.
+
+    Args:
+        records (list[EquityRecord]): The list of equity records to deduplicate.
+
+    Returns:
+        list[EquityRecord]: List of unique records, preserving first occurrence.
+    """
+    seen_symbols: set[str] = set()
+    unique: list[EquityRecord] = []
+
+    for record in records:
+        # Extract symbol from the data array
+        d = record.get("d", [])
+        symbol = d[0] if d and len(d) > 0 else None
+
+        if not symbol:
+            continue
+
+        if symbol not in seen_symbols:
+            seen_symbols.add(symbol)
+            unique.append(record)
+
+    return unique

equity_aggregator/adapters/data_sources/discovery_feeds/xetra/__init__.py

@@ -0,0 +1,7 @@
+# discovery_feeds/xetra/__init__.py
+
+from .xetra import fetch_equity_records
+
+__all__ = [
+    "fetch_equity_records",
+]

equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/xetra}/xetra.py

@@ -1,4 +1,4 @@
-# authoritative_feeds/xetra.py
+# xetra/xetra.py
 
 import asyncio
 import logging
@@ -6,27 +6,26 @@ import logging
 from httpx import AsyncClient
 
 from equity_aggregator.adapters.data_sources._utils import make_client
-from equity_aggregator.storage import load_cache, save_cache
-
-from ._record_types import (
+from equity_aggregator.adapters.data_sources._utils._record_types import (
     EquityRecord,
     RecordStream,
     RecordUniqueKeyExtractor,
     UniqueRecordStream,
 )
+from equity_aggregator.storage import load_cache, save_cache
 
 logger = logging.getLogger(__name__)
 
 _PAGE_SIZE = 100
 
-_XETRA_SEARCH_URL = "https://api.boerse-frankfurt.de/v1/search/equity_search"
+_XETRA_SEARCH_URL = "https://api.live.deutsche-boerse.com/v1/search/equity_search"
 
 _HEADERS = {
     "Accept": "application/json, text/plain, */*",
     "User-Agent": "Mozilla/5.0",
     "Content-Type": "application/json; charset=UTF-8",
-    "Referer": "https://www.boerse-frankfurt.de/",
-    "Origin": "https://www.boerse-frankfurt.de",
+    "Referer": "https://live.deutsche-boerse.com/",
+    "Origin": "https://live.deutsche-boerse.com",
     "Cache-Control": "no-cache",
     "Pragma": "no-cache",
 }
@@ -208,10 +207,8 @@ async def _produce_page(
         for record in _extract_records(page):
             await queue.put(record)
 
-        logger.debug("Xetra page at offset %s completed", offset)
-
     except Exception as error:
-        logger.fatal("Xetra page at offset %s failed: %s", offset, error, exc_info=True)
+        logger.error("Xetra page at offset %s failed: %s", offset, error, exc_info=True)
         raise
 
     finally:
@@ -269,7 +266,7 @@ async def _fetch_page(client: AsyncClient, offset: int) -> dict[str, object]:
         return response.json()
 
     except ValueError as error:
-        logger.fatal(
+        logger.error(
             "Xetra JSON decode error at offset %s: %s",
             offset,
             error,
@@ -288,7 +285,7 @@ def _extract_records(page_response_json: dict[str, object]) -> list[EquityRecord
 
     Returns:
         list[EquityRecord]: A list of normalised equity records, each as a dictionary
-            with standardised keys matching the eurONext schema.
+            with standardised keys matching the schema.
     """
     rows = page_response_json.get("data", [])
     return [

equity_aggregator/adapters/data_sources/enrichment_feeds/__init__.py

@@ -1,5 +1,10 @@
 # enrichment_feeds/__init__.py
 
+from .gleif import GleifFeed, open_gleif_feed
 from .yfinance import open_yfinance_feed
 
-__all__ = ["open_yfinance_feed"]
+__all__ = [
+    "GleifFeed",
+    "open_gleif_feed",
+    "open_yfinance_feed",
+]

equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/__init__.py

@@ -0,0 +1,5 @@
+# gleif/__init__.py
+
+from .gleif import GleifFeed, open_gleif_feed
+
+__all__ = ["GleifFeed", "open_gleif_feed"]

equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/api.py

@@ -0,0 +1,71 @@
+# gleif/api.py
+
+import logging
+from collections.abc import Callable
+
+import httpx
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+
+logger = logging.getLogger(__name__)
+
+GLEIF_ISIN_LEI_URL = "https://mapping.gleif.org/api/v2/isin-lei/latest"
+
+
+async def fetch_metadata(
+    *,
+    client_factory: Callable[[], httpx.AsyncClient] | None = None,
+) -> dict[str, object] | None:
+    """
+    Fetch the latest GLEIF ISIN->LEI mapping metadata from the API.
+
+    Calls the GLEIF mapping API to retrieve metadata about the latest
+    ISIN->LEI relationship file, including the download link.
+
+    Args:
+        client_factory: Factory function to create an HTTP client.
+
+    Returns:
+        Metadata dict with the following keys, or None on failure:
+            - id: UUID of the mapping file
+            - file_name: Name of the ZIP file
+            - uploaded_at: ISO-8601 upload timestamp
+            - download_link: URL to download the ZIP
+    """
+    logger.info("Fetching GLEIF ISIN->LEI metadata from API.")
+
+    factory = client_factory or make_client
+
+    try:
+        async with factory() as client:
+            return await _fetch_metadata_with_client(client)
+    except Exception as error:
+        logger.error("Failed to fetch GLEIF metadata: %s", error, exc_info=True)
+        return None
+
+
+async def _fetch_metadata_with_client(
+    client: httpx.AsyncClient,
+) -> dict[str, object]:
+    """
+    Fetch metadata using the provided HTTP client.
+
+    Args:
+        client: HTTP client to use for the request.
+
+    Returns:
+        Metadata dict with id, file_name, uploaded_at, and download_link.
+    """
+    response = await client.get(GLEIF_ISIN_LEI_URL)
+    response.raise_for_status()
+    payload = response.json()
+
+    data = payload.get("data", {})
+    attrs = data.get("attributes", {})
+
+    return {
+        "id": data.get("id"),
+        "file_name": attrs.get("fileName"),
+        "uploaded_at": attrs.get("uploadedAt"),
+        "download_link": attrs.get("downloadLink"),
+    }

equity_aggregator/adapters/data_sources/enrichment_feeds/gleif/download.py

@@ -0,0 +1,109 @@
+# gleif/download.py
+
+from collections.abc import Callable
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+import httpx
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+
+from .api import _fetch_metadata_with_client
+from .parser import parse_zip
+
+
+async def download_and_build_index(
+    *,
+    client_factory: Callable[[], httpx.AsyncClient] | None = None,
+) -> dict[str, str]:
+    """
+    Download the GLEIF ISIN->LEI mapping file and build a lookup index.
+
+    Fetches metadata to get the download link, downloads the ZIP file using
+    streaming, extracts the CSV, and builds a dictionary mapping ISINs to LEIs.
+
+    Args:
+        client_factory: Factory function to create an HTTP client.
+            Defaults to make_client from _utils.
+
+    Returns:
+        Dictionary mapping ISIN codes to LEI codes.
+
+    Raises:
+        ValueError: If metadata or download link is unavailable.
+    """
+    factory = client_factory or make_client
+
+    async with factory() as client:
+        try:
+            metadata = await _fetch_metadata_with_client(client)
+        except Exception as error:
+            raise ValueError("Failed to retrieve GLEIF ISIN->LEI metadata.") from error
+
+        download_link = metadata.get("download_link")
+        if not download_link:
+            raise ValueError("GLEIF metadata missing download_link.")
+
+        return await _download_and_parse(client, str(download_link))
+
+
+async def _download_and_parse(
+    client: httpx.AsyncClient,
+    download_link: str,
+) -> dict[str, str]:
+    """
+    Download the GLEIF mapping ZIP file and parse it into an index.
+
+    Uses a temporary directory for the download to avoid persisting large files.
+
+    Args:
+        client: HTTP client to use for the download.
+        download_link: URL to download the ZIP file from.
+
+    Returns:
+        Dictionary mapping ISIN codes to LEI codes.
+    """
+    with TemporaryDirectory() as temp_dir:
+        zip_path = Path(temp_dir) / "isin_lei.zip"
+        await _stream_download(client, download_link, zip_path)
+        return parse_zip(zip_path)
+
+
+async def _stream_download(
+    client: httpx.AsyncClient,
+    url: str,
+    destination: Path,
+) -> None:
+    """
+    Stream download a file from a URL to a local path.
+
+    Uses chunked transfer to handle large files efficiently without
+    loading the entire response into memory.
+
+    Args:
+        client: HTTP client to use for the download.
+        url: URL to download from.
+        destination: Local file path to write to.
+    """
+    await _stream_to_file(client, url, destination)
+
+
+async def _stream_to_file(
+    client: httpx.AsyncClient,
+    url: str,
+    destination: Path,
+) -> None:
+    """
+    Stream response body to a file.
+
+    Args:
+        client: HTTP client to use for the request.
+        url: URL to download from.
+        destination: Local file path to write to.
+    """
+    async with client.stream("GET", url) as response:
+        response.raise_for_status()
+
+        with destination.open("wb") as f:
+            async for chunk in response.aiter_bytes(chunk_size=65536):
+                f.write(chunk)