equity-aggregator 0.1.1__py3-none-any.whl → 0.1.5__py3-none-any.whl

equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/intrinio.py

@@ -0,0 +1,305 @@
+# intrinio/intrinio.py
+
+import asyncio
+import logging
+import os
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+from equity_aggregator.adapters.data_sources._utils._record_types import (
+    EquityRecord,
+    RecordStream,
+)
+from equity_aggregator.storage import load_cache, save_cache
+
+from ._utils import parse_companies_response, parse_securities_response
+from .session import IntrinioSession
+
+logger = logging.getLogger(__name__)
+
+_INTRINIO_COMPANIES_URL = "https://api-v2.intrinio.com/companies"
+_INTRINIO_SECURITIES_URL = "https://api-v2.intrinio.com/securities"
+_PAGE_SIZE = 5000
+
+
+async def fetch_equity_records(
+    session: IntrinioSession | None = None,
+    *,
+    cache_key: str = "intrinio_records",
+) -> RecordStream:
+    """
+    Yield each Intrinio security record with quote data, using cache if available.
+
+    Fetches all companies, then for each company fetches its securities. Each
+    security (keyed by share_class_figi) becomes a separate equity record with
+    quote data attached.
+
+    Args:
+        session (IntrinioSession | None): Optional Intrinio session for requests.
+        cache_key (str): The key under which to cache the records.
+
+    Yields:
+        EquityRecord: Parsed Intrinio security record with company and quote data.
+
+    Raises:
+        ValueError: If INTRINIO_API_KEY environment variable is not set.
+    """
+    cached = load_cache(cache_key)
+
+    if cached:
+        logger.info("Loaded %d Intrinio records from cache.", len(cached))
+        for record in cached:
+            yield record
+        return
+
+    _get_api_key()  # Validate API key is set before proceeding
+
+    session = session or IntrinioSession(make_client())
+
+    try:
+        async for record in _stream_and_cache(session, cache_key=cache_key):
+            yield record
+    finally:
+        await session.aclose()
+
+
+async def _stream_and_cache(
+    session: IntrinioSession,
+    *,
+    cache_key: str,
+) -> RecordStream:
+    """
+    Stream unique Intrinio security records with quotes, cache them, and yield each.
+
+    Fetches all companies, then fetches securities for each company. Each security
+    gets quote data attached. Records are deduplicated by share_class_figi.
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+        cache_key (str): The key under which to cache the records.
+
+    Yields:
+        EquityRecord: Each unique security record with company and quote data.
+
+    Side Effects:
+        Saves all streamed records to cache after streaming completes.
+    """
+    all_companies = await _fetch_all_companies(session)
+    all_securities = await _fetch_all_securities(session, all_companies)
+    all_records = await _attach_quotes_to_all(session, all_securities)
+    unique_records = _deduplicate_by_share_class_figi(all_records)
+
+    for record in unique_records:
+        yield record
+
+    save_cache(cache_key, unique_records)
+    logger.info("Saved %d Intrinio records to cache.", len(unique_records))
+
+
+async def _fetch_all_companies(session: IntrinioSession) -> list[EquityRecord]:
+    """
+    Fetch all company records from Intrinio, handling pagination.
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+
+    Returns:
+        list[EquityRecord]: Complete list of company records from all pages.
+    """
+    all_companies: list[EquityRecord] = []
+    next_page: str | None = None
+
+    while True:
+        page_records, next_page = await _fetch_companies_page(session, next_page)
+        all_companies.extend(page_records)
+
+        if not next_page:
+            return all_companies
+
+
+async def _fetch_all_securities(
+    session: IntrinioSession,
+    companies: list[EquityRecord],
+) -> list[EquityRecord]:
+    """
+    Fetch securities for all companies concurrently.
+
+    For each company, fetches its securities from the securities endpoint.
+    A company may have multiple securities (e.g., common stock, preferred shares).
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+        companies (list[EquityRecord]): List of company records.
+
+    Returns:
+        list[EquityRecord]: Flattened list of security records with company data.
+    """
+    tasks = [_fetch_company_securities(session, company) for company in companies]
+    results = await asyncio.gather(*tasks)
+
+    all_securities = []
+    for securities in results:
+        all_securities.extend(securities)
+
+    return all_securities
+
+
+async def _attach_quotes_to_all(
+    session: IntrinioSession,
+    securities: list[EquityRecord],
+) -> list[EquityRecord]:
+    """
+    Attach quote data to all securities concurrently.
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+        securities (list[EquityRecord]): List of security records.
+
+    Returns:
+        list[EquityRecord]: Security records with quote data attached.
+    """
+    tasks = [_attach_quote(session, security) for security in securities]
+    return await asyncio.gather(*tasks)
+
+
+def _deduplicate_by_share_class_figi(
+    records: list[EquityRecord],
+) -> list[EquityRecord]:
+    """
+    Deduplicate records by share_class_figi, maintaining insertion order.
+
+    Args:
+        records (list[EquityRecord]): The list of equity records to deduplicate.
+
+    Returns:
+        list[EquityRecord]: Deduplicated list of equity records.
+    """
+    seen: set[str] = set()
+    unique: list[EquityRecord] = []
+
+    for record in records:
+        share_class_figi = record.get("share_class_figi")
+
+        if not share_class_figi or share_class_figi in seen:
+            continue
+
+        seen.add(share_class_figi)
+        unique.append(record)
+
+    return unique
+
+
+async def _fetch_companies_page(
+    session: IntrinioSession,
+    next_page: str | None,
+) -> tuple[list[EquityRecord], str | None]:
+    """
+    Fetch a single page of results from the Intrinio companies endpoint.
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+        next_page (str | None): The pagination token for the next page.
+
+    Returns:
+        tuple[list[EquityRecord], str | None]: Tuple of (parsed records,
+            next_page token).
+    """
+    params = {"api_key": _get_api_key(), "page_size": str(_PAGE_SIZE)}
+
+    if next_page:
+        params["next_page"] = next_page
+
+    response = await session.get(_INTRINIO_COMPANIES_URL, params=params)
+    response.raise_for_status()
+    return parse_companies_response(response.json())
+
+
+async def _fetch_company_securities(
+    session: IntrinioSession,
+    company: EquityRecord,
+) -> list[EquityRecord]:
+    """
+    Fetch securities for a single company.
+
+    A company may have multiple securities. Company data is extracted from the
+    securities response itself (not the passed company record) to avoid ticker
+    reassignment issues where stale company records have incorrect identifiers.
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+        company (EquityRecord): Company record containing company_ticker.
+
+    Returns:
+        list[EquityRecord]: List of security records for this company.
+    """
+    company_ticker = company.get("company_ticker")
+    url = f"{_INTRINIO_COMPANIES_URL}/{company_ticker}/securities"
+    params = {"api_key": _get_api_key()}
+
+    try:
+        response = await session.get(url, params=params)
+        response.raise_for_status()
+        return parse_securities_response(response.json())
+    except Exception:
+        return []
+
+
+async def _attach_quote(
+    session: IntrinioSession,
+    security: EquityRecord,
+) -> EquityRecord:
+    """
+    Fetch quote data for a security and attach it to the record.
+
+    Uses share_class_figi as the identifier for the quote lookup.
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+        security (EquityRecord): Security record to attach quote to.
+
+    Returns:
+        EquityRecord: Security record with quote data attached.
+    """
+    share_class_figi = security.get("share_class_figi")
+    quote = await _fetch_quote(session, share_class_figi)
+    return {**security, "quote": quote}
+
+
+async def _fetch_quote(
+    session: IntrinioSession,
+    share_class_figi: str,
+) -> dict | None:
+    """
+    Fetch quote data from Intrinio API using share_class_figi.
+
+    Args:
+        session (IntrinioSession): The Intrinio session used for requests.
+        share_class_figi (str): The share class FIGI identifier.
+
+    Returns:
+        dict | None: Quote data dictionary, or None if fetch fails.
+    """
+    url = f"{_INTRINIO_SECURITIES_URL}/{share_class_figi}/quote"
+    params = {"api_key": _get_api_key()}
+
+    try:
+        response = await session.get(url, params=params)
+        response.raise_for_status()
+        return response.json()
+    except Exception:
+        return None
+
+
+def _get_api_key() -> str:
+    """
+    Retrieve the Intrinio API key from environment variables.
+
+    Returns:
+        str: The Intrinio API key.
+
+    Raises:
+        ValueError: If INTRINIO_API_KEY environment variable is not set.
+    """
+    api_key = os.getenv("INTRINIO_API_KEY")
+    if not api_key:
+        raise ValueError("INTRINIO_API_KEY environment variable not set")
+    return api_key

equity_aggregator/adapters/data_sources/discovery_feeds/intrinio/session.py

@@ -0,0 +1,197 @@
+# intrinio/session.py
+
+import asyncio
+import logging
+from collections.abc import Mapping
+
+import httpx
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+
+from ._utils import backoff_delays
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+
+class IntrinioSession:
+    """
+    Asynchronous session for Intrinio API endpoints.
+
+    Manages HTTP client lifecycle with automatic recovery on HTTP/2 connection
+    errors. Uses a class-level semaphore to limit concurrent streams and prevent
+    connection exhaustion.
+
+    Args:
+        client (httpx.AsyncClient | None): Optional pre-configured HTTP client.
+
+    Returns:
+        None
+    """
+
+    __slots__ = ("_client", "_lock")
+
+    # Limit HTTP/2 concurrent streams to prevent connection exhaustion
+    _concurrent_streams: asyncio.Semaphore = asyncio.Semaphore(10)
+
+    def __init__(self, client: httpx.AsyncClient | None = None) -> None:
+        """
+        Initialise IntrinioSession with optional HTTP client.
+
+        Args:
+            client (httpx.AsyncClient | None): Optional pre-configured HTTP client.
+
+        Returns:
+            None
+        """
+        self._client: httpx.AsyncClient = client or make_client()
+        self._lock: asyncio.Lock = asyncio.Lock()
+
+    async def aclose(self) -> None:
+        """
+        Close the underlying HTTP client.
+
+        Args:
+            None
+
+        Returns:
+            None
+        """
+        await self._client.aclose()
+
+    async def get(
+        self,
+        url: str,
+        *,
+        params: Mapping[str, str] | None = None,
+    ) -> httpx.Response:
+        """
+        Perform a GET request with concurrency limiting and connection recovery.
+
+        Automatically resets the HTTP client on protocol errors and retries
+        the request with exponential backoff. Handles 429 rate limit responses
+        with exponential backoff before returning.
+
+        Args:
+            url (str): Absolute URL to request.
+            params (Mapping[str, str] | None): Optional query parameters.
+
+        Returns:
+            httpx.Response: The HTTP response.
+
+        Raises:
+            httpx.HTTPError: If the request fails after all retries.
+        """
+        async with self.__class__._concurrent_streams:
+            return await self._get_with_rate_limit_retry(url, params=dict(params or {}))
+
+    async def _get_with_rate_limit_retry(
+        self,
+        url: str,
+        params: dict[str, str],
+    ) -> httpx.Response:
+        """
+        Perform GET request with 429 rate limit retry logic.
+
+        Applies exponential backoff when receiving 429 Too Many Requests responses
+        from the Intrinio API.
+
+        Args:
+            url (str): The absolute URL to request.
+            params (dict[str, str]): Query parameters for the request.
+
+        Returns:
+            httpx.Response: The HTTP response.
+
+        Raises:
+            httpx.HTTPStatusError: If still rate limited after all retries.
+        """
+        response = await self._get_with_retry(url, params=params)
+
+        if response.status_code != httpx.codes.TOO_MANY_REQUESTS:
+            return response
+
+        max_attempts = 5
+
+        for attempt, delay in enumerate(backoff_delays(attempts=max_attempts), 1):
+            logger.debug(
+                "429 Too Many Requests %s - sleeping %.1fs (attempt %d/%d)",
+                url,
+                delay,
+                attempt,
+                max_attempts,
+            )
+            await asyncio.sleep(delay)
+
+            response = await self._get_with_retry(url, params=params)
+
+            if response.status_code != httpx.codes.TOO_MANY_REQUESTS:
+                return response
+
+        return response
+
+    async def _get_with_retry(
+        self,
+        url: str,
+        params: dict[str, str],
+        *,
+        retries_remaining: int = 3,
+    ) -> httpx.Response:
+        """
+        Perform GET request with automatic client recovery on connection errors.
+
+        Uses optimistic retry: if the client was already reset by another task,
+        retries without consuming the retry budget.
+
+        Args:
+            url (str): The absolute URL to request.
+            params (dict[str, str]): Query parameters for the request.
+            retries_remaining (int): Number of retries left for connection errors.
+
+        Returns:
+            httpx.Response: The HTTP response.
+
+        Raises:
+            httpx.HTTPError: If all retry attempts fail.
+        """
+        if retries_remaining <= 0:
+            raise httpx.ConnectError("Connection failed after retries")
+
+        async with self._lock:
+            client, client_id = self._client, id(self._client)
+
+        try:
+            return await client.get(url, params=params)
+
+        except (httpx.TransportError, RuntimeError):
+            was_stale = await self._reset_if_needed(client_id)
+            next_retries = retries_remaining if was_stale else retries_remaining - 1
+
+            return await self._get_with_retry(
+                url,
+                params,
+                retries_remaining=next_retries,
+            )
+
+    async def _reset_if_needed(self, failed_client_id: int) -> bool:
+        """
+        Reset the HTTP client if it hasn't already been reset by another task.
+
+        Args:
+            failed_client_id (int): The id() of the client that failed.
+
+        Returns:
+            bool: True if client was already reset (free retry),
+                  False if this task reset it (counts against budget).
+        """
+        async with self._lock:
+            if failed_client_id != id(self._client):
+                return True  # Already reset by another task
+
+            old_client = self._client
+            self._client = make_client()
+
+        await old_client.aclose()
+        logger.debug(
+            "CLIENT_RESET: Transport error encountered, resetting Intrinio client",
+        )
+        return False

equity_aggregator/adapters/data_sources/discovery_feeds/lseg/__init__.py

@@ -0,0 +1,7 @@
+# discovery_feeds/lseg/__init__.py
+
+from .lseg import fetch_equity_records
+
+__all__ = [
+    "fetch_equity_records",
+]

equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/__init__.py

@@ -0,0 +1,9 @@
+# _utils/__init__.py
+
+from .backoff import backoff_delays
+from .parser import parse_response
+
+__all__ = [
+    "backoff_delays",
+    "parse_response",
+]

equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/backoff.py

@@ -0,0 +1,33 @@
+# lseg/_utils/backoff.py
+
+
+import random
+from collections.abc import Iterator
+
+
+def backoff_delays(
+    *,
+    base: float = 5.0,
+    cap: float = 128.0,
+    jitter: float = 0.10,
+    attempts: int = 5,
+) -> Iterator[float]:
+    """
+    Yield an exponential backoff sequence with bounded jitter for retry delays.
+
+    Each delay is calculated as: delay * (1 ± jitter), doubling each time up to cap.
+
+    Args:
+        base (float): Initial delay in seconds.
+        cap (float): Maximum delay in seconds.
+        jitter (float): Fractional jitter (+/-) applied to each delay.
+        attempts (int): Number of delay values to yield.
+
+    Returns:
+        Iterator[float]: Sequence of delay values in seconds.
+    """
+    delay: float = base
+    for _ in range(attempts):
+        delta: float = delay * jitter * (2 * random.random() - 1)
+        yield max(0.0, min(delay + delta, cap))
+        delay = min(delay * 2, cap)

equity_aggregator/adapters/data_sources/discovery_feeds/lseg/_utils/parser.py

@@ -0,0 +1,120 @@
+# _utils/parser.py
+
+from equity_aggregator.adapters.data_sources._utils._record_types import (
+    EquityRecord,
+)
+
+
+def parse_response(data: dict) -> tuple[list[EquityRecord], dict | None]:
+    """
+    Parse LSEG response to extract equity records and pagination metadata.
+
+    Extracts the price-explorer component, then the priceexplorersearch item,
+    then the value data containing the content array. Processes the content into
+    equity records with pagination info. Returns empty result if extraction fails.
+
+    Args:
+        data (dict): Raw JSON response from LSEG.
+
+    Returns:
+        tuple[list[EquityRecord], dict | None]: Tuple containing equity records
+            and pagination metadata, or empty list and None if no data found.
+    """
+    component = _find_price_explorer_component(data)
+
+    search_item = (
+        _find_content_item(component, "priceexplorersearch") if component else None
+    )
+
+    if not search_item:
+        return ([], None)
+
+    value_data = search_item.get("value")
+
+    if value_data and "content" in value_data:
+        return _process_value_data(value_data)
+
+    return ([], None)
+
+
+def _find_price_explorer_component(data: dict) -> dict | None:
+    """
+    Find the price-explorer component.
+
+    Args:
+        data (dict): Raw JSON response from LSEG API.
+
+    Returns:
+        dict | None: The price-explorer component if found, None otherwise.
+    """
+    return next(
+        (
+            component
+            for component in data.get("components", [])
+            if component.get("type") == "price-explorer"
+        ),
+        None,
+    )
+
+
+def _find_content_item(component: dict | None, item_name: str) -> dict | None:
+    """
+    Find a specific content item by name within a component.
+
+    Args:
+        component (dict | None): The component to search within.
+        item_name (str): Name of the content item to find.
+
+    Returns:
+        dict | None: The content item if found, None otherwise.
+    """
+    return component and next(
+        (
+            item
+            for item in component.get("content", [])
+            if item.get("name") == item_name
+        ),
+        None,
+    )
+
+
+def _process_value_data(value_data: dict) -> tuple[list[EquityRecord], dict]:
+    """
+    Process value data into records and pagination info.
+
+    Args:
+        value_data (dict): Value data containing content and pagination metadata.
+
+    Returns:
+        tuple[list[EquityRecord], dict]: Tuple containing processed equity
+            records and pagination information.
+    """
+    records = [_extract_equity_record(equity) for equity in value_data["content"]]
+    pagination_info = {"totalPages": value_data.get("totalPages")}
+    return records, pagination_info
+
+
+def _extract_equity_record(equity: dict) -> EquityRecord:
+    """
+    Normalise raw LSEG JSON equity data into EquityRecord dictionary.
+
+    Maps the raw API fields to the expected LsegFeedData schema fields.
+
+    Args:
+        equity (dict): Raw equity data from LSEG price-explorer API response
+            with equity information and market data fields.
+
+    Returns:
+        EquityRecord: Normalised equity record dictionary with field names
+            matching LsegFeedData schema expectations.
+    """
+    return {
+        "issuername": equity.get("issuername"),
+        "tidm": equity.get("tidm"),
+        "isin": equity.get("isin"),
+        "currency": equity.get("currency"),
+        "lastprice": equity.get("lastprice"),
+        "marketcapitalization": equity.get("marketcapitalization"),
+        "fiftyTwoWeeksMin": equity.get("fiftyTwoWeeksMin"),
+        "fiftyTwoWeeksMax": equity.get("fiftyTwoWeeksMax"),
+    }