equity-aggregator 0.1.1__py3-none-any.whl → 0.1.5__py3-none-any.whl

equity_aggregator/adapters/data_sources/discovery_feeds/lseg/lseg.py

@@ -0,0 +1,239 @@
+# lseg/lseg.py
+
+import logging
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+from equity_aggregator.adapters.data_sources._utils._record_types import (
+    EquityRecord,
+    RecordStream,
+)
+from equity_aggregator.storage import load_cache, save_cache
+
+from ._utils import parse_response
+from .session import (
+    LsegSession,
+)
+
+logger = logging.getLogger(__name__)
+
+_LSEG_SEARCH_URL = "https://api.londonstockexchange.com/api/v1/pages"
+_LSEG_PATH = "live-markets/market-data-dashboard/price-explorer"
+_LSEG_BASE_PARAMS = "categories=EQUITY"
+
+_HEADERS = {
+    "Accept": "application/json, text/plain, */*",
+    "Accept-Encoding": "gzip, deflate, br, zstd",
+    "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
+    "Cache-Control": "no-cache",
+    "Origin": "https://www.londonstockexchange.com",
+    "Pragma": "no-cache",
+    "Referer": "https://www.londonstockexchange.com/",
+    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
+}
+
+
+async def fetch_equity_records(
+    session: LsegSession | None = None,
+    *,
+    cache_key: str = "lseg_records",
+) -> RecordStream:
+    """
+    Yield each LSEG equity record exactly once, using cache if available.
+
+    If a cache is present, loads and yields records from cache. Otherwise, fetches
+    all equity records from the LSEG price-explorer endpoint by paginating through
+    all available pages, deduplicates by ISIN, yields records, and caches the results.
+
+    Args:
+        session (LsegSession | None): Optional LSEG session for requests.
+        cache_key (str): The key under which to cache the records.
+
+    Yields:
+        EquityRecord: Parsed LSEG equity record.
+    """
+    cached = load_cache(cache_key)
+    if cached:
+        logger.info("Loaded %d LSEG records from cache.", len(cached))
+        for record in cached:
+            yield record
+        return
+
+    session = session or LsegSession(make_client(headers=_HEADERS))
+
+    try:
+        async for record in _stream_and_cache(session, cache_key=cache_key):
+            yield record
+    finally:
+        await session.aclose()
+
+
+async def _stream_and_cache(
+    session: LsegSession,
+    *,
+    cache_key: str,
+) -> RecordStream:
+    """
+    Stream unique LSEG equity records, cache them, and yield each.
+
+    Fetches all records, deduplicates by ISIN (filtering out records with
+    missing or empty ISINs), then yields and caches the unique records.
+
+    Args:
+        session (LsegSession): The LSEG session used for requests.
+        cache_key (str): The key under which to cache the records.
+
+    Yields:
+        EquityRecord: Each unique LSEG equity record as retrieved.
+
+    Side Effects:
+        Saves all streamed records to cache after streaming completes.
+    """
+    all_records = await _fetch_all_records(session)
+    unique_records = _deduplicate_by_isin(all_records)
+
+    for record in unique_records:
+        yield record
+
+    save_cache(cache_key, unique_records)
+    logger.info("Saved %d LSEG records to cache.", len(unique_records))
+
+
+async def _fetch_all_records(
+    session: LsegSession,
+) -> list[EquityRecord]:
+    """
+    Fetch all equity records from the price-explorer endpoint, handling pagination.
+
+    Retrieves the first page, determines total pages, then fetches
+    remaining pages sequentially with resilient error handling.
+
+    Args:
+        session: HTTP session for API requests.
+
+    Returns:
+        Complete list of equity records from all pages.
+    """
+    # Fetch first page and extract pagination metadata
+    first_page_data, pagination_info = await _fetch_page(session, 0)
+    total_pages = _extract_total_pages(pagination_info)
+
+    if total_pages <= 1:
+        return first_page_data
+
+    # Fetch remaining pages with error resilience
+    remaining_pages_data = await _fetch_remaining_pages(session, total_pages)
+
+    return first_page_data + remaining_pages_data
+
+
+async def _fetch_page(
+    session: LsegSession,
+    page: int,
+) -> tuple[list[EquityRecord], dict | None]:
+    """
+    Fetch a single page of results from LSEG price-explorer endpoint.
+
+    Sends GET request to LSEG pages endpoint with the specified page number,
+    returns parsed equity records and pagination metadata.
+
+    Args:
+        session (LsegSession): LSEG session used to send the request.
+        page (int): Zero-based page number to fetch.
+
+    Returns:
+        tuple[list[EquityRecord], dict | None]: Tuple containing parsed equity
+            records and pagination metadata from LSEG feed.
+
+    Raises:
+        httpx.HTTPStatusError: If response status is not successful.
+        httpx.ReadError: If there is a network or connection error.
+        ValueError: If response body cannot be parsed as JSON.
+    """
+    parameters = f"{_LSEG_BASE_PARAMS}&page={page}"
+    response = await session.get(
+        _LSEG_SEARCH_URL,
+        params={
+            "path": _LSEG_PATH,
+            "parameters": parameters,
+        },
+    )
+    response.raise_for_status()
+    return parse_response(response.json())
+
+
+def _extract_total_pages(pagination_info: dict | None) -> int:
+    """
+    Extract the total page count from LSEG API pagination metadata.
+
+    Safely retrieves the totalPages field from pagination info, providing a
+    sensible default of 1 when pagination data is missing or invalid.
+
+    Args:
+        pagination_info (dict | None): Pagination metadata from API response,
+            expected to contain a 'totalPages' field, or None if unavailable.
+
+    Returns:
+        int: Total number of pages available, defaulting to 1 if pagination
+            info is missing or does not contain the totalPages field.
+    """
+    return pagination_info.get("totalPages", 1) if pagination_info else 1
+
+
+async def _fetch_remaining_pages(
+    session: LsegSession,
+    total_pages: int,
+) -> list[EquityRecord]:
+    """
+    Fetch all remaining pages sequentially with error handling.
+
+    Args:
+        session: HTTP session for API requests.
+        total_pages: Total number of pages to fetch.
+
+    Returns:
+        Combined records from all successfully fetched remaining pages.
+    """
+    all_remaining_records = []
+
+    for page in range(1, total_pages):
+        try:
+            page_data, _ = await _fetch_page(session, page)
+            all_remaining_records.extend(page_data)
+        except Exception as error:
+            logger.warning(
+                "Failed to fetch page %d: %s",
+                page,
+                error,
+            )
+            break  # Stop on first error to avoid cascade failures
+
+    return all_remaining_records
+
+
+def _deduplicate_by_isin(records: list[EquityRecord]) -> list[EquityRecord]:
+    """
+    Deduplicate equity records by ISIN, maintaining insertion order.
+
+    Filters out records with missing or empty ISINs, then removes duplicates
+    by keeping the first occurrence of each unique ISIN.
+
+    Args:
+        records (list[EquityRecord]): List of equity records to deduplicate.
+
+    Returns:
+        list[EquityRecord]: Deduplicated list of equity records.
+    """
+    seen_isins: set[str] = set()
+    unique: list[EquityRecord] = []
+
+    for record in records:
+        isin = record.get("isin")
+
+        if not isin:
+            continue
+
+        if isin not in seen_isins:
+            seen_isins.add(isin)
+            unique.append(record)
+
+    return unique

equity_aggregator/adapters/data_sources/discovery_feeds/lseg/session.py

@@ -0,0 +1,162 @@
+# lseg/session.py
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable, Mapping
+
+import httpx
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+
+from ._utils import backoff_delays
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+# Type alias for HTTP request functions
+HttpRequestFunc = Callable[..., Awaitable[httpx.Response]]
+
+
+class LsegSession:
+    """
+    Asynchronous session for LSEG JSON endpoints.
+
+    This class manages HTTP requests to the LSEG trading platform API, handling
+    rate limits and 403 Forbidden responses with exponential backoff retry logic.
+    It is lightweight and reusable, maintaining only a client and session state.
+
+    Args:
+        client (httpx.AsyncClient | None): Optional pre-configured HTTP client.
+
+    Returns:
+        None
+    """
+
+    __slots__: tuple[str, ...] = ("_client",)
+
+    def __init__(
+        self,
+        client: httpx.AsyncClient | None = None,
+    ) -> None:
+        """
+        Initialise a new LsegSession for LSEG JSON endpoints.
+
+        Args:
+            client (httpx.AsyncClient | None): Optional pre-configured HTTP client.
+
+        Returns:
+            None
+        """
+        self._client: httpx.AsyncClient = client or make_client()
+
+    async def aclose(self) -> None:
+        """
+        Asynchronously close the underlying HTTP client.
+
+        Args:
+            None
+
+        Returns:
+            None
+        """
+        await self._client.aclose()
+
+    async def get(
+        self,
+        url: str,
+        *,
+        params: Mapping[str, str] | None = None,
+    ) -> httpx.Response:
+        """
+        Perform a resilient asynchronous GET request to LSEG endpoints.
+
+        This method applies exponential backoff on 403 Forbidden responses
+        to handle rate limiting gracefully.
+
+        Args:
+            url (str): Absolute URL to request.
+            params (Mapping[str, str] | None): Optional query parameters.
+
+        Returns:
+            httpx.Response: The successful HTTP response.
+        """
+        return await self._request_with_retry(
+            self._client.get,
+            url,
+            params=dict(params or {}),
+        )
+
+    async def post(
+        self,
+        url: str,
+        *,
+        json: dict[str, object] | None = None,
+    ) -> httpx.Response:
+        """
+        Perform a resilient asynchronous POST request to LSEG endpoints.
+
+        This method applies exponential backoff on 403 Forbidden responses
+        to handle rate limiting gracefully.
+
+        Args:
+            url (str): Absolute URL to request.
+            json (dict[str, object] | None): Optional JSON payload.
+
+        Returns:
+            httpx.Response: The successful HTTP response.
+        """
+        return await self._request_with_retry(
+            self._client.post,
+            url,
+            json=json or {},
+        )
+
+    async def _request_with_retry(
+        self,
+        request_func: HttpRequestFunc,
+        url: str,
+        **kwargs: object,
+    ) -> httpx.Response:
+        """
+        Perform a HTTP request with 403 Forbidden retry logic.
+
+        This method is the core retry mechanism that handles exponential backoff
+        for 403 responses regardless of the HTTP method used.
+
+        Args:
+            request_func (HttpRequestFunc): The HTTP client method to call (get/post).
+            url (str): The absolute URL to request.
+            **kwargs: Additional keyword arguments to pass to the request function.
+
+        Returns:
+            httpx.Response: The successful HTTP response.
+
+        Raises:
+            LookupError: If the final response is still 403 after all retries.
+        """
+        response = await request_func(url, **kwargs)
+
+        # If not a 403, return immediately
+        if response.status_code != httpx.codes.FORBIDDEN:
+            return response
+
+        # Apply exponential backoff for 403 responses
+        max_attempts = 5
+
+        for attempt, delay in enumerate(backoff_delays(attempts=max_attempts), 1):
+            logger.debug(
+                "403 Forbidden %s - sleeping %.1fs (attempt %d/%d)",
+                url,
+                delay,
+                attempt,
+                max_attempts,
+            )
+            await asyncio.sleep(delay)
+
+            response = await request_func(url, **kwargs)
+
+            # If we get a non-403 response, return it
+            if response.status_code != httpx.codes.FORBIDDEN:
+                return response
+
+        # If we still have a 403 after all retries, raise an error
+        raise LookupError(f"HTTP 403 Forbidden after retries for {url}")

equity_aggregator/adapters/data_sources/discovery_feeds/sec/__init__.py

@@ -0,0 +1,7 @@
+# discovery_feeds/sec/__init__.py
+
+from .sec import fetch_equity_records
+
+__all__ = [
+    "fetch_equity_records",
+]

equity_aggregator/adapters/data_sources/{authoritative_feeds → discovery_feeds/sec}/sec.py

@@ -1,18 +1,17 @@
-# authoritative_feeds/sec.py
+# sec/sec.py
 
 import logging
 
 from httpx import AsyncClient
 
 from equity_aggregator.adapters.data_sources._utils import make_client
-from equity_aggregator.storage import load_cache, save_cache
-
-from ._record_types import (
+from equity_aggregator.adapters.data_sources._utils._record_types import (
     EquityRecord,
     RecordStream,
     RecordUniqueKeyExtractor,
     UniqueRecordStream,
 )
+from equity_aggregator.storage import load_cache, save_cache
 
 logger = logging.getLogger(__name__)
 
@@ -93,7 +92,7 @@ async def _stream_and_cache(
 
 async def _stream_sec(client: AsyncClient) -> RecordStream:
     """
-    Fetch and stream SEC equity records from the authoritative JSON endpoint.
+    Fetch and stream SEC equity records from the discovery JSON endpoint.
 
     Args:
         client (AsyncClient): HTTP client for making requests.

equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/__init__.py

@@ -0,0 +1,7 @@
+# stock_analysis/__init__.py
+
+from .stock_analysis import fetch_equity_records
+
+__all__ = [
+    "fetch_equity_records",
+]

equity_aggregator/adapters/data_sources/discovery_feeds/stock_analysis/stock_analysis.py

@@ -0,0 +1,150 @@
+# stock_analysis/stock_analysis.py
+
+import logging
+
+from httpx import AsyncClient
+
+from equity_aggregator.adapters.data_sources._utils import make_client
+from equity_aggregator.adapters.data_sources._utils._record_types import (
+    EquityRecord,
+    RecordStream,
+    RecordUniqueKeyExtractor,
+    UniqueRecordStream,
+)
+from equity_aggregator.storage import load_cache, save_cache
+
+logger = logging.getLogger(__name__)
+
+_STOCK_ANALYSIS_SEARCH_URL = "https://stockanalysis.com/api/screener/s/f"
+
+_PARAMS = {
+    # Primary metric to use for screening/sorting
+    "m": "marketCap",
+    # Sort order (desc = descending, asc = ascending)
+    "s": "desc",
+    # Comma-separated list of columns/fields to return in the response
+    "c": (
+        "s,n,cusip,isin,marketCap,price,volume,peRatio,sector,"
+        "industry,revenue,fcf,roe,roa,ebitda"
+    ),
+    # Instrument type/universe to screen (allstocks = all available stocks)
+    "i": "allstocks",
+}
+
+
+async def fetch_equity_records(
+    client: AsyncClient | None = None,
+    *,
+    cache_key: str = "stock_analysis_records",
+) -> RecordStream:
+    """
+    Yield each Stock Analysis equity record exactly once, using cache if available.
+
+    If a cache is present, loads and yields records from cache. Otherwise, streams
+    all records in a single request, yields records as they arrive, and caches the
+    results.
+
+    Args:
+        client (AsyncClient | None): Optional HTTP client to use for requests.
+        cache_key (str): The key under which to cache the records.
+
+    Yields:
+        EquityRecord: Parsed Stock Analysis equity record.
+    """
+    cached = load_cache(cache_key)
+
+    if cached:
+        logger.info("Loaded %d Stock Analysis records from cache.", len(cached))
+        for record in cached:
+            yield record
+        return
+
+    # use provided client or create a bespoke stock analysis client
+    client = client or make_client()
+
+    async with client:
+        async for record in _stream_and_cache(client, cache_key=cache_key):
+            yield record
+
+
+async def _stream_and_cache(
+    client: AsyncClient,
+    *,
+    cache_key: str,
+) -> RecordStream:
+    """
+    Stream Stock Analysis equity records, deduplicate by ISIN, cache them, and yield
+    each record.
+
+    Args:
+        client (AsyncClient): HTTP client for Stock Analysis requests.
+        cache_key (str): Key under which to store cached records.
+
+    Returns:
+        RecordStream: Async iterator yielding unique EquityRecord objects.
+    """
+    buffer: list[EquityRecord] = []
+
+    async for record in _deduplicate_records(lambda record: record.get("isin"))(
+        _stream_stock_analysis(client),
+    ):
+        buffer.append(record)
+        yield record
+
+    save_cache(cache_key, buffer)
+    logger.info("Saved %d Stock Analysis records to cache.", len(buffer))
+
+
+async def _stream_stock_analysis(client: AsyncClient) -> RecordStream:
+    """
+    Fetch and stream Stock Analysis equity records from the screener endpoint.
+
+    Args:
+        client (AsyncClient): HTTP client for making requests.
+
+    Yields:
+        EquityRecord: Each valid Stock Analysis equity record.
+    """
+    response = await client.get(_STOCK_ANALYSIS_SEARCH_URL, params=_PARAMS)
+    response.raise_for_status()
+
+    payload = response.json()
+    data_wrapper = payload.get("data", {})
+    records = data_wrapper.get("data", [])
+
+    for record in records:
+        if record:
+            yield record
+
+
+def _deduplicate_records(extract_key: RecordUniqueKeyExtractor) -> UniqueRecordStream:
+    """
+    Creates a deduplication coroutine for async iterators of dictionaries, yielding only
+    unique records based on a key extracted from each record.
+    Args:
+        extract_key (RecordUniqueKeyExtractor): A function that takes a
+            dictionary record and returns a value used to determine uniqueness.
+    Returns:
+        UniqueRecordStream: A coroutine that accepts an async iterator of dictionaries,
+            yields only unique records, as determined by the extracted key.
+    """
+
+    async def deduplicator(records: RecordStream) -> RecordStream:
+        """
+        Deduplicate async iterator of dicts by a key extracted from each record.
+
+        Args:
+            records (RecordStream): Async iterator of records to deduplicate.
+
+        Yields:
+            EquityRecord: Unique records, as determined by the extracted key.
+        """
+        seen: set[object] = set()
+        async for record in records:
+            key = extract_key(record)
+            if key in seen:
+                continue
+            seen.add(key)
+            yield record
+
+    return deduplicator

equity_aggregator/adapters/data_sources/discovery_feeds/tradingview/__init__.py

@@ -0,0 +1,5 @@
+from .tradingview import fetch_equity_records
+
+__all__ = [
+    "fetch_equity_records",
+]