equity-aggregator 0.1.1__py3-none-any.whl → 0.1.5__py3-none-any.whl

equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/auth.py

@@ -0,0 +1,130 @@
+# yfinance/auth.py
+
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+
+import httpx
+
+logger: logging.Logger = logging.getLogger(__name__)
+
+# Type alias for the fetch function signature
+FetchFn = Callable[[str, dict[str, str]], Awaitable[httpx.Response]]
+
+
+class CrumbManager:
+    """
+    Manages Yahoo Finance anti-CSRF crumb lifecycle.
+
+    Handles crumb acquisition with double-checked locking for thread safety.
+    The crumb is lazily fetched on first use and cleared when the underlying
+    HTTP client is reset.
+
+    Args:
+        crumb_url (str): URL to fetch the crumb from.
+
+    Returns:
+        None
+    """
+
+    __slots__ = ("_crumb", "_crumb_url", "_lock")
+
+    def __init__(self, crumb_url: str) -> None:
+        """
+        Initialise CrumbManager with the crumb endpoint URL.
+
+        Args:
+            crumb_url (str): URL to fetch the crumb from.
+
+        Returns:
+            None
+        """
+        self._crumb: str | None = None
+        self._crumb_url: str = crumb_url
+        self._lock: asyncio.Lock = asyncio.Lock()
+
+    @property
+    def crumb(self) -> str | None:
+        """
+        Get the current crumb value, if available.
+
+        Args:
+            None
+
+        Returns:
+            str | None: The cached crumb, or None if not yet fetched.
+        """
+        return self._crumb
+
+    def clear(self) -> None:
+        """
+        Clear the cached crumb.
+
+        Called when the HTTP client is reset, as the crumb is tied to
+        session cookies that are invalidated on client replacement.
+
+        Args:
+            None
+
+        Returns:
+            None
+        """
+        self._crumb = None
+
+    async def ensure_crumb(self, ticker: str, fetch: FetchFn) -> str:
+        """
+        Ensure a valid crumb is available, bootstrapping if necessary.
+
+        Uses double-checked locking: fast path returns cached crumb,
+        slow path acquires lock and bootstraps session if needed.
+
+        Args:
+            ticker (str): Symbol to use for session priming requests.
+            fetch (FetchFn): Async function to perform HTTP GET requests.
+
+        Returns:
+            str: Valid crumb string.
+
+        Raises:
+            httpx.HTTPStatusError: If crumb fetch fails.
+        """
+        if self._crumb is not None:
+            return self._crumb
+
+        async with self._lock:
+            if self._crumb is not None:
+                return self._crumb
+
+            await self._bootstrap(ticker, fetch)
+            return self._crumb
+
+    async def _bootstrap(self, ticker: str, fetch: FetchFn) -> None:
+        """
+        Prime session cookies and fetch the crumb.
+
+        Makes requests to Yahoo Finance endpoints to establish session
+        cookies, then fetches the crumb from the crumb endpoint.
+
+        Args:
+            ticker (str): Symbol to use for session priming.
+            fetch (FetchFn): Async function to perform HTTP GET requests.
+
+        Returns:
+            None
+
+        Raises:
+            httpx.HTTPStatusError: If crumb fetch fails.
+        """
+        seeds: tuple[str, ...] = (
+            "https://fc.yahoo.com",
+            "https://finance.yahoo.com",
+            f"https://finance.yahoo.com/quote/{ticker}",
+        )
+
+        for seed in seeds:
+            await fetch(seed, {})
+
+        response: httpx.Response = await fetch(self._crumb_url, {})
+        response.raise_for_status()
+
+        self._crumb = response.text.strip().strip('"')

equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/config.py

@@ -25,14 +25,14 @@ class FeedConfig:
     """
 
     # search URL for searching equities by symbol, name, ISIN, or CUSIP
-    search_url: str = "https://query1.finance.yahoo.com/v1/finance/search"
+    search_url: str = "https://query2.finance.yahoo.com/v1/finance/search"
 
     # quote summary URL for fetching equity data
-    quote_summary_url: str = (
+    quote_summary_primary_url: str = (
         "https://query2.finance.yahoo.com/v10/finance/quoteSummary/"
     )
 
-    # quote summary URL fallback endpoint for fetching equity data
+    # fallback quote URL for simpler equity data
     quote_summary_fallback_url: str = (
         "https://query1.finance.yahoo.com/v7/finance/quote"
     )

equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/ranking.py

@@ -0,0 +1,97 @@
+# yfinance/ranking.py
+
+from ._utils import rank_all_symbols
+
+
+def filter_equities(quotes: list[dict]) -> list[dict]:
+    """
+    Filter out any quotes lacking a longname or symbol.
+
+    Note:
+        The Yahoo Finance search quote query endpoint returns 'longname' and 'shortname'
+        fields in lowercase.
+
+    Args:
+        quotes (list[dict]): Raw list of quote dicts from Yahoo Finance.
+
+    Returns:
+        list[dict]: Only those quotes that have both 'longname' and 'symbol'.
+    """
+    return [
+        quote
+        for quote in quotes
+        if (quote.get("longname") or quote.get("shortname")) and quote.get("symbol")
+    ]
+
+
+def rank_symbols(
+    viable: list[dict],
+    *,
+    expected_name: str,
+    expected_symbol: str,
+    min_score: int,
+) -> list[str]:
+    """
+    Rank Yahoo Finance quote candidates by fuzzy match quality.
+
+    Returns ALL viable candidates as a ranked list ordered by match confidence
+    (best match first), filtered by minimum score threshold. All candidates are
+    scored and validated, even if there's only one or they share identical names.
+
+    Args:
+        viable (list[dict]): List of filtered Yahoo Finance quote dictionaries.
+        expected_name (str): Expected company or equity name for fuzzy matching.
+        expected_symbol (str): Expected ticker symbol for fuzzy matching.
+        min_score (int): Minimum fuzzy score required to accept a match.
+
+    Returns:
+        list[str]: Ranked symbols (best first), empty if none meet threshold.
+    """
+    # Try longname first, then shortname
+    for name_key in ("longname", "shortname"):
+        ranked = rank_by_name_key(
+            viable,
+            name_key=name_key,
+            expected_name=expected_name,
+            expected_symbol=expected_symbol,
+            min_score=min_score,
+        )
+        if ranked:
+            return ranked
+
+    return []
+
+
+def rank_by_name_key(
+    viable: list[dict],
+    *,
+    name_key: str,
+    expected_name: str,
+    expected_symbol: str,
+    min_score: int,
+) -> list[str]:
+    """
+    Rank symbols using specified name field (longname or shortname).
+
+    Args:
+        viable (list[dict]): List of quote dictionaries to rank.
+        name_key (str): The key to use for name comparison.
+        expected_name (str): Expected company or equity name.
+        expected_symbol (str): Expected ticker symbol.
+        min_score (int): Minimum fuzzy score threshold.
+
+    Returns:
+        list[str]: Ranked symbols, or empty list if no matches meet threshold.
+    """
+    candidates_with_name = [quote for quote in viable if quote.get(name_key)]
+
+    if not candidates_with_name:
+        return []
+
+    return rank_all_symbols(
+        candidates_with_name,
+        name_key=name_key,
+        expected_name=expected_name,
+        expected_symbol=expected_symbol,
+        min_score=min_score,
+    )

equity_aggregator/adapters/data_sources/enrichment_feeds/yfinance/session.py

@@ -6,10 +6,10 @@ from collections.abc import Mapping
 
 import httpx
 
-from equity_aggregator.adapters.data_sources._utils import make_client
-
+from ._utils import backoff_delays
+from .auth import CrumbManager
 from .config import FeedConfig
-from .utils import backoff_delays
+from .transport import HttpTransport
 
 logger: logging.Logger = logging.getLogger(__name__)
 
@@ -18,10 +18,9 @@ class YFSession:
     """
     Asynchronous session for Yahoo Finance JSON endpoints.
 
-    This class manages HTTP requests to Yahoo Finance, handling authentication,
-    rate limits, and crumb renewal. It is lightweight and reusable, maintaining
-    only a client and session state. Concurrency is limited by a shared
-    semaphore to respect Yahoo's HTTP/2 stream restriction.
+    Composes HttpTransport for connection management, CrumbManager for
+    authentication, and applies retry policies for rate limiting.
+    Concurrency is limited by a shared semaphore.
 
     Args:
         config (FeedConfig): Immutable feed configuration.
@@ -31,13 +30,19 @@ class YFSession:
         None
     """
 
-    __slots__: tuple[str, ...] = ("_client", "_config", "_crumb", "_crumb_lock")
+    __slots__ = ("_auth", "_config", "_transport")
+
+    # Limit HTTP/2 concurrent streams to 10 for maximum throughput.
+    _concurrent_streams: asyncio.Semaphore = asyncio.Semaphore(10)
 
-    # Limit HTTP/2 concurrent streams to 1; in effect serialising requests.
-    # This is workaround to alleviate reliability issues with Yahoo Finance's endpoints.
-    # With this approach, the HTTP/2 protocol is still in use, without having to drop to
-    # HTTP/1.1.
-    _concurrent_streams: asyncio.Semaphore = asyncio.Semaphore(1)
+    _RETRYABLE_STATUS_CODES: frozenset[int] = frozenset(
+        {
+            httpx.codes.TOO_MANY_REQUESTS,  # 429
+            httpx.codes.BAD_GATEWAY,  # 502
+            httpx.codes.SERVICE_UNAVAILABLE,  # 503
+            httpx.codes.GATEWAY_TIMEOUT,  # 504
+        },
+    )
 
     def __init__(
         self,
@@ -45,7 +50,7 @@ class YFSession:
         client: httpx.AsyncClient | None = None,
     ) -> None:
         """
-        Initialise a new YFSession for Yahoo Finance JSON endpoints.
+        Initialise YFSession with configuration.
 
         Args:
             config (FeedConfig): Immutable feed configuration.
@@ -55,26 +60,28 @@ class YFSession:
             None
         """
         self._config: FeedConfig = config
-        self._client: httpx.AsyncClient = client or make_client()
-        self._crumb: str | None = None
-        self._crumb_lock: asyncio.Lock = asyncio.Lock()
+        self._auth: CrumbManager = CrumbManager(config.crumb_url)
+        self._transport: HttpTransport = HttpTransport(
+            client=client,
+            on_reset=self._auth.clear,
+        )
 
     @property
     def config(self) -> FeedConfig:
         """
-        Gets the immutable configuration associated with this session.
+        Get the immutable configuration associated with this session.
 
         Args:
             None
 
         Returns:
-            FeedConfig: The configuration object bound to this session instance.
+            FeedConfig: The configuration object bound to this session.
         """
         return self._config
 
     async def aclose(self) -> None:
         """
-        Asynchronously close the underlying HTTP client.
+        Close the underlying HTTP transport.
 
         Args:
             None
@@ -82,7 +89,7 @@ class YFSession:
         Returns:
             None
         """
-        await self._client.aclose()
+        await self._transport.aclose()
 
     async def get(
         self,
@@ -93,9 +100,12 @@ class YFSession:
         """
         Perform a resilient asynchronous GET request to Yahoo Finance endpoints.
 
-        This method injects the crumb if required, renews it on a single 401
-        response, and applies exponential backoff on 429 responses. Concurrency
-        is limited to comply with Yahoo's HTTP/2 stream limits.
+        This method renews the crumb on a single 401 response and applies
+        exponential backoff on 429 responses. Concurrency is limited to comply
+        with Yahoo's HTTP/2 stream limits.
+
+        All httpx exceptions are converted to LookupError for consistent
+        error handling at the domain boundary.
 
         Args:
             url (str): Absolute URL to request.
@@ -103,121 +113,93 @@ class YFSession:
 
         Returns:
             httpx.Response: The successful HTTP response.
+
+        Raises:
+            LookupError: If the request fails due to network or HTTP errors.
         """
         async with self.__class__._concurrent_streams:
-            merged_params: dict[str, str] = self._attach_crumb(url, dict(params or {}))
-            return await self._fetch_with_retry(url, merged_params)
+            params_dict: dict[str, str] = dict(params or {})
+
+            try:
+                return await self._fetch_with_retry(url, params_dict)
+            except httpx.HTTPError as error:
+                raise LookupError("Request failed") from error
 
-    async def _safe_get(
+    async def _fetch_with_retry(
         self,
         url: str,
         params: dict[str, str],
         *,
-        retries: int = 3,
+        delays: list[float] | None = None,
     ) -> httpx.Response:
         """
-        Perform a GET request with up to 3 retries on HTTP/2 protocol errors.
+        Perform GET request with unified 401 and rate limit handling.
 
-        If a ProtocolError occurs, the HTTP client is reset and the request
-        retried. After 3 failed attempts, the last exception is raised.
+        Each attempt (initial + retries) passes through the full response handling
+        chain: connection retry → 401 check/crumb renewal → retryable status check.
+        This ensures that if a retry hits 401 (e.g., crumb cleared by client reset),
+        the crumb is renewed before continuing.
 
         Args:
             url (str): The absolute URL to request.
-            params (dict[str, str]): Query parameters for the request.
+            params (dict[str, str]): Query parameters (mutated with crumb).
+            delays (list[float] | None): Optional delay sequence for testing.
+                If None, uses exponential backoff with 5 retry attempts.
 
         Returns:
             httpx.Response: The successful HTTP response.
 
         Raises:
-            httpx.ProtocolError: If all retry attempts fail.
+            LookupError: If response is still retryable after all attempts.
         """
-        last_exc: Exception | None = None
+        max_backoff_attempts = 5
 
-        for attempt in range(1, retries + 1):
-            try:
-                return await self._client.get(url, params=params)
-            except httpx.ProtocolError as exc:
-                last_exc = exc
+        if delays is None:
+            delays = [0, *backoff_delays(attempts=max_backoff_attempts)]
+
+        for backoff_attempt, delay in enumerate(delays):
+            if delay > 0:
                 logger.debug(
-                    (
-                        "HTTP/2 protocol error (%s) on %s - "
-                        "recreating client (attempt %d/3)"
-                    ),
-                    exc,
-                    url,
-                    attempt,
+                    "RATE_LIMIT: YFinance feed data request paused. "
+                    "Retrying in %.1fs (attempt %d/%d)",
+                    delay,
+                    backoff_attempt,
+                    max_backoff_attempts,
                 )
-                await self._reset_client()
+                await asyncio.sleep(delay)
 
-        assert last_exc is not None
-        raise last_exc
+            response = await self._attempt_request(url, params)
 
-    async def _fetch_with_retry(
+            # If response is not retryable, return it (success or permanent error)
+            if response.status_code not in self._RETRYABLE_STATUS_CODES:
+                return response
+
+        # All attempts exhausted, response still retryable
+        raise LookupError(f"HTTP {response.status_code} after retries for {url}")
+
+    async def _attempt_request(
         self,
         url: str,
-        params: Mapping[str, str],
+        params: dict[str, str],
     ) -> httpx.Response:
         """
-        Perform a single GET request, transparently handling 401 and invalid responses.
-
-        If a 401 (Unauthorized) is received, the crumb is renewed and the request
-        retried once. If other 4xx or 5xx responses are received, exponential backoff
-        is applied and the request is retried up to the configured limit.
+        Perform a single request attempt with 401 handling.
 
         Args:
             url (str): The absolute URL to request.
-            params (Mapping[str, str]): Query parameters after crumb injection.
+            params (dict[str, str]): Query parameters (mutated with crumb on 401).
 
         Returns:
-            httpx.Response: The successful HTTP response.
-
-        Raises:
-            httpx.HTTPStatusError: If the final response is not successful.
+            httpx.Response: The HTTP response.
         """
-        # make mutable copy
-        params = dict(params)
-
-        response = await self._safe_get(url, params)
+        response = await self._transport.get(url, params)
 
-        # handle 401 Unauthorized
+        # Handle 401 by renewing crumb (could happen after client reset)
         if response.status_code == httpx.codes.UNAUTHORIZED:
             response = await self._renew_crumb_once(url, params)
 
-        # Retry transient server / rate‑limit responses
-        retryable = {
-            httpx.codes.TOO_MANY_REQUESTS,  # 429
-            httpx.codes.INTERNAL_SERVER_ERROR,  # 500
-            httpx.codes.BAD_GATEWAY,  # 502
-            httpx.codes.SERVICE_UNAVAILABLE,  # 503
-            httpx.codes.GATEWAY_TIMEOUT,  # 504
-        }
-
-        if response.status_code in retryable:
-            response = await self._get_with_backoff(url, params, response)
-
-        # If the response is still not successful after retries, raise an error
-        if response.status_code in retryable:
-            raise LookupError(f"HTTP {response.status_code} after retries for {url}")
-
         return response
 
-    async def _reset_client(self) -> None:
-        """
-        Reset the HTTP client instance asynchronously.
-
-        Closes the current client and creates a new one. Also clears the crumb
-        to ensure session state is refreshed after protocol errors.
-
-        Args:
-            None
-
-        Returns:
-            None
-        """
-        self._crumb = None
-        await self._client.aclose()
-        self._client = make_client()
-
     async def _renew_crumb_once(
         self,
         url: str,
@@ -238,94 +220,11 @@ class YFSession:
         """
         ticker: str = self._extract_ticker(url)
 
-        await self._bootstrap_and_fetch_crumb(ticker)
-
-        params["crumb"] = self._crumb
-
-        return await self._client.get(url, params=params)
-
-    async def _get_with_backoff(
-        self,
-        url: str,
-        params: dict[str, str],
-        response: httpx.Response,
-    ) -> httpx.Response:
-        """
-        Retry a GET request after receiving any 4xx or 5xx response,
-        using exponential backoff.
-
-        Retries up to `max_attempts` times, waiting for delays generated by
-        `backoff_delays()`. Each retry uses `_safe_get`, which handles protocol
-        errors. If a non-retryable response is received, it is returned immediately.
-
-        Args:
-            url (str): The absolute URL to request.
-            params (dict[str, str]): Query parameters for the request.
-            response (httpx.Response): The initial 429 response.
-
-        Returns:
-            httpx.Response: The successful HTTP response or the last response
-                after all retries.
-        """
-        max_attempts = 5
-
-        retryable = {
-            httpx.codes.TOO_MANY_REQUESTS,  # 429
-            httpx.codes.INTERNAL_SERVER_ERROR,  # 500
-            httpx.codes.BAD_GATEWAY,  # 502
-            httpx.codes.SERVICE_UNAVAILABLE,  # 503
-            httpx.codes.GATEWAY_TIMEOUT,  # 504
-        }
-
-        for attempt, delay in enumerate(backoff_delays(attempts=max_attempts), 1):
-            if response.status_code not in retryable:
-                return response
-
-            logger.debug(
-                "%d %s – sleeping %.1fs (attempt %d/%d)",
-                response.status_code,
-                url,
-                delay,
-                attempt,
-                max_attempts,
-            )
-            await asyncio.sleep(delay)
-
-            try:
-                response = await self._safe_get(url, params)
-            except httpx.ProtocolError:
-                raise
+        crumb = await self._auth.ensure_crumb(ticker, self._transport.get)
 
-        return response
-
-    def _attach_crumb(
-        self,
-        url: str,
-        params: dict[str, str],
-    ) -> dict[str, str]:
-        """
-        Inject the anti-CSRF crumb into query parameters if required.
-
-        The crumb is added only for quote-summary endpoint requests when available.
-        If the crumb is not set or the URL does not match, the original parameters
-        are returned unchanged.
-
-        Args:
-            url (str): Target request URL.
-            params (dict[str, str]): Query parameters to update.
-
-        Returns:
-            dict[str, str]: Updated query parameters with crumb if needed.
-        """
-        # needs_crumb = self._crumb is not None and url.startswith(
-        #     self._config.quote_summary_url,
-        # )
-
-        # if not needs_crumb:
-        #     return params
+        params["crumb"] = crumb
 
-        # return {**params, "crumb": self._crumb}
-        return params
+        return await self._transport.get(url, params)
 
     def _extract_ticker(self, url: str) -> str:
         """
@@ -337,40 +236,8 @@ class YFSession:
         Returns:
             str: The ticker symbol found in the URL path.
         """
-        remainder: str = url[len(self._config.quote_summary_url) :]
+        remainder: str = url[len(self._config.quote_summary_primary_url) :]
 
         first_segment: str = remainder.split("/", 1)[0]
 
         return first_segment.split("?", 1)[0].split("#", 1)[0]
-
-    async def _bootstrap_and_fetch_crumb(self, ticker: str) -> None:
-        """
-        Initialise session cookies and retrieve the anti-CSRF crumb.
-
-        This method primes the session by making requests to Yahoo Finance endpoints
-        using the provided ticker, then fetches the crumb required for authenticated
-        requests. The crumb is cached for future use and protected by a lock.
-
-        Args:
-            ticker (str): Symbol used to prime the session.
-
-        Returns:
-            None
-        """
-        if self._crumb is not None:
-            return
-
-        async with self._crumb_lock:
-            if self._crumb is not None:
-                return
-            seeds: tuple[str, ...] = (
-                "https://fc.yahoo.com",
-                "https://finance.yahoo.com",
-                f"https://finance.yahoo.com/quote/{ticker}",
-            )
-            for seed in seeds:
-                await self._client.get(seed)
-
-            resp: httpx.Response = await self._client.get(self._config.crumb_url)
-            resp.raise_for_status()
-            self._crumb = resp.text.strip().strip('"')