second-read 0.1.1__py3-none-any.whl

secondread/providers/search/base.py

@@ -0,0 +1,218 @@
+"""Shared HTTP plumbing for search adapters.
+
+Each adapter subclasses ``HttpSearchClient`` and implements ``_build_request``
+(what to send) and ``_parse`` (JSON to ``SearchResult``s). The base class
+handles retries, error mapping, result cleanup and key hygiene: the key is
+held in a ``Secret`` whose ``repr`` is masked and it never appears in error
+messages.
+"""
+
+from __future__ import annotations
+
+import html
+import re
+from abc import ABC, abstractmethod
+from dataclasses import dataclass, field
+from types import TracebackType
+from typing import Any, Self
+from urllib.parse import urlsplit
+
+import httpx
+
+from secondread.contracts import AuthError, ProviderError, RateLimitError, SearchResult
+from secondread.providers.search.retry import RetryPolicy, parse_retry_after
+
+MAX_SNIPPET_CHARS = 500
+MAX_TITLE_CHARS = 300
+DEFAULT_TIMEOUT = httpx.Timeout(10.0, connect=5.0)
+USER_AGENT = "SecondRead/0.1 (document reviewer)"
+
+_TAG_RE = re.compile(r"<[^>]+>")
+_WS_RE = re.compile(r"\s+")
+
+
+class Secret:
+    """Holds an API key. ``repr`` and ``str`` are masked."""
+
+    __slots__ = ("_value",)
+
+    def __init__(self, value: str) -> None:
+        self._value = value
+
+    def reveal(self) -> str:
+        """Return the raw key (only for building request headers)."""
+        return self._value
+
+    def __repr__(self) -> str:
+        return "Secret('***')"
+
+    __str__ = __repr__
+
+
+@dataclass(frozen=True)
+class SearchRequest:
+    """One HTTP request an adapter wants sent."""
+
+    method: str
+    url: str
+    headers: dict[str, str] = field(default_factory=dict)
+    params: dict[str, str | int] | None = None
+    json: dict[str, Any] | None = None
+
+    def __repr__(self) -> str:
+        # Headers carry the API key; never show them.
+        return f"SearchRequest(method={self.method!r}, url={self.url!r})"
+
+
+def clean_text(value: object, limit: int) -> str:
+    """Strip tags and entities, collapse whitespace and truncate."""
+    if not isinstance(value, str):
+        return ""
+    text = _WS_RE.sub(" ", html.unescape(_TAG_RE.sub("", value))).strip()
+    if len(text) > limit:
+        text = text[: limit - 1].rstrip() + "…"
+    return text
+
+
+def is_http_url(value: object) -> bool:
+    """True for absolute ``http``/``https`` URLs with a host."""
+    if not isinstance(value, str):
+        return False
+    try:
+        parts = urlsplit(value)
+    except ValueError:
+        return False
+    return parts.scheme in ("http", "https") and bool(parts.netloc)
+
+
+class HttpSearchClient(ABC):
+    """Base ``SearchClient`` over a JSON HTTP API.
+
+    ``transport`` lets tests inject ``httpx.MockTransport``; ``retry`` controls
+    backoff (inject a no-op sleep in tests). Use as an async context manager
+    or call ``aclose`` to release connections.
+    """
+
+    #: Provider name used in error messages and the comparison script.
+    name: str = "search"
+    #: Largest ``k`` the API accepts in one call.
+    max_k: int = 20
+
+    def __init__(
+        self,
+        api_key: str,
+        *,
+        transport: httpx.AsyncBaseTransport | None = None,
+        retry: RetryPolicy | None = None,
+        timeout: httpx.Timeout | float = DEFAULT_TIMEOUT,
+        base_url: str | None = None,
+    ) -> None:
+        if not api_key or not api_key.strip():
+            raise AuthError(f"{self.name}: missing API key")
+        self._key = Secret(api_key.strip())
+        self._retry = retry or RetryPolicy()
+        self._base_url = base_url
+        self._client = httpx.AsyncClient(
+            transport=transport,
+            timeout=timeout,
+            headers={"User-Agent": USER_AGENT},
+            trust_env=False,
+            follow_redirects=False,
+        )
+
+    def __repr__(self) -> str:
+        return f"{type(self).__name__}(api_key=***)"
+
+    async def __aenter__(self) -> Self:
+        return self
+
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        await self.aclose()
+
+    async def aclose(self) -> None:
+        """Close the underlying HTTP client."""
+        await self._client.aclose()
+
+    @abstractmethod
+    def _build_request(self, query: str, k: int) -> SearchRequest:
+        """Return the HTTP request for ``query`` asking for ``k`` hits."""
+
+    @abstractmethod
+    def _parse(self, data: Any) -> list[SearchResult]:
+        """Turn the decoded JSON body into results (ranks are reassigned)."""
+
+    async def search(self, query: str, *, k: int = 5) -> list[SearchResult]:
+        """Return at most ``k`` results for ``query``, best first."""
+        query = query.strip()
+        if not query or k < 1:
+            return []
+        request = self._build_request(query, min(k, self.max_k))
+        data = await self._send(request)
+        try:
+            raw = self._parse(data)
+        except (KeyError, TypeError, ValueError, AttributeError) as exc:
+            raise ProviderError(f"{self.name}: unexpected response shape") from exc
+        return _finalize(raw, k)
+
+    async def _send(self, request: SearchRequest) -> Any:
+        last_error: ProviderError | None = None
+        for attempt in range(self._retry.attempts):
+            retry_after: float | None = None
+            try:
+                response = await self._client.request(
+                    request.method,
+                    request.url,
+                    headers=request.headers,
+                    params=request.params,
+                    json=request.json,
+                )
+            except httpx.TimeoutException:
+                last_error = ProviderError(f"{self.name}: request timed out", retryable=True)
+            except httpx.TransportError:
+                last_error = ProviderError(f"{self.name}: connection failed", retryable=True)
+            else:
+                status = response.status_code
+                if status in (401, 403):
+                    raise AuthError(f"{self.name}: API key rejected (HTTP {status})")
+                if status == 402:
+                    raise AuthError(f"{self.name}: out of credit or plan limit (HTTP 402)")
+                if status == 429:
+                    retry_after = parse_retry_after(response.headers.get("Retry-After"))
+                    last_error = RateLimitError(f"{self.name}: rate limited (HTTP 429)")
+                elif status >= 500:
+                    last_error = ProviderError(
+                        f"{self.name}: server error (HTTP {status})", retryable=True
+                    )
+                elif status >= 400:
+                    raise ProviderError(f"{self.name}: request rejected (HTTP {status})")
+                else:
+                    try:
+                        return response.json()
+                    except ValueError as exc:
+                        raise ProviderError(f"{self.name}: response was not JSON") from exc
+            if attempt + 1 < self._retry.attempts:
+                await self._retry.wait(attempt, retry_after)
+        assert last_error is not None
+        raise last_error
+
+    def _url(self, default: str) -> str:
+        return self._base_url or default
+
+
+def _finalize(raw: list[SearchResult], k: int) -> list[SearchResult]:
+    """Drop non-HTTP and duplicate URLs, cap at ``k``, and renumber ranks."""
+    seen: set[str] = set()
+    out: list[SearchResult] = []
+    for hit in raw:
+        if not is_http_url(hit.url) or hit.url in seen:
+            continue
+        seen.add(hit.url)
+        out.append(hit.model_copy(update={"rank": len(out) + 1}))
+        if len(out) >= k:
+            break
+    return out

secondread/providers/search/brave.py

@@ -0,0 +1,55 @@
+"""Brave Search API adapter (``GET /res/v1/web/search``).
+
+Auth: ``X-Subscription-Token`` header. Snippets arrive with ``<strong>``
+highlighting, which is stripped.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from secondread.contracts import SearchResult
+from secondread.providers.search.base import (
+    MAX_SNIPPET_CHARS,
+    MAX_TITLE_CHARS,
+    HttpSearchClient,
+    SearchRequest,
+    clean_text,
+)
+
+BRAVE_URL = "https://api.search.brave.com/res/v1/web/search"
+
+
+class BraveSearch(HttpSearchClient):
+    """``SearchClient`` backed by the Brave Search web endpoint."""
+
+    name = "brave"
+    max_k = 20
+
+    def _build_request(self, query: str, k: int) -> SearchRequest:
+        return SearchRequest(
+            method="GET",
+            url=self._url(BRAVE_URL),
+            headers={
+                "Accept": "application/json",
+                "X-Subscription-Token": self._key.reveal(),
+            },
+            params={"q": query, "count": k},
+        )
+
+    def _parse(self, data: Any) -> list[SearchResult]:
+        web = data.get("web") or {}
+        results: list[SearchResult] = []
+        for item in web.get("results") or []:
+            if not isinstance(item, dict):
+                continue
+            published = item.get("page_age") or item.get("age")
+            results.append(
+                SearchResult(
+                    url=str(item.get("url", "")),
+                    title=clean_text(item.get("title"), MAX_TITLE_CHARS),
+                    snippet=clean_text(item.get("description"), MAX_SNIPPET_CHARS),
+                    published=published if isinstance(published, str) else None,
+                )
+            )
+        return results

secondread/providers/search/config.py

@@ -0,0 +1,63 @@
+"""Build search clients from environment variables.
+
+| Provider | Env var | Class |
+| --- | --- | --- |
+| ``brave`` | ``BRAVE_SEARCH_API_KEY`` | ``BraveSearch`` |
+| ``tavily`` | ``TAVILY_API_KEY`` | ``TavilySearch`` |
+| ``exa`` | ``EXA_API_KEY`` | ``ExaSearch`` |
+
+Providers whose variable is unset or blank are skipped.
+"""
+
+from __future__ import annotations
+
+import os
+from collections.abc import Mapping
+from typing import Any
+
+from secondread.contracts import MissingSetup
+from secondread.providers.search.base import HttpSearchClient
+from secondread.providers.search.brave import BraveSearch
+from secondread.providers.search.exa import ExaSearch
+from secondread.providers.search.tavily import TavilySearch
+
+PROVIDERS: dict[str, tuple[str, type[HttpSearchClient]]] = {
+    "brave": ("BRAVE_SEARCH_API_KEY", BraveSearch),
+    "tavily": ("TAVILY_API_KEY", TavilySearch),
+    "exa": ("EXA_API_KEY", ExaSearch),
+}
+"""Provider name -> (env var holding its key, adapter class)."""
+
+
+def make_search_client(name: str, api_key: str, **kwargs: Any) -> HttpSearchClient:
+    """Build the adapter called ``name`` with ``api_key``."""
+    if name not in PROVIDERS:
+        known = ", ".join(PROVIDERS)
+        raise MissingSetup(f"unknown search provider {name!r} (known: {known})")
+    _env_var, cls = PROVIDERS[name]
+    return cls(api_key, **kwargs)
+
+
+def search_clients_from_env(
+    env: Mapping[str, str] | None = None,
+    *,
+    only: list[str] | None = None,
+    **kwargs: Any,
+) -> dict[str, HttpSearchClient]:
+    """Build every provider whose key is set in ``env`` (default ``os.environ``).
+
+    ``only`` restricts to the named providers; ``kwargs`` go to each adapter
+    (e.g. ``transport`` or ``retry``).
+    """
+    source: Mapping[str, str] = os.environ if env is None else env
+    names = only if only is not None else list(PROVIDERS)
+    clients: dict[str, HttpSearchClient] = {}
+    for name in names:
+        if name not in PROVIDERS:
+            known = ", ".join(PROVIDERS)
+            raise MissingSetup(f"unknown search provider {name!r} (known: {known})")
+        env_var, cls = PROVIDERS[name]
+        key = source.get(env_var, "").strip()
+        if key:
+            clients[name] = cls(key, **kwargs)
+    return clients

secondread/providers/search/exa.py

@@ -0,0 +1,62 @@
+"""Exa search API adapter (``POST /search``).
+
+Auth: ``x-api-key`` header. Requests ``type=auto`` with highlights, which
+become the snippet (falling back to the first part of ``text``).
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from secondread.contracts import SearchResult
+from secondread.providers.search.base import (
+    MAX_SNIPPET_CHARS,
+    MAX_TITLE_CHARS,
+    HttpSearchClient,
+    SearchRequest,
+    clean_text,
+)
+
+EXA_URL = "https://api.exa.ai/search"
+
+
+class ExaSearch(HttpSearchClient):
+    """``SearchClient`` backed by Exa's search endpoint."""
+
+    name = "exa"
+    max_k = 25
+
+    def _build_request(self, query: str, k: int) -> SearchRequest:
+        return SearchRequest(
+            method="POST",
+            url=self._url(EXA_URL),
+            headers={"Accept": "application/json", "x-api-key": self._key.reveal()},
+            json={
+                "query": query,
+                "numResults": k,
+                "type": "auto",
+                "contents": {"highlights": {"numSentences": 2, "highlightsPerUrl": 2}},
+            },
+        )
+
+    def _parse(self, data: Any) -> list[SearchResult]:
+        results: list[SearchResult] = []
+        for item in data.get("results") or []:
+            if not isinstance(item, dict):
+                continue
+            highlights = item.get("highlights")
+            snippet_src: object
+            if isinstance(highlights, list) and highlights:
+                snippet_src = " … ".join(h for h in highlights if isinstance(h, str))
+            else:
+                snippet_src = item.get("text")
+            published = item.get("publishedDate")
+            results.append(
+                SearchResult(
+                    url=str(item.get("url", "")),
+                    title=clean_text(item.get("title"), MAX_TITLE_CHARS),
+                    snippet=clean_text(snippet_src, MAX_SNIPPET_CHARS),
+                    published=published if isinstance(published, str) else None,
+                )
+            )
+        return results

secondread/providers/search/retry.py

@@ -0,0 +1,60 @@
+"""Retry policy shared by the search adapters and the fetcher.
+
+The sleep function is injectable so tests run instantly: pass
+``RetryPolicy(sleep=no_sleep)`` or any coroutine that records delays.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Awaitable, Callable
+from dataclasses import dataclass, field
+
+Sleep = Callable[[float], Awaitable[None]]
+
+
+async def no_sleep(_seconds: float) -> None:
+    """A sleep that returns immediately (for tests)."""
+
+
+@dataclass(frozen=True)
+class RetryPolicy:
+    """Exponential backoff for transient failures.
+
+    ``attempts`` is the total number of tries (1 means no retry). The delay
+    before retry ``n`` (0-based) is ``base_delay * 2**n``, capped at
+    ``max_delay``. A server-sent ``Retry-After`` replaces the computed delay
+    but is still capped, so a hostile or confused server cannot stall a job.
+    """
+
+    attempts: int = 3
+    base_delay: float = 0.5
+    max_delay: float = 8.0
+    sleep: Sleep = field(default=asyncio.sleep, compare=False)
+
+    def __post_init__(self) -> None:
+        if self.attempts < 1:
+            raise ValueError("attempts must be at least 1")
+        if self.base_delay < 0 or self.max_delay < 0:
+            raise ValueError("delays must be non-negative")
+
+    def delay(self, retry_index: int, retry_after: float | None = None) -> float:
+        """Seconds to wait before retry number ``retry_index`` (0-based)."""
+        if retry_after is not None and retry_after >= 0:
+            return min(self.max_delay, retry_after)
+        return min(self.max_delay, self.base_delay * float(2**retry_index))
+
+    async def wait(self, retry_index: int, retry_after: float | None = None) -> None:
+        """Sleep for ``delay(retry_index, retry_after)`` using the injected sleep."""
+        await self.sleep(self.delay(retry_index, retry_after))
+
+
+def parse_retry_after(value: str | None) -> float | None:
+    """Parse a ``Retry-After`` header given in seconds; dates are ignored."""
+    if not value:
+        return None
+    try:
+        seconds = float(value.strip())
+    except ValueError:
+        return None
+    return seconds if seconds >= 0 else None

secondread/providers/search/tavily.py

@@ -0,0 +1,60 @@
+"""Tavily Search API adapter (``POST /search``).
+
+Auth: ``Authorization: Bearer <key>``. Uses ``search_depth=basic`` (the
+cheaper tier); ``content`` becomes the snippet.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from secondread.contracts import SearchResult
+from secondread.providers.search.base import (
+    MAX_SNIPPET_CHARS,
+    MAX_TITLE_CHARS,
+    HttpSearchClient,
+    SearchRequest,
+    clean_text,
+)
+
+TAVILY_URL = "https://api.tavily.com/search"
+
+
+class TavilySearch(HttpSearchClient):
+    """``SearchClient`` backed by Tavily's search endpoint."""
+
+    name = "tavily"
+    max_k = 20
+
+    def _build_request(self, query: str, k: int) -> SearchRequest:
+        return SearchRequest(
+            method="POST",
+            url=self._url(TAVILY_URL),
+            headers={
+                "Accept": "application/json",
+                "Authorization": f"Bearer {self._key.reveal()}",
+            },
+            json={
+                "query": query,
+                "max_results": k,
+                "search_depth": "basic",
+                "include_answer": False,
+                "include_raw_content": False,
+            },
+        )
+
+    def _parse(self, data: Any) -> list[SearchResult]:
+        results: list[SearchResult] = []
+        for item in data.get("results") or []:
+            if not isinstance(item, dict):
+                continue
+            published = item.get("published_date")
+            results.append(
+                SearchResult(
+                    url=str(item.get("url", "")),
+                    title=clean_text(item.get("title"), MAX_TITLE_CHARS),
+                    snippet=clean_text(item.get("content"), MAX_SNIPPET_CHARS),
+                    published=published if isinstance(published, str) else None,
+                )
+            )
+        return results