reefapi 0.1.0__py3-none-any.whl

reefapi/__init__.py

@@ -0,0 +1,29 @@
+"""Official Python SDK for ReefAPI: one API key for 600+ live web-data APIs.
+
+Quick start::
+
+    from reefapi import ReefAPI
+
+    client = ReefAPI()  # reads the REEFAPI_KEY environment variable
+    listings = client.ebay.search(query="iphone 13 pro")
+    product = client.call("amazon", "product/detail", asin="B0DGHMNQ5Z")
+
+Docs: https://reefapi.com/docs
+"""
+
+from ._client import DEFAULT_BASE_URL, AsyncReefAPI, ReefAPI
+from ._errors import ReefAPIError
+from ._types import ActionInfo, EngineInfo, ParamSpec, ReefResponse
+from ._version import __version__
+
+__all__ = [
+    "ReefAPI",
+    "AsyncReefAPI",
+    "ReefAPIError",
+    "ReefResponse",
+    "EngineInfo",
+    "ActionInfo",
+    "ParamSpec",
+    "DEFAULT_BASE_URL",
+    "__version__",
+]

reefapi/_client.py

@@ -0,0 +1,473 @@
+from __future__ import annotations
+
+import asyncio
+import copy
+import os
+import random
+import threading
+import time
+from collections import deque
+from typing import Any, Callable, Deque, Dict, List, Mapping, Optional, Tuple, TypeVar, Union, cast
+
+import httpx
+
+from ._engines import AsyncEngines, SyncEngines
+from ._errors import ReefAPIError
+from ._types import EngineInfo, ReefResponse
+from ._version import __version__
+
+__all__ = ["ReefAPI", "AsyncReefAPI", "DEFAULT_BASE_URL"]
+
+DEFAULT_BASE_URL = "https://api.reefapi.com"
+DEFAULT_TIMEOUT = 180.0
+DEFAULT_CONNECT_TIMEOUT = 10.0
+DEFAULT_MAX_RETRIES = 2
+DEFAULT_RATE_LIMIT: Tuple[int, float] = (30, 3.0)  # gateway limit: 30 requests / 3 s per key
+DEFAULT_MAX_RETRY_WAIT = 60.0
+_BACKOFF_BASE = 0.5
+_BACKOFF_CAP = 8.0
+ENV_KEY = "REEFAPI_KEY"
+ENV_BASE_URL = "REEFAPI_BASE_URL"
+
+_T = TypeVar("_T")
+RateLimit = Optional[Tuple[int, float]]
+
+# The request never left the machine for these, so retrying a POST cannot double-charge.
+_NOT_SENT = (httpx.ConnectError, httpx.ConnectTimeout, httpx.PoolTimeout)
+
+
+def _now() -> float:  # patched in tests
+    return time.monotonic()
+
+
+def _sleep(seconds: float) -> None:  # patched in tests
+    time.sleep(seconds)
+
+
+async def _async_sleep(seconds: float) -> None:  # patched in tests
+    await asyncio.sleep(seconds)
+
+
+class _RateLimiter:
+    """Client-side sliding window so a burst of calls stays inside the gateway's per-key limit."""
+
+    def __init__(self, requests: int, window: float) -> None:
+        if requests < 1 or window <= 0:
+            raise ValueError("rate_limit must be (requests >= 1, window_seconds > 0)")
+        self.requests = requests
+        self.window = window
+        self._stamps: Deque[float] = deque()
+        self._lock = threading.Lock()
+
+    def reserve(self, now: float) -> float:
+        """Take a slot if one is free and return 0, else return how long to wait."""
+        with self._lock:
+            while self._stamps and self._stamps[0] <= now - self.window:
+                self._stamps.popleft()
+            if len(self._stamps) < self.requests:
+                self._stamps.append(now)
+                return 0.0
+            return max(self._stamps[0] + self.window - now, 0.001)
+
+
+def _parse_retry_after(headers: Mapping[str, str]) -> Optional[float]:
+    raw = headers.get("retry-after")
+    if raw is None:
+        return None
+    try:
+        return max(float(raw.strip()), 0.0)
+    except ValueError:
+        return None  # HTTP-date form: fall back to backoff
+
+
+def _clean_params(params: Optional[Mapping[str, Any]], extra: Mapping[str, Any]) -> Dict[str, Any]:
+    merged: Dict[str, Any] = {}
+    if params:
+        merged.update(params)
+    merged.update(extra)
+    return {k: v for k, v in merged.items() if v is not None}
+
+
+def _envelope_error(resp: httpx.Response, body: Any) -> ReefAPIError:
+    status = resp.status_code
+    default_retryable = status == 429 or status >= 500
+    if isinstance(body, dict):
+        err_raw, meta_raw = body.get("error"), body.get("meta")
+        err: Dict[str, Any] = err_raw if isinstance(err_raw, dict) else {}
+        meta: Dict[str, Any] = meta_raw if isinstance(meta_raw, dict) else {}
+        code = str(err.get("code") or f"HTTP_{status}")
+        message = str(err.get("message") or body.get("detail") or f"HTTP {status}")
+        retryable = err.get("retryable")
+        return ReefAPIError(
+            code,
+            message,
+            bool(default_retryable if retryable is None else retryable),
+            status,
+            request_id=meta.get("request_id"),
+            meta=meta,
+            details=err.get("details"),
+        )
+    snippet = resp.text[:200].strip() if resp.text else ""
+    return ReefAPIError(
+        f"HTTP_{status}" if status >= 400 else "INVALID_RESPONSE",
+        f"Unexpected non-JSON response (HTTP {status}){': ' + snippet if snippet else ''}",
+        default_retryable,
+        status,
+    )
+
+
+def _parse_envelope(resp: httpx.Response) -> ReefResponse:
+    try:
+        body = resp.json()
+    except ValueError:
+        body = None
+    if isinstance(body, dict) and body.get("ok") is True and resp.status_code < 400:
+        meta_raw = body.get("meta")
+        meta: Dict[str, Any] = meta_raw if isinstance(meta_raw, dict) else {}
+        return ReefResponse(data=body.get("data"), meta=meta, status=resp.status_code, headers=resp.headers)
+    raise _envelope_error(resp, body)
+
+
+def _parse_plain(resp: httpx.Response) -> Any:
+    try:
+        body = resp.json()
+    except ValueError:
+        body = None
+    if resp.status_code < 400 and body is not None:
+        return body
+    raise _envelope_error(resp, body)
+
+
+def _transport_error(exc: Exception) -> ReefAPIError:
+    if isinstance(exc, httpx.TimeoutException):
+        return ReefAPIError("TIMEOUT", f"Request timed out: {exc!r}", True, None)
+    return ReefAPIError("NETWORK_ERROR", f"Network error: {exc!r}", True, None)
+
+
+class _BaseClient:
+    def __init__(
+        self,
+        api_key: Optional[str],
+        base_url: Optional[str],
+        timeout: Union[float, httpx.Timeout],
+        max_retries: int,
+        rate_limit: RateLimit,
+        max_retry_wait: float,
+        default_headers: Optional[Mapping[str, str]],
+    ) -> None:
+        self.api_key = api_key if api_key is not None else os.environ.get(ENV_KEY)
+        self.base_url = (base_url or os.environ.get(ENV_BASE_URL) or DEFAULT_BASE_URL).rstrip("/")
+        self.timeout = timeout
+        self.max_retries = max(int(max_retries), 0)
+        self.max_retry_wait = max_retry_wait
+        self._limiter: Optional[_RateLimiter] = _RateLimiter(*rate_limit) if rate_limit else None
+        self._default_headers = dict(default_headers or {})
+        self._owns_http = True
+
+    # -- helpers ---------------------------------------------------------------------------------
+    def __repr__(self) -> str:  # never show the key
+        return f"{type(self).__name__}(base_url={self.base_url!r})"
+
+    def _timeout(self) -> httpx.Timeout:
+        if isinstance(self.timeout, httpx.Timeout):
+            return self.timeout
+        return httpx.Timeout(self.timeout, connect=min(DEFAULT_CONNECT_TIMEOUT, self.timeout))
+
+    def _headers(self, with_key: bool) -> Dict[str, str]:
+        h = {
+            "accept": "application/json",
+            "user-agent": f"reefapi-python/{__version__}",
+        }
+        h.update(self._default_headers)
+        if with_key:
+            if not self.api_key:
+                raise ReefAPIError(
+                    "MISSING_API_KEY",
+                    f"No API key. Pass ReefAPI(api_key=...) or set the {ENV_KEY} environment variable. "
+                    "Get a key at https://reefapi.com",
+                    False,
+                    None,
+                )
+            h["x-api-key"] = self.api_key
+        return h
+
+    def _action_url(self, engine: str, action: str) -> str:
+        engine = engine.strip("/")
+        action = action.strip("/")
+        if not engine or not action:
+            raise ValueError("engine and action are required, e.g. call('ebay', 'search', query='...')")
+        return f"{self.base_url}/{engine}/v1/{action}"
+
+    def _retry_delay(self, err: ReefAPIError, attempt: int, headers: Mapping[str, str]) -> Optional[float]:
+        if attempt >= self.max_retries:
+            return None
+        if not (err.retryable or err.status == 429):
+            return None
+        after = _parse_retry_after(headers)
+        if after is not None:
+            return after if after <= self.max_retry_wait else None
+        return min(_BACKOFF_CAP, _BACKOFF_BASE * (2.0 ** attempt)) * random.uniform(0.5, 1.0)
+
+    def _transport_delay(self, exc: Exception, attempt: int, idempotent: bool) -> Optional[float]:
+        if attempt >= self.max_retries:
+            return None
+        if not (idempotent or isinstance(exc, _NOT_SENT)):
+            return None  # a POST that may have reached the server is not repeated automatically
+        return min(_BACKOFF_CAP, _BACKOFF_BASE * (2.0 ** attempt)) * random.uniform(0.5, 1.0)
+
+    def _with(self: _T, **changes: Any) -> _T:
+        clone = copy.copy(self)
+        for k, v in changes.items():
+            if v is not None:
+                setattr(clone, k, v)
+        clone._owns_http = False  # type: ignore[attr-defined]
+        return clone
+
+
+class ReefAPI(SyncEngines, _BaseClient):
+    """Synchronous ReefAPI client.
+
+    One API key for 600+ live web-data APIs (``POST https://api.reefapi.com/<engine>/v1/<action>``).
+
+    Example::
+
+        from reefapi import ReefAPI
+
+        client = ReefAPI()  # reads REEFAPI_KEY
+        data = client.call("ebay", "search", query="iphone 13 pro")
+        data = client.ebay.search(query="iphone 13 pro")  # typed helper, same call
+
+    Args:
+        api_key: Your key. Defaults to the ``REEFAPI_KEY`` environment variable.
+        base_url: Defaults to ``https://api.reefapi.com``.
+        timeout: Seconds per request (float) or an ``httpx.Timeout``. Default 180 s; some engines
+            render pages in a real browser and legitimately take over a minute.
+        max_retries: Automatic retries for errors the gateway marks ``retryable`` and for HTTP 429.
+            Default 2. ``0`` disables retries.
+        rate_limit: Client-side ``(requests, window_seconds)`` limiter, default ``(30, 3.0)`` = the
+            gateway's per-key limit. ``None`` disables it.
+        max_retry_wait: A ``Retry-After`` longer than this (seconds) is not waited for; the error is raised.
+        http_client: Bring your own ``httpx.Client`` (proxies, transports). It is not closed by ``close()``.
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        timeout: Union[float, httpx.Timeout] = DEFAULT_TIMEOUT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        rate_limit: RateLimit = DEFAULT_RATE_LIMIT,
+        max_retry_wait: float = DEFAULT_MAX_RETRY_WAIT,
+        default_headers: Optional[Mapping[str, str]] = None,
+        http_client: Optional[httpx.Client] = None,
+    ) -> None:
+        _BaseClient.__init__(
+            self, api_key, base_url, timeout, max_retries, rate_limit, max_retry_wait, default_headers
+        )
+        self._http = http_client or httpx.Client()
+        self._owns_http = http_client is None
+
+    # -- public API ------------------------------------------------------------------------------
+    def call(self, engine: str, action: str, params: Optional[Mapping[str, Any]] = None, /, **kwargs: Any) -> Any:
+        """Call ``POST /<engine>/v1/<action>`` and return the response ``data``.
+
+        Parameters can be passed as a dict, as keyword arguments, or both; ``None`` values are dropped.
+        Raises :class:`ReefAPIError` when the gateway answers ``ok: false`` (failed calls are not charged).
+
+        Example: ``client.call("amazon", "product/detail", asin="B0DGHMNQ5Z")``
+        """
+        return self.call_raw(engine, action, params, **kwargs).data
+
+    def call_raw(
+        self, engine: str, action: str, params: Optional[Mapping[str, Any]] = None, /, **kwargs: Any
+    ) -> ReefResponse:
+        """Like :meth:`call` but returns the whole envelope (``data``, ``meta``, ``status``, ``headers``)."""
+        url = self._action_url(engine, action)
+        body = _clean_params(params, kwargs)
+        headers = self._headers(with_key=True)
+        return self._send(lambda: self._http.post(url, json=body, headers=headers, timeout=self._timeout()),
+                          _parse_envelope, idempotent=False)
+
+    def engines(self) -> List[EngineInfo]:
+        """List every engine in the live catalog (name, title, category, actions). No key needed."""
+        headers = self._headers(with_key=False)
+        url = f"{self.base_url}/catalog"
+        cat = self._send(lambda: self._http.get(url, params={"light": "1"}, headers=headers, timeout=self._timeout()),
+                         _parse_plain, idempotent=True)
+        return list(cat.get("engines") or [])
+
+    def spec(self, engine: str) -> EngineInfo:
+        """Full spec of one engine: every action with typed params, allowed values and examples. No key needed."""
+        headers = self._headers(with_key=False)
+        url = f"{self.base_url}/catalog"
+        cat = self._send(lambda: self._http.get(url, params={"engine": engine}, headers=headers, timeout=self._timeout()),
+                         _parse_plain, idempotent=True)
+        found = [e for e in (cat.get("engines") or []) if e.get("name") == engine]
+        if not found:
+            raise ReefAPIError("UNKNOWN_API", f"No engine named {engine!r}. See client.engines().", False, 404)
+        return cast(EngineInfo, found[0])
+
+    def with_options(
+        self,
+        *,
+        timeout: Union[float, httpx.Timeout, None] = None,
+        max_retries: Optional[int] = None,
+        max_retry_wait: Optional[float] = None,
+    ) -> "ReefAPI":
+        """Return a copy with different per-request settings (shares the connection pool and limiter)."""
+        return self._with(timeout=timeout, max_retries=max_retries, max_retry_wait=max_retry_wait)
+
+    def close(self) -> None:
+        if self._owns_http:
+            self._http.close()
+
+    def __enter__(self) -> "ReefAPI":
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+    # -- internals -------------------------------------------------------------------------------
+    def _send(self, send: Callable[[], httpx.Response], parse: Callable[[httpx.Response], _T], idempotent: bool) -> _T:
+        attempt = 0
+        while True:
+            if self._limiter is not None:
+                while True:
+                    wait = self._limiter.reserve(_now())
+                    if wait <= 0:
+                        break
+                    _sleep(wait)
+            try:
+                resp = send()
+            except httpx.TransportError as exc:
+                delay = self._transport_delay(exc, attempt, idempotent)
+                if delay is None:
+                    raise _transport_error(exc) from exc
+                _sleep(delay)
+                attempt += 1
+                continue
+            try:
+                return parse(resp)
+            except ReefAPIError as err:
+                delay = self._retry_delay(err, attempt, resp.headers)
+                if delay is None:
+                    raise
+                _sleep(delay)
+                attempt += 1
+
+
+class AsyncReefAPI(AsyncEngines, _BaseClient):
+    """Asynchronous ReefAPI client (same surface as :class:`ReefAPI`, every method is ``async``).
+
+    Example::
+
+        import asyncio
+        from reefapi import AsyncReefAPI
+
+        async def main():
+            async with AsyncReefAPI() as client:
+                data = await client.ebay.search(query="iphone 13 pro")
+
+        asyncio.run(main())
+    """
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        timeout: Union[float, httpx.Timeout] = DEFAULT_TIMEOUT,
+        max_retries: int = DEFAULT_MAX_RETRIES,
+        rate_limit: RateLimit = DEFAULT_RATE_LIMIT,
+        max_retry_wait: float = DEFAULT_MAX_RETRY_WAIT,
+        default_headers: Optional[Mapping[str, str]] = None,
+        http_client: Optional[httpx.AsyncClient] = None,
+    ) -> None:
+        _BaseClient.__init__(
+            self, api_key, base_url, timeout, max_retries, rate_limit, max_retry_wait, default_headers
+        )
+        self._http = http_client or httpx.AsyncClient()
+        self._owns_http = http_client is None
+
+    async def call(self, engine: str, action: str, params: Optional[Mapping[str, Any]] = None, /, **kwargs: Any) -> Any:
+        """Call ``POST /<engine>/v1/<action>`` and return the response ``data`` (see :meth:`ReefAPI.call`)."""
+        return (await self.call_raw(engine, action, params, **kwargs)).data
+
+    async def call_raw(
+        self, engine: str, action: str, params: Optional[Mapping[str, Any]] = None, /, **kwargs: Any
+    ) -> ReefResponse:
+        """Like :meth:`call` but returns the whole envelope (``data``, ``meta``, ``status``, ``headers``)."""
+        url = self._action_url(engine, action)
+        body = _clean_params(params, kwargs)
+        headers = self._headers(with_key=True)
+        return await self._send(lambda: self._http.post(url, json=body, headers=headers, timeout=self._timeout()),
+                                _parse_envelope, idempotent=False)
+
+    async def engines(self) -> List[EngineInfo]:
+        """List every engine in the live catalog. No key needed."""
+        headers = self._headers(with_key=False)
+        url = f"{self.base_url}/catalog"
+        cat = await self._send(
+            lambda: self._http.get(url, params={"light": "1"}, headers=headers, timeout=self._timeout()),
+            _parse_plain, idempotent=True)
+        return list(cat.get("engines") or [])
+
+    async def spec(self, engine: str) -> EngineInfo:
+        """Full spec of one engine (typed params, allowed values, examples). No key needed."""
+        headers = self._headers(with_key=False)
+        url = f"{self.base_url}/catalog"
+        cat = await self._send(
+            lambda: self._http.get(url, params={"engine": engine}, headers=headers, timeout=self._timeout()),
+            _parse_plain, idempotent=True)
+        found = [e for e in (cat.get("engines") or []) if e.get("name") == engine]
+        if not found:
+            raise ReefAPIError("UNKNOWN_API", f"No engine named {engine!r}. See client.engines().", False, 404)
+        return cast(EngineInfo, found[0])
+
+    def with_options(
+        self,
+        *,
+        timeout: Union[float, httpx.Timeout, None] = None,
+        max_retries: Optional[int] = None,
+        max_retry_wait: Optional[float] = None,
+    ) -> "AsyncReefAPI":
+        """Return a copy with different per-request settings (shares the connection pool and limiter)."""
+        return self._with(timeout=timeout, max_retries=max_retries, max_retry_wait=max_retry_wait)
+
+    async def aclose(self) -> None:
+        if self._owns_http:
+            await self._http.aclose()
+
+    async def __aenter__(self) -> "AsyncReefAPI":
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        await self.aclose()
+
+    async def _send(self, send: Callable[[], Any], parse: Callable[[httpx.Response], _T], idempotent: bool) -> _T:
+        attempt = 0
+        while True:
+            if self._limiter is not None:
+                while True:
+                    wait = self._limiter.reserve(_now())
+                    if wait <= 0:
+                        break
+                    await _async_sleep(wait)
+            try:
+                resp = await send()
+            except httpx.TransportError as exc:
+                delay = self._transport_delay(exc, attempt, idempotent)
+                if delay is None:
+                    raise _transport_error(exc) from exc
+                await _async_sleep(delay)
+                attempt += 1
+                continue
+            try:
+                return parse(resp)
+            except ReefAPIError as err:
+                delay = self._retry_delay(err, attempt, resp.headers)
+                if delay is None:
+                    raise
+                await _async_sleep(delay)
+                attempt += 1