scrapelyserp 0.1.0__py3-none-any.whl

scrapelyserp/__init__.py

@@ -0,0 +1,50 @@
+"""Official Python SDK for the Scrapely API (https://scrapelyserp.com).
+
+    from scrapelyserp import Scrapely
+
+    client = Scrapely()  # reads SCRAPELY_API_KEY
+    print(client.account.get()["balance"])
+"""
+
+from . import types
+from ._base import DEFAULT_BASE_URL, RequestOptions, normalize_base_url
+from ._errors import (
+    APIConnectionError,
+    APIError,
+    APITimeoutError,
+    AuthenticationError,
+    BadRequestError,
+    ComingSoonError,
+    ConflictError,
+    InsufficientCreditsError,
+    InternalServerError,
+    NotFoundError,
+    PermissionDeniedError,
+    RateLimitError,
+    ScrapelyError,
+)
+from ._resources import AsyncScrapely, Scrapely
+from ._version import __version__
+
+__all__ = [
+    "Scrapely",
+    "AsyncScrapely",
+    "RequestOptions",
+    "DEFAULT_BASE_URL",
+    "normalize_base_url",
+    "types",
+    "ScrapelyError",
+    "APIError",
+    "BadRequestError",
+    "AuthenticationError",
+    "InsufficientCreditsError",
+    "PermissionDeniedError",
+    "NotFoundError",
+    "ConflictError",
+    "RateLimitError",
+    "InternalServerError",
+    "ComingSoonError",
+    "APIConnectionError",
+    "APITimeoutError",
+    "__version__",
+]

scrapelyserp/_base.py

@@ -0,0 +1,272 @@
+"""The transport every resource method goes through: auth, retries, errors, pagination."""
+
+import asyncio
+import email.utils
+import json
+import os
+import random
+import re
+import time
+import uuid
+from typing import Any, AsyncIterator, Dict, Iterator, Mapping, Optional, Tuple, TypedDict
+from urllib.parse import quote
+
+import httpx
+
+from ._errors import APIConnectionError, APITimeoutError, AuthenticationError, error_for
+from ._routes import ROUTES
+from ._version import __version__
+
+DEFAULT_BASE_URL = "https://scrapelyserp.com"
+DEFAULT_TIMEOUT = 360.0  # seconds per attempt: a large product call may take up to 300 s
+
+
+class RequestOptions(TypedDict, total=False):
+    """Per-call overrides."""
+
+    headers: Mapping[str, str]
+    timeout: float
+    max_retries: int
+
+
+def normalize_base_url(url: str) -> str:
+    """``https://api.scrapelyserp.com/v1/`` -> ``https://api.scrapelyserp.com``: the routes carry ``/api/v1``."""
+    return re.sub(r"/(api/)?v1$", "", url.rstrip("/"))
+
+
+def _merge(body: Optional[Mapping[str, Any]], fields: Mapping[str, Any]) -> Optional[Dict[str, Any]]:
+    if body is None and not fields:
+        return None
+    return {**(body or {}), **fields}
+
+
+def _retry_after(value: Optional[str]) -> Optional[float]:
+    if not value:
+        return None
+    try:
+        return max(0.0, float(value))
+    except ValueError:
+        parsed = email.utils.parsedate_to_datetime(value) if value else None
+        return max(0.0, parsed.timestamp() - time.time()) if parsed else None
+
+
+class BaseClient:
+    """Shared by the sync and async clients: configuration, request building, retry policy."""
+
+    def __init__(
+        self,
+        api_key: Optional[str] = None,
+        *,
+        base_url: Optional[str] = None,
+        max_retries: int = 2,
+        timeout: float = DEFAULT_TIMEOUT,
+        headers: Optional[Mapping[str, str]] = None,
+    ) -> None:
+        self.api_key = api_key if api_key is not None else os.environ.get("SCRAPELY_API_KEY") or None
+        self.base_url = normalize_base_url(base_url or os.environ.get("SCRAPELY_BASE_URL") or DEFAULT_BASE_URL)
+        self.max_retries = max_retries
+        self.timeout = timeout
+        self._headers = dict(headers or {})
+
+    def _build(
+        self,
+        operation_id: str,
+        path: Optional[Mapping[str, Any]],
+        query: Optional[Mapping[str, Any]],
+        body: Any,
+        idempotency_key: Optional[str],
+        options: Optional[RequestOptions],
+    ) -> Tuple[str, str, Dict[str, str], Dict[str, str], Optional[bytes]]:
+        if operation_id not in ROUTES:
+            raise ValueError(f"Unknown operation: {operation_id}")
+        method, template, flags = ROUTES[operation_id]
+        if not self.api_key and not flags & 1:
+            raise AuthenticationError(401, {"error": {"code": "UNAUTHORIZED", "message": "No API key: pass api_key or set SCRAPELY_API_KEY."}})
+
+        def sub(m: "re.Match[str]") -> str:
+            value = (path or {}).get(m.group(1))
+            if value is None or value == "":
+                raise ValueError(f"{operation_id}: missing path parameter `{m.group(1)}`")
+            return quote(str(value), safe="")
+
+        url = self.base_url + re.sub(r"\{(\w+)\}", sub, template)
+        params: Dict[str, str] = {}
+        for k, v in (query or {}).items():
+            if v is None:
+                continue
+            if isinstance(v, bool):
+                v = "true" if v else "false"
+            elif isinstance(v, (list, tuple)):
+                v = ",".join(str(x) for x in v)
+            params[k] = str(v)
+        headers = {"accept": "application/json", "user-agent": f"scrapelyserp-python/{__version__}", **self._headers}
+        if self.api_key:
+            headers["authorization"] = f"Bearer {self.api_key}"
+        content = None
+        if body is not None:
+            headers["content-type"] = "application/json"
+            content = json.dumps(body).encode()
+        if flags & 2:
+            # Generated once per call, so a retry can never create a second resource.
+            headers["idempotency-key"] = idempotency_key or str(uuid.uuid4())
+        headers.update((options or {}).get("headers") or {})
+        return method, url, params, headers, content
+
+    @staticmethod
+    def _parse(response: httpx.Response) -> Any:
+        text = response.text
+        if text and "json" in response.headers.get("content-type", ""):
+            try:
+                return json.loads(text)
+            except ValueError:
+                return text
+        return text or None
+
+    @staticmethod
+    def _retryable(status: int) -> bool:
+        return status == 429 or (status >= 500 and status != 501)
+
+    @staticmethod
+    def _backoff(attempt: int, retry_after: Optional[str]) -> float:
+        """``Retry-After`` when sent (capped at 60 s), else 0.5 s, 1 s, 2 s... (max 8 s) with jitter."""
+        ra = _retry_after(retry_after)
+        if ra is not None:
+            return min(ra, 60.0)
+        return min(0.5 * 2**attempt, 8.0) * (0.75 + random.random() * 0.25)
+
+
+class SyncBaseClient(BaseClient):
+    def __init__(self, api_key: Optional[str] = None, *, http_client: Optional[httpx.Client] = None, **kwargs: Any) -> None:
+        super().__init__(api_key, **kwargs)
+        self._http = http_client or httpx.Client()
+
+    def close(self) -> None:
+        self._http.close()
+
+    def __enter__(self) -> "SyncBaseClient":
+        return self
+
+    def __exit__(self, *exc: Any) -> None:
+        self.close()
+
+    def request(
+        self,
+        operation_id: str,
+        *,
+        path: Optional[Mapping[str, Any]] = None,
+        query: Optional[Mapping[str, Any]] = None,
+        body: Any = None,
+        idempotency_key: Optional[str] = None,
+        options: Optional[RequestOptions] = None,
+    ) -> Any:
+        """Calls any operation by its operationId. The resource methods are typed wrappers of this."""
+        return self._request(operation_id, path=path, query=query, body=body, idempotency_key=idempotency_key, options=options)
+
+    def _request(self, operation_id: str, *, path: Any = None, query: Any = None, body: Any = None, idempotency_key: Optional[str] = None, options: Optional[RequestOptions] = None) -> Any:
+        method, url, params, headers, content = self._build(operation_id, path, query, body, idempotency_key, options)
+        opts = options or {}
+        max_retries = opts.get("max_retries", self.max_retries)
+        attempt = 0
+        while True:
+            try:
+                res = self._http.request(method, url, params=params, headers=headers, content=content, timeout=opts.get("timeout", self.timeout))
+            except httpx.TimeoutException as e:
+                if attempt < max_retries:
+                    time.sleep(self._backoff(attempt, None))
+                    attempt += 1
+                    continue
+                raise APITimeoutError(f"Timed out: {e}") from e
+            except httpx.TransportError as e:
+                if attempt < max_retries:
+                    time.sleep(self._backoff(attempt, None))
+                    attempt += 1
+                    continue
+                raise APIConnectionError(f"Connection error: {e}") from e
+            data = self._parse(res)
+            if res.is_success:
+                return data
+            if self._retryable(res.status_code) and attempt < max_retries:
+                time.sleep(self._backoff(attempt, res.headers.get("retry-after")))
+                attempt += 1
+                continue
+            raise error_for(res.status_code, data, res.headers)
+
+    def _paginate(self, operation_id: str, *, path: Any = None, query: Any = None, options: Optional[RequestOptions] = None, **_: Any) -> Iterator[Any]:
+        q = dict(query or {})
+        while True:
+            page = self._request(operation_id, path=path, query=q, options=options) or {}
+            data = page.get("data") or []
+            yield from data
+            cursor = page.get("next_starting_after") or (data[-1].get("id") if data and isinstance(data[-1], dict) else None)
+            if not page.get("has_more") or not cursor:
+                return
+            q["starting_after"] = cursor
+
+
+class AsyncBaseClient(BaseClient):
+    def __init__(self, api_key: Optional[str] = None, *, http_client: Optional[httpx.AsyncClient] = None, **kwargs: Any) -> None:
+        super().__init__(api_key, **kwargs)
+        self._http = http_client or httpx.AsyncClient()
+
+    async def close(self) -> None:
+        await self._http.aclose()
+
+    async def __aenter__(self) -> "AsyncBaseClient":
+        return self
+
+    async def __aexit__(self, *exc: Any) -> None:
+        await self.close()
+
+    async def request(
+        self,
+        operation_id: str,
+        *,
+        path: Optional[Mapping[str, Any]] = None,
+        query: Optional[Mapping[str, Any]] = None,
+        body: Any = None,
+        idempotency_key: Optional[str] = None,
+        options: Optional[RequestOptions] = None,
+    ) -> Any:
+        """Calls any operation by its operationId. The resource methods are typed wrappers of this."""
+        return await self._request(operation_id, path=path, query=query, body=body, idempotency_key=idempotency_key, options=options)
+
+    async def _request(self, operation_id: str, *, path: Any = None, query: Any = None, body: Any = None, idempotency_key: Optional[str] = None, options: Optional[RequestOptions] = None) -> Any:
+        method, url, params, headers, content = self._build(operation_id, path, query, body, idempotency_key, options)
+        opts = options or {}
+        max_retries = opts.get("max_retries", self.max_retries)
+        attempt = 0
+        while True:
+            try:
+                res = await self._http.request(method, url, params=params, headers=headers, content=content, timeout=opts.get("timeout", self.timeout))
+            except httpx.TimeoutException as e:
+                if attempt < max_retries:
+                    await asyncio.sleep(self._backoff(attempt, None))
+                    attempt += 1
+                    continue
+                raise APITimeoutError(f"Timed out: {e}") from e
+            except httpx.TransportError as e:
+                if attempt < max_retries:
+                    await asyncio.sleep(self._backoff(attempt, None))
+                    attempt += 1
+                    continue
+                raise APIConnectionError(f"Connection error: {e}") from e
+            data = self._parse(res)
+            if res.is_success:
+                return data
+            if self._retryable(res.status_code) and attempt < max_retries:
+                await asyncio.sleep(self._backoff(attempt, res.headers.get("retry-after")))
+                attempt += 1
+                continue
+            raise error_for(res.status_code, data, res.headers)
+
+    async def _paginate(self, operation_id: str, *, path: Any = None, query: Any = None, options: Optional[RequestOptions] = None, **_: Any) -> AsyncIterator[Any]:
+        q = dict(query or {})
+        while True:
+            page = await self._request(operation_id, path=path, query=q, options=options) or {}
+            data = page.get("data") or []
+            for item in data:
+                yield item
+            cursor = page.get("next_starting_after") or (data[-1].get("id") if data and isinstance(data[-1], dict) else None)
+            if not page.get("has_more") or not cursor:
+                return
+            q["starting_after"] = cursor

scrapelyserp/_errors.py

@@ -0,0 +1,92 @@
+"""Every error the SDK raises is a ScrapelyError; API answers are an APIError subclass by status."""
+
+from typing import Any, Dict, Optional, Type
+
+import httpx
+
+
+class ScrapelyError(Exception):
+    """Base class of every error this package raises."""
+
+
+class APIError(ScrapelyError):
+    """The API answered with an error: ``{"error": {"code", "message", "param?", "docs"}}``."""
+
+    def __init__(self, status: int, body: Any, headers: Optional[httpx.Headers] = None) -> None:
+        err: Dict[str, Any] = body.get("error", {}) if isinstance(body, dict) and isinstance(body.get("error"), dict) else {}
+        #: HTTP status.
+        self.status: int = status
+        #: Stable, machine-readable (``INVALID_PARAM``, ``INSUFFICIENT_CREDITS``...). Branch on this.
+        self.code: str = err.get("code") or f"HTTP_{status}"
+        #: The parameter at fault, when there is one.
+        self.param: Optional[str] = err.get("param")
+        #: Link to this code's documentation.
+        self.docs: Optional[str] = err.get("docs")
+        #: ``X-Request-Id``: quote it to support.
+        self.request_id: Optional[str] = headers.get("x-request-id") if headers is not None else None
+        self.body: Any = body
+        self.headers = headers
+        message = err.get("message") or (body[:200] if isinstance(body, str) and body else f"HTTP {status}")
+        self.message: str = message
+        super().__init__(f"{self.code}: {message}" + (f" (param: {self.param})" if self.param else ""))
+
+
+class BadRequestError(APIError):
+    pass
+
+
+class AuthenticationError(APIError):
+    pass
+
+
+class InsufficientCreditsError(APIError):
+    pass
+
+
+class PermissionDeniedError(APIError):
+    pass
+
+
+class NotFoundError(APIError):
+    pass
+
+
+class ConflictError(APIError):
+    pass
+
+
+class RateLimitError(APIError):
+    pass
+
+
+class InternalServerError(APIError):
+    pass
+
+
+class ComingSoonError(APIError):
+    """501 ``NOT_IMPLEMENTED``: announced but not live yet. Nothing was charged."""
+
+
+class APIConnectionError(ScrapelyError):
+    """The request never got an answer (DNS, connection reset, offline)."""
+
+
+class APITimeoutError(APIConnectionError):
+    pass
+
+
+_BY_STATUS: Dict[int, Type[APIError]] = {
+    400: BadRequestError,
+    401: AuthenticationError,
+    402: InsufficientCreditsError,
+    403: PermissionDeniedError,
+    404: NotFoundError,
+    409: ConflictError,
+    429: RateLimitError,
+    501: ComingSoonError,
+}
+
+
+def error_for(status: int, body: Any, headers: Optional[httpx.Headers] = None) -> APIError:
+    cls = _BY_STATUS.get(status) or (InternalServerError if status >= 500 else APIError)
+    return cls(status, body, headers)