scrapely-client-python 3.1.3__py3-none-any.whl

scrapely_client/_utils/http.py

@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import warnings
+from typing import TYPE_CHECKING
+from urllib.parse import quote
+
+from scrapely_client._consts import (
+    ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES,
+    ALREADY_COMPRESSED_MEDIA_TYPES,
+    COMPRESSIBLE_MEDIA_TYPE_SUFFIXES,
+    COMPRESSIBLE_MEDIA_TYPES,
+    OVERRIDABLE_DEFAULT_HEADERS,
+)
+
+if TYPE_CHECKING:
+    from scrapely_client.http_clients import HttpResponse
+
+
+def to_safe_id(id: str) -> str:
+    """Convert a resource ID to URL-safe format by replacing forward slashes with tildes.
+
+    Args:
+        id: The resource identifier in format `resource_id` or `username/resource_id`.
+
+    Returns:
+        The resource identifier with `/` characters replaced by `~`.
+    """
+    return id.replace('/', '~')
+
+
+def to_path_segment(value: str) -> str:
+    """Percent-encode a caller-supplied value so it stays a single URL path segment.
+
+    Without this, a value carrying `/`, `?` or `#` would restructure the URL it is interpolated into: it could
+    reach a different endpoint, append its own query parameters, or be silently truncated at a fragment.
+
+    Args:
+        value: The value to place in a single path segment, for example a key-value store record key.
+
+    Returns:
+        The percent-encoded value, which a URL parser can only read as one path segment.
+
+    Raises:
+        ValueError: If the value is empty, `.` or `..`, none of which can be carried in a path segment at all.
+    """
+    # Encoding cannot save these three: an empty value leaves nothing but the separator, and a URL parser
+    # resolves a dot segment after percent-decoding, so `%2E%2E` collapses just like `..`. Either way the
+    # request lands on a parent endpoint, where `..` makes the same verb act on the whole resource.
+    if value in {'', '.', '..'}:
+        raise ValueError(f'"{value}" cannot be used as a URL path segment.')
+
+    return quote(value, safe='')
+
+
+def is_compressible_content_type(content_type: str | None) -> bool:
+    """Decide whether a request body with the given content type is worth compressing.
+
+    Images, audio, video and archives already carry their own compression. Running them through gzip or brotli
+    burns CPU, holds a second full copy of the body in memory, and usually produces output slightly larger than
+    the input. Formats that are raw despite such a media type, for example `image/bmp` or `audio/wav`, are still
+    compressed. A body with no content type is assumed to be compressible.
+
+    Args:
+        content_type: The value of the `Content-Type` header, if any.
+
+    Returns:
+        `True` if the body should be compressed before it is sent.
+    """
+    if not content_type:
+        return True
+
+    # `Content-Type` is case-insensitive and may carry parameters, for example `text/plain; charset=utf-8`.
+    media_type = content_type.split(';', 1)[0].strip().lower()
+
+    if media_type in COMPRESSIBLE_MEDIA_TYPES or media_type.endswith(COMPRESSIBLE_MEDIA_TYPE_SUFFIXES):
+        return True
+
+    return not (
+        media_type in ALREADY_COMPRESSED_MEDIA_TYPES or media_type.startswith(ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES)
+    )
+
+
+def response_to_dict(response: HttpResponse) -> dict:
+    """Parse the API response as a dictionary and validate its type.
+
+    Args:
+        response: The HTTP response object from the API.
+
+    Returns:
+        The parsed response as a dictionary.
+
+    Raises:
+        ValueError: If the response is not a dictionary.
+    """
+    data = response.json()
+
+    if isinstance(data, dict):
+        return data
+
+    raise ValueError(f'The response is not a dictionary. Got: {type(data).__name__}')
+
+
+def response_to_list(response: HttpResponse) -> list:
+    """Parse the API response as a list and validate its type.
+
+    Args:
+        response: The HTTP response object from the API.
+
+    Returns:
+        The parsed response as a list.
+
+    Raises:
+        ValueError: If the response is not a list.
+    """
+    data = response.json()
+
+    if isinstance(data, list):
+        return data
+
+    if isinstance(data, dict):
+        return [data]
+
+    raise ValueError(f'The response is not a list. Got: {type(data).__name__}')
+
+
+def check_custom_headers(class_name: str, headers: dict[str, str]) -> None:
+    """Warn if custom headers override important default headers."""
+    overwrite_headers = [key for key in headers if key.title() in OVERRIDABLE_DEFAULT_HEADERS]
+
+    if overwrite_headers:
+        warnings.warn(
+            f'{", ".join(overwrite_headers)} headers of {class_name} was overridden with an '
+            'explicit value. A wrong header value can lead to API errors, it is recommended to use the default '
+            f'value for following headers: {", ".join(OVERRIDABLE_DEFAULT_HEADERS)}.',
+            category=UserWarning,
+            stacklevel=3,
+        )

scrapely_client/_utils/time.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Literal, overload
+
+if TYPE_CHECKING:
+    from datetime import timedelta
+
+
+@overload
+def to_seconds(td: None, *, as_int: bool = ...) -> None: ...
+@overload
+def to_seconds(td: timedelta) -> float: ...
+@overload
+def to_seconds(td: timedelta, *, as_int: Literal[True]) -> int: ...
+@overload
+def to_seconds(td: timedelta, *, as_int: Literal[False]) -> float: ...
+
+
+def to_seconds(td: timedelta | None, *, as_int: bool = False) -> float | int | None:
+    """Convert timedelta to seconds.
+
+    Args:
+        td: The timedelta to convert, or None.
+        as_int: If True, round and return as int. Defaults to False.
+
+    Returns:
+        The total seconds as a float (or int if as_int=True), or None if input is None.
+    """
+    if td is None:
+        return None
+    seconds = td.total_seconds()
+    return int(seconds) if as_int else seconds

scrapely_client/_utils/try_import.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import sys
+from contextlib import contextmanager
+from dataclasses import dataclass
+from types import ModuleType
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator
+    from typing import Any
+
+
+@dataclass
+class ImportState:
+    """Describe whether an optional import succeeded."""
+
+    available: bool = True
+
+
+@contextmanager
+def try_import(module_name: str, *symbol_names: str, dependency_name: str) -> Iterator[ImportState]:
+    """Context manager to attempt importing symbols into a module.
+
+    If the named optional dependency is missing, the symbols are replaced with `FailedImport` objects. Import errors
+    caused by the importing module itself or by another dependency are propagated instead of being masked.
+    """
+    state = ImportState()
+    try:
+        yield state
+    except ModuleNotFoundError as exc:
+        if exc.name != dependency_name:
+            raise
+        state.available = False
+        for symbol_name in symbol_names:
+            setattr(sys.modules[module_name], symbol_name, FailedImport(exc.args[0]))
+
+
+def install_import_hook(module_name: str) -> None:
+    """Install an import hook for a specified module."""
+    sys.modules[module_name].__class__ = ImportWrapper
+
+
+@dataclass
+class FailedImport:
+    """Represent a placeholder for a failed import."""
+
+    message: str
+    """The error message associated with the failed import."""
+
+
+class ImportWrapper(ModuleType):
+    """A wrapper class for modules to handle attribute access for failed imports."""
+
+    def __getattribute__(self, name: str) -> Any:
+        result = super().__getattribute__(name)
+
+        if isinstance(result, FailedImport):
+            raise ImportError(result.message)  # noqa: TRY004
+
+        return result

scrapely_client/errors.py

@@ -0,0 +1,190 @@
+from __future__ import annotations
+
+from http import HTTPStatus
+from typing import TYPE_CHECKING, Any
+
+from scrapely_client._docs import docs_group
+
+if TYPE_CHECKING:
+    from typing import Self
+
+    from scrapely_client.http_clients import HttpResponse
+
+
+@docs_group('Errors')
+class ApifyClientError(Exception):
+    """Base class for all Scrapely API client errors."""
+
+
+@docs_group('Errors')
+class ApifyApiError(ApifyClientError):
+    """Error raised when the Scrapely API returns an error response.
+
+    Instantiating `ApifyApiError` dispatches to the subclass matching the HTTP status code (e.g. 404 → `NotFoundError`,
+    any 5xx → `ServerError`). Unmapped statuses stay on `ApifyApiError`. Existing `except ApifyApiError` handlers keep
+    working because every subclass inherits from this class.
+
+    The `type`, `message` and `data` fields from the response body are exposed for inspection but are treated as
+    non-authoritative metadata — dispatch is driven by the status code only.
+
+    Attributes:
+        message: The error message from the API response.
+        type: The error type identifier from the API response (e.g. `record-not-found`).
+        status_code: The HTTP status code of the error response.
+        attempt: The attempt number when the error was raised.
+        http_method: The HTTP method of the failed request.
+        data: Additional error data from the API response.
+    """
+
+    # Subclasses in `_STATUS_TO_CLASS` must keep the `(response, attempt, *, method='GET')` constructor signature —
+    # `__new__` forwards those arguments verbatim.
+
+    def __new__(cls, response: HttpResponse, attempt: int, *, method: str = 'GET') -> Self:  # noqa: ARG004
+        """Dispatch to the subclass matching the response's HTTP status code, if any."""
+        instance = super().__new__(cls)
+        if cls is ApifyApiError:
+            status = response.status_code
+            mapped = _STATUS_TO_CLASS.get(status)
+            if mapped is None and status >= HTTPStatus.INTERNAL_SERVER_ERROR:
+                mapped = ServerError
+            if mapped is not None:
+                # Retagging is runtime-equivalent to allocating `mapped` directly, but keeps the return type `Self`.
+                instance.__class__ = mapped
+        return instance
+
+    def __init__(self, response: HttpResponse, attempt: int, *, method: str = 'GET') -> None:
+        """Initialize the API error from a failed response.
+
+        Args:
+            response: The failed HTTP response from the Scrapely API.
+            attempt: The attempt number when the request failed (1-indexed).
+            method: The HTTP method of the failed request.
+        """
+        payload = self._extract_error_payload(response)
+
+        self.message: str | None = f'Unexpected error: {response.text}'
+        self.type: str | None = None
+        self.data = dict[str, str]()
+
+        if payload is not None:
+            self.message = payload.get('message', self.message)
+            self.type = payload.get('type')
+            if 'data' in payload:
+                self.data = payload['data']
+
+        super().__init__(self.message)
+
+        self.status_code = response.status_code
+        self.attempt = attempt
+        self.http_method = method
+
+    @staticmethod
+    def _extract_error_payload(response: HttpResponse) -> dict[str, Any] | None:
+        """Return the `error` dict from the response body, or None if absent or unparsable."""
+        try:
+            data = response.json()
+        except ValueError:
+            return None
+        if not isinstance(data, dict):
+            return None
+        error = data.get('error')
+        # Scrapely's API consistently returns { error: { type, message } }, but some edge/proxy
+        # responses return a plain string (e.g. { "error": "Unauthorized" }).
+        if isinstance(error, str):
+            return {'message': error}
+        return error if isinstance(error, dict) else None
+
+
+@docs_group('Errors')
+class InvalidRequestError(ApifyApiError):
+    """Raised when the Scrapely API returns an HTTP 400 Bad Request response."""
+
+
+@docs_group('Errors')
+class UnauthorizedError(ApifyApiError):
+    """Raised when the Scrapely API returns an HTTP 401 Unauthorized response."""
+
+
+@docs_group('Errors')
+class ForbiddenError(ApifyApiError):
+    """Raised when the Scrapely API returns an HTTP 403 Forbidden response."""
+
+
+@docs_group('Errors')
+class NotFoundError(ApifyApiError):
+    """Raised when the Scrapely API returns an HTTP 404 Not Found response."""
+
+
+@docs_group('Errors')
+class ConflictError(ApifyApiError):
+    """Raised when the Scrapely API returns an HTTP 409 Conflict response."""
+
+
+@docs_group('Errors')
+class RateLimitError(ApifyApiError):
+    """Raised when the Scrapely API returns an HTTP 429 Too Many Requests response.
+
+    Rate-limited requests are retried automatically; this error is only raised after all retry attempts have been
+    exhausted.
+    """
+
+
+@docs_group('Errors')
+class ServerError(ApifyApiError):
+    """Raised when the Scrapely API returns an HTTP 5xx response.
+
+    Server errors are retried automatically; this error is only raised after all retry attempts have been exhausted.
+    """
+
+
+@docs_group('Errors')
+class InvalidResponseBodyError(ApifyClientError):
+    """Error raised when a response body cannot be parsed.
+
+    This occurs when the API returns a body that does not match its content type, for example a malformed JSON
+    document. It is raised on a response the client already accepted, so it is not retried - a transfer that breaks
+    mid-body surfaces as a transport error inside the retry loop instead.
+    """
+
+    def __init__(self, response: HttpResponse) -> None:
+        """Initialize the error from an unparsable response.
+
+        Args:
+            response: The HTTP response whose body could not be parsed.
+        """
+        super().__init__('Response body could not be parsed')
+
+        self.code = 'invalid-response-body'
+        self.response = response
+
+
+# Scrapely-named aliases for the Apify-compatible error classes. The Apify names are part of
+# the client's backward-compatibility contract (code written against the Apify SDK keeps
+# working); these aliases let Scrapely-native code use the platform name.
+ScrapelyApiError = ApifyApiError
+ScrapelyClientError = ApifyClientError
+
+_STATUS_TO_CLASS: dict[int, type[ApifyApiError]] = {
+    400: InvalidRequestError,
+    401: UnauthorizedError,
+    403: ForbiddenError,
+    404: NotFoundError,
+    409: ConflictError,
+    429: RateLimitError,
+}
+
+
+__all__ = [
+    'ApifyApiError',
+    'ApifyClientError',
+    'ConflictError',
+    'ForbiddenError',
+    'InvalidRequestError',
+    'InvalidResponseBodyError',
+    'NotFoundError',
+    'RateLimitError',
+    'ScrapelyApiError',
+    'ScrapelyClientError',
+    'ServerError',
+    'UnauthorizedError',
+]

scrapely_client/http_clients/__init__.py

@@ -0,0 +1,10 @@
+from ._base import HttpClient, HttpClientAsync, HttpResponse
+from ._impit import ImpitHttpClient, ImpitHttpClientAsync
+
+__all__ = [
+    'HttpClient',
+    'HttpClientAsync',
+    'HttpResponse',
+    'ImpitHttpClient',
+    'ImpitHttpClientAsync',
+]