scrapely-client-python 1.0.0__py3-none-any.whl

scrapely_client/_typeddicts.py

@@ -0,0 +1,408 @@
+# generated by datamodel-codegen
+
+from __future__ import annotations
+
+from typing import Any, Literal, NotRequired, TypeAlias, TypedDict
+
+from scrapely_client._docs import docs_group
+
+
+@docs_group('Typed dicts')
+class RequestBaseDict(TypedDict):
+    unique_key: NotRequired[str]
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+    url: NotRequired[str]
+    """
+    The URL of the request.
+    """
+    method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]
+    retry_count: NotRequired[int]
+    """
+    The number of times this request has been retried.
+    """
+    loaded_url: NotRequired[str | None]
+    """
+    The final URL that was loaded, after redirects (if any).
+    """
+    payload: NotRequired[str | dict[str, Any] | None]
+    """
+    The request payload, typically used with POST or PUT requests.
+    """
+    headers: NotRequired[dict[str, Any] | None]
+    """
+    HTTP headers sent with the request.
+    """
+    user_data: NotRequired[RequestUserDataDict]
+    no_retry: NotRequired[bool | None]
+    """
+    Indicates whether the request should not be retried if processing fails.
+    """
+    error_messages: NotRequired[list[str] | None]
+    """
+    Error messages recorded from failed processing attempts.
+    """
+    handled_at: NotRequired[str | None]
+    """
+    The timestamp when the request was marked as handled, if applicable.
+    """
+
+
+@docs_group('Typed dicts')
+class RequestBaseCamelDict(TypedDict):
+    uniqueKey: NotRequired[str]
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+    url: NotRequired[str]
+    """
+    The URL of the request.
+    """
+    method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]
+    retryCount: NotRequired[int]
+    """
+    The number of times this request has been retried.
+    """
+    loadedUrl: NotRequired[str | None]
+    """
+    The final URL that was loaded, after redirects (if any).
+    """
+    payload: NotRequired[str | dict[str, Any] | None]
+    """
+    The request payload, typically used with POST or PUT requests.
+    """
+    headers: NotRequired[dict[str, Any] | None]
+    """
+    HTTP headers sent with the request.
+    """
+    userData: NotRequired[RequestUserDataCamelDict]
+    noRetry: NotRequired[bool | None]
+    """
+    Indicates whether the request should not be retried if processing fails.
+    """
+    errorMessages: NotRequired[list[str] | None]
+    """
+    Error messages recorded from failed processing attempts.
+    """
+    handledAt: NotRequired[str | None]
+    """
+    The timestamp when the request was marked as handled, if applicable.
+    """
+
+
+@docs_group('Typed dicts')
+class RequestDict(RequestBaseDict):
+    """A request stored in the request queue, including its metadata and processing state."""
+
+    id: NotRequired[str]
+    """
+    A unique identifier assigned to the request.
+    """
+
+
+@docs_group('Typed dicts')
+class RequestCamelDict(RequestBaseCamelDict):
+    """A request stored in the request queue, including its metadata and processing state."""
+
+    id: NotRequired[str]
+    """
+    A unique identifier assigned to the request.
+    """
+
+
+@docs_group('Typed dicts')
+class RequestDraftDict(TypedDict):
+    """A request that failed to be processed during a request queue operation and can be retried."""
+
+    id: NotRequired[str]
+    """
+    A unique identifier assigned to the request.
+    """
+    unique_key: str
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+    url: str
+    """
+    The URL of the request.
+    """
+    method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]
+
+
+@docs_group('Typed dicts')
+class RequestDraftCamelDict(TypedDict):
+    """A request that failed to be processed during a request queue operation and can be retried."""
+
+    id: NotRequired[str]
+    """
+    A unique identifier assigned to the request.
+    """
+    uniqueKey: str
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+    url: str
+    """
+    The URL of the request.
+    """
+    method: NotRequired[Literal['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'CONNECT', 'OPTIONS', 'TRACE', 'PATCH']]
+
+
+@docs_group('Typed dicts')
+class RequestDraftDeleteByIdDict(TypedDict):
+    """A request that should be deleted, identified by its ID."""
+
+    id: str
+    """
+    A unique identifier assigned to the request.
+    """
+    unique_key: NotRequired[str]
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+
+
+@docs_group('Typed dicts')
+class RequestDraftDeleteByIdCamelDict(TypedDict):
+    """A request that should be deleted, identified by its ID."""
+
+    id: str
+    """
+    A unique identifier assigned to the request.
+    """
+    uniqueKey: NotRequired[str]
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+
+
+@docs_group('Typed dicts')
+class RequestDraftDeleteByUniqueKeyDict(TypedDict):
+    """A request that should be deleted, identified by its unique key."""
+
+    id: NotRequired[str]
+    """
+    A unique identifier assigned to the request.
+    """
+    unique_key: str
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+
+
+@docs_group('Typed dicts')
+class RequestDraftDeleteByUniqueKeyCamelDict(TypedDict):
+    """A request that should be deleted, identified by its unique key."""
+
+    id: NotRequired[str]
+    """
+    A unique identifier assigned to the request.
+    """
+    uniqueKey: str
+    """
+    A unique key used for request de-duplication. Requests with the same unique key are considered identical.
+    """
+
+
+RequestDraftDeleteDict: TypeAlias = RequestDraftDeleteByIdDict | RequestDraftDeleteByUniqueKeyDict
+"""
+A request that should be deleted.
+"""
+
+RequestDraftDeleteCamelDict: TypeAlias = RequestDraftDeleteByIdCamelDict | RequestDraftDeleteByUniqueKeyCamelDict
+"""
+A request that should be deleted.
+"""
+
+
+RequestUserDataDict: TypeAlias = dict[str, Any]
+
+RequestUserDataCamelDict: TypeAlias = dict[str, Any]
+
+
+TaskInputDict: TypeAlias = dict[str, Any]
+
+TaskInputCamelDict: TypeAlias = dict[str, Any]
+
+
+@docs_group('Typed dicts')
+class WebhookConditionDict(TypedDict):
+    actor_id: NotRequired[str | None]
+    actor_task_id: NotRequired[str | None]
+    actor_run_id: NotRequired[str | None]
+
+
+@docs_group('Typed dicts')
+class WebhookConditionCamelDict(TypedDict):
+    actorId: NotRequired[str | None]
+    actorTaskId: NotRequired[str | None]
+    actorRunId: NotRequired[str | None]
+
+
+@docs_group('Typed dicts')
+class WebhookCreateDict(TypedDict):
+    is_ad_hoc: NotRequired[bool | None]
+    event_types: list[
+        Literal[
+            'ACTOR.BUILD.ABORTED',
+            'ACTOR.BUILD.CREATED',
+            'ACTOR.BUILD.FAILED',
+            'ACTOR.BUILD.SUCCEEDED',
+            'ACTOR.BUILD.TIMED_OUT',
+            'ACTOR.RUN.ABORTED',
+            'ACTOR.RUN.CREATED',
+            'ACTOR.RUN.FAILED',
+            'ACTOR.RUN.RESURRECTED',
+            'ACTOR.RUN.SUCCEEDED',
+            'ACTOR.RUN.TIMED_OUT',
+            'TEST',
+        ]
+    ]
+    condition: WebhookConditionDict
+    idempotency_key: NotRequired[str | None]
+    ignore_ssl_errors: NotRequired[bool | None]
+    do_not_retry: NotRequired[bool | None]
+    request_url: str
+    payload_template: NotRequired[str | None]
+    headers_template: NotRequired[str | None]
+    description: NotRequired[str | None]
+    should_interpolate_strings: NotRequired[bool | None]
+
+
+@docs_group('Typed dicts')
+class WebhookCreateCamelDict(TypedDict):
+    isAdHoc: NotRequired[bool | None]
+    eventTypes: list[
+        Literal[
+            'ACTOR.BUILD.ABORTED',
+            'ACTOR.BUILD.CREATED',
+            'ACTOR.BUILD.FAILED',
+            'ACTOR.BUILD.SUCCEEDED',
+            'ACTOR.BUILD.TIMED_OUT',
+            'ACTOR.RUN.ABORTED',
+            'ACTOR.RUN.CREATED',
+            'ACTOR.RUN.FAILED',
+            'ACTOR.RUN.RESURRECTED',
+            'ACTOR.RUN.SUCCEEDED',
+            'ACTOR.RUN.TIMED_OUT',
+            'TEST',
+        ]
+    ]
+    condition: WebhookConditionCamelDict
+    idempotencyKey: NotRequired[str | None]
+    ignoreSslErrors: NotRequired[bool | None]
+    doNotRetry: NotRequired[bool | None]
+    requestUrl: str
+    payloadTemplate: NotRequired[str | None]
+    headersTemplate: NotRequired[str | None]
+    description: NotRequired[str | None]
+    shouldInterpolateStrings: NotRequired[bool | None]
+
+
+@docs_group('Typed dicts')
+class WebhookRepresentationDict(TypedDict):
+    """Minimal representation of an ad-hoc webhook attached to a single Actor run or build via the
+    `webhooks` query parameter. The query parameter value is a Base64-encoded JSON array whose
+    items match this schema. Persistent webhook fields (e.g. `condition`) are not used here.
+
+    """
+
+    event_types: list[
+        Literal[
+            'ACTOR.BUILD.ABORTED',
+            'ACTOR.BUILD.CREATED',
+            'ACTOR.BUILD.FAILED',
+            'ACTOR.BUILD.SUCCEEDED',
+            'ACTOR.BUILD.TIMED_OUT',
+            'ACTOR.RUN.ABORTED',
+            'ACTOR.RUN.CREATED',
+            'ACTOR.RUN.FAILED',
+            'ACTOR.RUN.RESURRECTED',
+            'ACTOR.RUN.SUCCEEDED',
+            'ACTOR.RUN.TIMED_OUT',
+            'TEST',
+        ]
+    ]
+    request_url: str
+    """
+    The URL to which the webhook sends its payload.
+    """
+    payload_template: NotRequired[str | None]
+    """
+    Optional template for the JSON payload sent by the webhook.
+    """
+    headers_template: NotRequired[str | None]
+    """
+    Optional template for the HTTP headers sent by the webhook.
+    """
+    should_interpolate_strings: NotRequired[bool | None]
+    """
+    Flag to also interpolate `{{...}}` variables inside string values of the payload and headers templates.
+    """
+    idempotency_key: NotRequired[str | None]
+    """
+    Key that prevents creating duplicate webhooks, e.g. when the run-starting request is retried.
+    """
+    ignore_ssl_errors: NotRequired[bool | None]
+    """
+    Flag to ignore SSL errors when the webhook sends the request.
+    """
+    do_not_retry: NotRequired[bool | None]
+    """
+    Flag to skip retrying the webhook request on failure.
+    """
+
+
+@docs_group('Typed dicts')
+class WebhookRepresentationCamelDict(TypedDict):
+    """Minimal representation of an ad-hoc webhook attached to a single Actor run or build via the
+    `webhooks` query parameter. The query parameter value is a Base64-encoded JSON array whose
+    items match this schema. Persistent webhook fields (e.g. `condition`) are not used here.
+
+    """
+
+    eventTypes: list[
+        Literal[
+            'ACTOR.BUILD.ABORTED',
+            'ACTOR.BUILD.CREATED',
+            'ACTOR.BUILD.FAILED',
+            'ACTOR.BUILD.SUCCEEDED',
+            'ACTOR.BUILD.TIMED_OUT',
+            'ACTOR.RUN.ABORTED',
+            'ACTOR.RUN.CREATED',
+            'ACTOR.RUN.FAILED',
+            'ACTOR.RUN.RESURRECTED',
+            'ACTOR.RUN.SUCCEEDED',
+            'ACTOR.RUN.TIMED_OUT',
+            'TEST',
+        ]
+    ]
+    requestUrl: str
+    """
+    The URL to which the webhook sends its payload.
+    """
+    payloadTemplate: NotRequired[str | None]
+    """
+    Optional template for the JSON payload sent by the webhook.
+    """
+    headersTemplate: NotRequired[str | None]
+    """
+    Optional template for the HTTP headers sent by the webhook.
+    """
+    shouldInterpolateStrings: NotRequired[bool | None]
+    """
+    Flag to also interpolate `{{...}}` variables inside string values of the payload and headers templates.
+    """
+    idempotencyKey: NotRequired[str | None]
+    """
+    Key that prevents creating duplicate webhooks, e.g. when the run-starting request is retried.
+    """
+    ignoreSslErrors: NotRequired[bool | None]
+    """
+    Flag to ignore SSL errors when the webhook sends the request.
+    """
+    doNotRetry: NotRequired[bool | None]
+    """
+    Flag to skip retrying the webhook request on failure.
+    """

scrapely_client/_utils/crypto.py

@@ -0,0 +1,87 @@
+from __future__ import annotations
+
+import hashlib
+import hmac
+import string
+import time
+from base64 import urlsafe_b64encode
+from typing import TYPE_CHECKING
+
+from scrapely_client._utils.time import to_seconds
+
+if TYPE_CHECKING:
+    from datetime import timedelta
+
+_BASE62_CHARSET = string.digits + string.ascii_letters
+
+
+def encode_base62(num: int) -> str:
+    """Encode an integer to a base62 string.
+
+    Args:
+        num: The number to encode.
+
+    Returns:
+        The base62-encoded string.
+    """
+    if num == 0:
+        return _BASE62_CHARSET[0]
+
+    # Use list to build result for O(n) complexity instead of O(n^2) string concatenation.
+    parts = []
+    while num > 0:
+        num, remainder = divmod(num, 62)
+        parts.append(_BASE62_CHARSET[remainder])
+
+    # Reverse and join once at the end.
+    return ''.join(reversed(parts))
+
+
+def create_hmac_signature(secret_key: str, message: str) -> str:
+    """Generate an HMAC-SHA256 signature and encode it using base62.
+
+    The HMAC signature is truncated to 30 characters and then encoded in base62 to reduce the signature length.
+
+    Args:
+        secret_key: The secret key used for signing.
+        message: The message to be signed.
+
+    Returns:
+        The base62-encoded signature.
+    """
+    signature = hmac.new(secret_key.encode('utf-8'), message.encode('utf-8'), hashlib.sha256).hexdigest()[:30]
+
+    decimal_signature = int(signature, 16)
+
+    return encode_base62(decimal_signature)
+
+
+def create_storage_content_signature(
+    resource_id: str,
+    url_signing_secret_key: str,
+    *,
+    expires_in: timedelta | None = None,
+    version: int = 0,
+) -> str:
+    """Create a secure signature for a storage resource like a dataset or key-value store.
+
+    This signature is used to generate a signed URL for authenticated access, which can be expiring or permanent.
+    The signature is created using HMAC with the provided secret key and includes the resource ID, expiration time,
+    and version.
+
+    Args:
+        resource_id: The unique identifier of the storage resource.
+        url_signing_secret_key: The secret key for signing the URL.
+        expires_in: Optional expiration duration; if None, the signature never expires.
+        version: The signature version number (default: 0).
+
+    Returns:
+        The base64url-encoded signature string.
+    """
+    expires_at = int(time.time() * 1000) + int(to_seconds(expires_in) * 1000) if expires_in is not None else 0
+
+    message_to_sign = f'{version}.{expires_at}.{resource_id}'
+    hmac_sig = create_hmac_signature(url_signing_secret_key, message_to_sign)
+
+    base64url_encoded_payload = urlsafe_b64encode(f'{version}.{expires_at}.{hmac_sig}'.encode())
+    return base64url_encoded_payload.decode('utf-8')

scrapely_client/_utils/encoding.py

@@ -0,0 +1,133 @@
+from __future__ import annotations
+
+import json
+from base64 import b64encode
+from functools import cache
+from inspect import isawaitable, iscoroutine
+from typing import TYPE_CHECKING, Any
+
+from scrapely_client._models import WebhookCreate, WebhookRepresentation
+
+if TYPE_CHECKING:
+    from scrapely_client.types import WebhooksList
+
+
+def encode_key_value_store_record_value(
+    value: Any, *, content_type: str | None = None, content_encoding: str | None = None
+) -> tuple[bytes | bytearray | str, str]:
+    """Encode a value for storage in a key-value store record.
+
+    Args:
+        value: The value to encode. Anything exposing a callable `read` is treated as a file-like object: `read`
+            is called with no arguments, so the value is consumed from its current position and buffered in
+            memory whole - the object is neither rewound nor closed, and async file-like objects are rejected.
+            Any other value is JSON-serialized unless it is already bytes or a string.
+        content_type: The content type; if None, it's inferred from the value type.
+        content_encoding: The encoding the caller declares the value already carries, if any. Anything other than
+            `identity` means the value is compressed, which only a bytes-like payload can be, so any other value
+            is rejected. The check belongs here because a file-like value has to be read before its payload type
+            is known, and reading it a second time in the caller is not possible.
+
+    Returns:
+        A tuple of (encoded_value, content_type).
+
+    Raises:
+        TypeError: If the value cannot be encoded into a body the transport accepts, or if it cannot be carrying
+            the declared `content_encoding`.
+    """
+    # Read file-like values into memory; the transport only accepts bytes-like bodies. Detect them by a
+    # callable `read` (not `io.IOBase`) so duck-typed file-likes are read, not JSON-serialized. Impit exposes
+    # no streaming `content=` API, so the value has to be buffered whole.
+    read = getattr(value, 'read', None)
+    if callable(read):
+        value = read()
+
+        if isawaitable(value):
+            if iscoroutine(value):
+                value.close()  # Prevent a "coroutine was never awaited" warning.
+            raise TypeError(
+                'Async file-like objects are not supported. Await the read yourself and pass the resulting '
+                'bytes or string.'
+            )
+
+        if not isinstance(value, (bytes, bytearray, str)):
+            raise TypeError(f'Reading the file-like value returned {type(value).__name__}, expected bytes or str.')
+
+    # A declared compression describes bytes the caller compressed. A string, a JSON-serializable object, or a
+    # text-mode file cannot be carrying one, and would otherwise be stored under a header that misdescribes it -
+    # the client forwards the header untouched and never inspects the body.
+    declared_encoding = (content_encoding or '').strip().lower()
+    if declared_encoding not in ('', 'identity') and not isinstance(value, (bytes, bytearray)):
+        raise TypeError(
+            f'Cannot upload a {type(value).__name__} value with `Content-Encoding: {content_encoding}`. An encoding '
+            'other than `identity` declares the value is already compressed, so pass the compressed bytes, or a '
+            'file-like object that reads them.'
+        )
+
+    if not content_type:
+        if isinstance(value, (bytes, bytearray)):
+            content_type = 'application/octet-stream'
+        elif isinstance(value, str):
+            content_type = 'text/plain; charset=utf-8'
+        else:
+            content_type = 'application/json; charset=utf-8'
+
+    if 'application/json' in content_type and not isinstance(value, (bytes, bytearray, str)):
+        # Don't use indentation to reduce size.
+        value = json.dumps(
+            value,
+            ensure_ascii=False,
+            allow_nan=False,
+            default=str,
+        ).encode('utf-8')
+
+    # A non-JSON content type skips the serialization above, so anything that is not bytes-like would reach the
+    # transport unencoded and fail there with an opaque error.
+    if not isinstance(value, (bytes, bytearray, str)):
+        raise TypeError(
+            f'Cannot encode a {type(value).__name__} value as {content_type!r}. Pass bytes, a string, or a '
+            'file-like object, or use a JSON content type.'
+        )
+
+    return (value, content_type)
+
+
+@cache
+def _webhook_representation_keys() -> frozenset[str]:
+    """Return all field names and aliases declared on `WebhookRepresentation`."""
+    keys = set[str]()
+    for name, info in WebhookRepresentation.model_fields.items():
+        keys.add(name)
+        if info.alias is not None:
+            keys.add(info.alias)
+    return frozenset(keys)
+
+
+def encode_webhooks_to_base64(webhooks: WebhooksList | None) -> str | None:
+    """Encode a list of ad-hoc webhooks to a base64 string for the `webhooks` query parameter.
+
+    Returns `None` for `None` or an empty list, so the query parameter is omitted.
+
+    See `WebhooksList` for the accepted shapes. `WebhookRepresentation` instances are used as-is. `WebhookCreate`
+    instances and dict shapes are projected onto the fields `WebhookRepresentation` declares, dropping anything else
+    (e.g. persistent-only fields like `condition`). Filtering by the declared field names and aliases means new
+    ad-hoc fields added to `WebhookRepresentation` flow through automatically, without touching this function.
+    """
+    if not webhooks:
+        return None
+
+    representations = list[WebhookRepresentation]()
+    allowed = _webhook_representation_keys()
+
+    for webhook in webhooks:
+        if isinstance(webhook, WebhookRepresentation):
+            representations.append(webhook)
+            continue
+
+        data = webhook.model_dump(by_alias=True) if isinstance(webhook, WebhookCreate) else dict(webhook)
+        filtered = {key: value for key, value in data.items() if key in allowed}
+        representations.append(WebhookRepresentation.model_validate(filtered))
+
+    data = [r.model_dump(by_alias=True, exclude_none=True) for r in representations]
+    json_string = json.dumps(data).encode(encoding='utf-8')
+    return b64encode(json_string).decode(encoding='ascii')

scrapely_client/_utils/errors.py

@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from scrapely_client.errors import NotFoundError
+
+if TYPE_CHECKING:
+    from scrapely_client.errors import ApifyApiError
+
+
+def catch_not_found_or_throw(exc: ApifyApiError) -> None:
+    """Suppress 404 Not Found errors and re-raise all other API errors.
+
+    Args:
+        exc: The API error to check.
+
+    Raises:
+        ApifyApiError: If the error is not a 404 Not Found error.
+    """
+    if not isinstance(exc, NotFoundError):
+        raise exc
+
+
+def catch_not_found_for_resource_or_throw(exc: ApifyApiError, resource_id: str | None) -> None:
+    """Like `catch_not_found_or_throw`, but only suppress 404s when the client targets a specific resource by ID.
+
+    For chained clients without a `resource_id` (e.g. `run.dataset()`, `run.log()`), a 404 could mean either the
+    parent or the default sub-resource is missing — the API body cannot disambiguate — so the error propagates
+    rather than being swallowed.
+    """
+    if resource_id is None:
+        raise exc
+    catch_not_found_or_throw(exc)