scrapely-client-python 1.0.0__py3-none-any.whl

scrapely_client/__init__.py

@@ -0,0 +1,30 @@
+from importlib import metadata
+
+from ._charging import (
+    USES_PUSH_DATA_INTERCEPTION,
+    ChargeResult,
+    ChargingHooks,
+    PpeAwarePushDataContext,
+    normalize_items,
+    push_data_charging_context,
+)
+from ._scrapely_client import ApifyClient, ApifyClientAsync, ScrapelyClient, ScrapelyClientAsync
+
+try:
+    __version__ = metadata.version('scrapely-client-python')
+except metadata.PackageNotFoundError:
+    __version__ = '0.0.0'
+
+__all__ = [
+    'USES_PUSH_DATA_INTERCEPTION',
+    'ApifyClient',
+    'ApifyClientAsync',
+    'ChargeResult',
+    'ChargingHooks',
+    'PpeAwarePushDataContext',
+    'ScrapelyClient',
+    'ScrapelyClientAsync',
+    '__version__',
+    'normalize_items',
+    'push_data_charging_context',
+]

scrapely_client/_charging.py

@@ -0,0 +1,186 @@
+"""Opt-in pay-per-event charging interception for dataset pushes.
+
+Port of ``scrapely-python``'s ``patched_apify_client.py`` semantics into the client
+itself, with the charging logic supplied by the host SDK via the
+:class:`ChargingHooks` protocol (no SDK import — the SDK adapts its
+``ChargingManager`` to these hooks later).
+
+Without hooks (the default), the client behaves exactly like stock
+``apify-client``.
+"""
+
+from __future__ import annotations
+
+import inspect
+import json
+from contextvars import ContextVar
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any, Protocol, cast, runtime_checkable
+
+if TYPE_CHECKING:
+    from collections.abc import Awaitable, Callable, Sequence
+
+USES_PUSH_DATA_INTERCEPTION = '__scrapely_uses_push_data_interception__'
+"""Attribute marking dataset clients whose ``push_items`` charges pay-per-event items.
+
+The host SDK checks this attribute to detect that push-data interception is active."""
+
+
+@dataclass(frozen=True)
+class ChargeResult:
+    """Result of a charged push-data operation.
+
+    Mirrors the SDK-side ``ChargeResult`` contract; the client only stores and merges
+    these values, it never computes them.
+    """
+
+    event_charge_limit_reached: bool
+    """Whether an event charge limit was reached during the operation."""
+
+    charged_count: int
+    """Total number of charged items."""
+
+    chargeable_within_limit: dict[str, int | None] = field(default_factory=dict)
+    """How many more items could be charged per event within the current limits."""
+
+
+@dataclass
+class PpeAwarePushDataContext:
+    """Context carried through ``push_data_charging_context``.
+
+    Wrapping a push in ``push_data_charging_context.run({...}, fn)`` makes every
+    intercepted ``push_items()`` inside ``fn`` charge ``event_name`` and merge its
+    result into the context.
+    """
+
+    event_name: str | None = None
+    charge_result: ChargeResult | None = None
+
+
+push_data_charging_context: ContextVar[PpeAwarePushDataContext | None] = ContextVar(
+    'scrapely_push_data_charging_context', default=None
+)
+"""ContextVar for the current :class:`PpeAwarePushDataContext` (async-local in asyncio)."""
+
+
+@runtime_checkable
+class ChargingHooks(Protocol):
+    """Charging hooks supplied by the host SDK (e.g. ``scrapely``'s ``ChargingManager``).
+
+    The client knows nothing about pricing, budgets or ledgers — it only routes
+    ``push_items()`` through these hooks when they are configured and the dataset
+    is the actor run's default dataset.
+    """
+
+    def should_charge_dataset_pushes(self, dataset_id: str) -> bool:
+        """Whether pushes to this dataset should be intercepted and charged."""
+        ...
+
+    def push_and_charge(
+        self,
+        *,
+        items: Sequence[dict[str, Any]],
+        event_name: str | None,
+        push_fn: Callable[[Sequence[dict[str, Any]]], Any],
+    ) -> ChargeResult | Awaitable[ChargeResult]:
+        """Charge for (and persist) the given items.
+
+        ``push_fn`` persists the budget-limited subset of items; implementations must
+        call it with only the items that were actually accepted for charging. Both
+        sync and async implementations are supported (async clients await awaitables).
+        """
+        ...
+
+    def merge_charge_results(self, a: ChargeResult, b: ChargeResult) -> ChargeResult:
+        """Merge two charge results (e.g. multiple pushes within one context)."""
+        ...
+
+
+def normalize_items(items: Any) -> list[dict[str, Any]]:
+    """Normalize ``push_items`` input (str, dict, or list of either) to a list of dicts."""
+    if isinstance(items, str):
+        parsed = json.loads(items)
+        return parsed if isinstance(parsed, list) else [parsed]
+    if isinstance(items, list):
+        result: list[dict[str, Any]] = []
+        for item in items:
+            if isinstance(item, str):
+                parsed = json.loads(item)
+                if isinstance(parsed, list):
+                    result.extend(parsed)
+                else:
+                    result.append(parsed)
+            else:
+                result.append(item)
+        return result
+    return [items]
+
+
+def _merge_context_charge_result(context: PpeAwarePushDataContext, hooks: ChargingHooks, result: ChargeResult) -> None:
+    if context.charge_result is None:
+        context.charge_result = result
+    else:
+        context.charge_result = hooks.merge_charge_results(context.charge_result, result)
+
+
+def wrap_dataset_client_with_charging(client: Any, hooks: ChargingHooks, *, is_async: bool) -> Any:
+    """Wrap a dataset client's ``push_items`` with charging interception (method swap by design).
+
+    Semantics match ``scrapely``'s ``patched_apify_client.py``: normalize items →
+    ``push_and_charge`` with a ``push_fn`` that pushes the accepted subset via the
+    original ``push_items``, and accumulate the charge result into the ambient
+    ``push_data_charging_context`` when present.
+    """
+    original_push_items = client.push_items
+
+    if is_async:
+
+        async def patched_push_items(items: Any, **kwargs: Any) -> None:
+            context = push_data_charging_context.get()
+            normalized = normalize_items(items)
+
+            async def push_fn(limited_items: Sequence[dict[str, Any]]) -> None:
+                await original_push_items(limited_items, **kwargs)
+
+            outcome = hooks.push_and_charge(
+                items=normalized, event_name=context.event_name if context else None, push_fn=push_fn
+            )
+            if inspect.isawaitable(outcome):
+                outcome = await outcome
+            result = cast('ChargeResult', outcome)
+            if context is not None:
+                _merge_context_charge_result(context, hooks, result)
+
+    else:
+
+        def patched_push_items(items: Any, **kwargs: Any) -> None:
+            context = push_data_charging_context.get()
+            normalized = normalize_items(items)
+
+            def push_fn(limited_items: Sequence[dict[str, Any]]) -> None:
+                original_push_items(limited_items, **kwargs)
+
+            outcome = hooks.push_and_charge(
+                items=normalized, event_name=context.event_name if context else None, push_fn=push_fn
+            )
+            result = cast('ChargeResult', outcome)
+            if inspect.isawaitable(outcome):
+                msg = 'Sync clients require synchronous ChargingHooks.push_and_charge'
+                raise TypeError(msg)
+            if context is not None:
+                _merge_context_charge_result(context, hooks, result)
+
+    client.push_items = patched_push_items  # type: ignore[method-assign]
+    setattr(client, USES_PUSH_DATA_INTERCEPTION, True)
+    return client
+
+
+__all__ = [
+    'USES_PUSH_DATA_INTERCEPTION',
+    'ChargeResult',
+    'ChargingHooks',
+    'PpeAwarePushDataContext',
+    'normalize_items',
+    'push_data_charging_context',
+    'wrap_dataset_client_with_charging',
+]

scrapely_client/_client_registry.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from scrapely_client._resource_clients import (
+        ActorClient,
+        ActorClientAsync,
+        ActorCollectionClient,
+        ActorCollectionClientAsync,
+        ActorEnvVarClient,
+        ActorEnvVarClientAsync,
+        ActorEnvVarCollectionClient,
+        ActorEnvVarCollectionClientAsync,
+        ActorVersionClient,
+        ActorVersionClientAsync,
+        ActorVersionCollectionClient,
+        ActorVersionCollectionClientAsync,
+        BuildClient,
+        BuildClientAsync,
+        BuildCollectionClient,
+        BuildCollectionClientAsync,
+        DatasetClient,
+        DatasetClientAsync,
+        DatasetCollectionClient,
+        DatasetCollectionClientAsync,
+        KeyValueStoreClient,
+        KeyValueStoreClientAsync,
+        KeyValueStoreCollectionClient,
+        KeyValueStoreCollectionClientAsync,
+        LogClient,
+        LogClientAsync,
+        RequestQueueClient,
+        RequestQueueClientAsync,
+        RequestQueueCollectionClient,
+        RequestQueueCollectionClientAsync,
+        RunClient,
+        RunClientAsync,
+        RunCollectionClient,
+        RunCollectionClientAsync,
+        ScheduleClient,
+        ScheduleClientAsync,
+        ScheduleCollectionClient,
+        ScheduleCollectionClientAsync,
+        StoreCollectionClient,
+        StoreCollectionClientAsync,
+        TaskClient,
+        TaskClientAsync,
+        TaskCollectionClient,
+        TaskCollectionClientAsync,
+        UserClient,
+        UserClientAsync,
+        WebhookClient,
+        WebhookClientAsync,
+        WebhookCollectionClient,
+        WebhookCollectionClientAsync,
+        WebhookDispatchClient,
+        WebhookDispatchClientAsync,
+        WebhookDispatchCollectionClient,
+        WebhookDispatchCollectionClientAsync,
+    )
+
+
+@dataclass
+class ClientRegistry:
+    """Bundle of all sync client classes for dependency injection.
+
+    This config object is passed to the resource clients to avoid circular dependencies. Each resource client
+    receives this config and can instantiate other clients as needed.
+    """
+
+    actor_client: type[ActorClient]
+    actor_collection_client: type[ActorCollectionClient]
+    actor_env_var_client: type[ActorEnvVarClient]
+    actor_env_var_collection_client: type[ActorEnvVarCollectionClient]
+    actor_version_client: type[ActorVersionClient]
+    actor_version_collection_client: type[ActorVersionCollectionClient]
+    build_client: type[BuildClient]
+    build_collection_client: type[BuildCollectionClient]
+    dataset_client: type[DatasetClient]
+    dataset_collection_client: type[DatasetCollectionClient]
+    key_value_store_client: type[KeyValueStoreClient]
+    key_value_store_collection_client: type[KeyValueStoreCollectionClient]
+    log_client: type[LogClient]
+    request_queue_client: type[RequestQueueClient]
+    request_queue_collection_client: type[RequestQueueCollectionClient]
+    run_client: type[RunClient]
+    run_collection_client: type[RunCollectionClient]
+    schedule_client: type[ScheduleClient]
+    schedule_collection_client: type[ScheduleCollectionClient]
+    store_collection_client: type[StoreCollectionClient]
+    task_client: type[TaskClient]
+    task_collection_client: type[TaskCollectionClient]
+    user_client: type[UserClient]
+    webhook_client: type[WebhookClient]
+    webhook_collection_client: type[WebhookCollectionClient]
+    webhook_dispatch_client: type[WebhookDispatchClient]
+    webhook_dispatch_collection_client: type[WebhookDispatchCollectionClient]
+
+
+@dataclass
+class ClientRegistryAsync:
+    """Bundle of all async client classes for dependency injection.
+
+    This config object is passed to the resource clients to avoid circular dependencies. Each resource client
+    receives this config and can instantiate other clients as needed.
+    """
+
+    actor_client: type[ActorClientAsync]
+    actor_collection_client: type[ActorCollectionClientAsync]
+    actor_env_var_client: type[ActorEnvVarClientAsync]
+    actor_env_var_collection_client: type[ActorEnvVarCollectionClientAsync]
+    actor_version_client: type[ActorVersionClientAsync]
+    actor_version_collection_client: type[ActorVersionCollectionClientAsync]
+    build_client: type[BuildClientAsync]
+    build_collection_client: type[BuildCollectionClientAsync]
+    dataset_client: type[DatasetClientAsync]
+    dataset_collection_client: type[DatasetCollectionClientAsync]
+    key_value_store_client: type[KeyValueStoreClientAsync]
+    key_value_store_collection_client: type[KeyValueStoreCollectionClientAsync]
+    log_client: type[LogClientAsync]
+    request_queue_client: type[RequestQueueClientAsync]
+    request_queue_collection_client: type[RequestQueueCollectionClientAsync]
+    run_client: type[RunClientAsync]
+    run_collection_client: type[RunCollectionClientAsync]
+    schedule_client: type[ScheduleClientAsync]
+    schedule_collection_client: type[ScheduleCollectionClientAsync]
+    store_collection_client: type[StoreCollectionClientAsync]
+    task_client: type[TaskClientAsync]
+    task_collection_client: type[TaskCollectionClientAsync]
+    user_client: type[UserClientAsync]
+    webhook_client: type[WebhookClientAsync]
+    webhook_collection_client: type[WebhookCollectionClientAsync]
+    webhook_dispatch_client: type[WebhookDispatchClientAsync]
+    webhook_dispatch_collection_client: type[WebhookDispatchCollectionClientAsync]

scrapely_client/_consts.py

@@ -0,0 +1,96 @@
+from __future__ import annotations
+
+from datetime import timedelta
+
+DEFAULT_API_URL = 'https://api-staging.scrape.ly'
+"""Default base URL for the Scrapely API."""
+
+API_VERSION = 'v2'
+"""Current Scrapely API version."""
+
+DEFAULT_TIMEOUT_SHORT = timedelta(seconds=5)
+"""Default timeout for fast CRUD operations (e.g., get, update, delete)."""
+
+DEFAULT_TIMEOUT_MEDIUM = timedelta(seconds=30)
+"""Default timeout for batch, list, and data transfer operations."""
+
+DEFAULT_TIMEOUT_LONG = timedelta(seconds=360)
+"""Default timeout for long-polling, streaming, and other heavy operations."""
+
+DEFAULT_TIMEOUT_MAX = timedelta(seconds=360)
+"""Default maximum timeout cap for individual API requests (limits exponential growth)."""
+
+DEFAULT_MAX_RETRIES = 4
+"""Default maximum number of retries for failed requests."""
+
+DEFAULT_MIN_DELAY_BETWEEN_RETRIES = timedelta(milliseconds=500)
+"""Default minimum delay between retries."""
+
+DEFAULT_WAIT_FOR_FINISH = timedelta(seconds=999999)
+"""Default maximum wait time for job completion (effectively infinite)."""
+
+DEFAULT_WAIT_WHEN_JOB_NOT_EXIST = timedelta(seconds=3)
+"""How long to wait for a job to exist before giving up."""
+
+OVERRIDABLE_DEFAULT_HEADERS = {'Accept', 'Authorization', 'Accept-Encoding', 'User-Agent'}
+"""Headers that can be overridden by users, but will trigger a warning if they do so, as it may lead to API errors."""
+
+MIN_COMPRESSION_SIZE = 1024
+"""Smallest request body, in bytes, that is worth compressing.
+
+A smaller body already fits in a single network packet, so compressing it costs CPU time without
+saving a round trip.
+"""
+
+ALREADY_COMPRESSED_MEDIA_TYPE_PREFIXES = ('audio/', 'image/', 'video/')
+"""Media type prefixes whose payloads carry their own compression, so compressing the request body is wasted work."""
+
+ALREADY_COMPRESSED_MEDIA_TYPES = frozenset(
+    {
+        'application/epub+zip',
+        'application/gzip',
+        'application/java-archive',
+        'application/vnd.android.package-archive',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        'application/vnd.rar',
+        'application/x-7z-compressed',
+        'application/x-bzip',
+        'application/x-bzip2',
+        'application/x-gzip',
+        'application/x-rar-compressed',
+        'application/x-xz',
+        'application/x-zip-compressed',
+        'application/zip',
+        'application/zstd',
+        'font/woff',
+        'font/woff2',
+    }
+)
+"""Exact media types whose payloads carry their own compression."""
+
+COMPRESSIBLE_MEDIA_TYPES = frozenset(
+    {
+        'audio/aiff',
+        'audio/basic',
+        'audio/l16',
+        'audio/l24',
+        'audio/midi',
+        'audio/vnd.wave',
+        'audio/wav',
+        'audio/wave',
+        'audio/x-aiff',
+        'audio/x-wav',
+        'image/bmp',
+        'image/tiff',
+        'image/vnd.adobe.photoshop',
+        'image/vnd.microsoft.icon',
+        'image/x-icon',
+        'image/x-ms-bmp',
+    }
+)
+"""Uncompressed media types that sit under an already-compressed prefix, so compressing them still pays off."""
+
+COMPRESSIBLE_MEDIA_TYPE_SUFFIXES = ('+json', '+xml')
+"""Structured syntax suffixes marking a media type as text even under an already-compressed prefix (`image/svg+xml`)."""

scrapely_client/_docs.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import Any, Literal, TypeVar
+
+# The order of the rendered API groups is defined by GROUP_ORDER in website/transformDocs.js
+# and applied via groupSort in website/docusaurus.config.js.
+GroupName = Literal[
+    'Scrapely API clients',
+    'HTTP clients',
+    'Resource clients',
+    'Errors',
+    'Models',
+    'Typed dicts',
+    'Other',
+]
+
+T = TypeVar('T', bound=Callable[..., Any])
+
+
+def docs_group(group_name: GroupName) -> Callable[[T], T]:  # noqa: ARG001
+    """Mark a symbol for rendering and grouping in documentation.
+
+    This decorator is used solely for documentation purposes and does not modify the behavior
+    of the decorated callable.
+
+    Args:
+        group_name: The documentation group to which the symbol belongs.
+
+    Returns:
+        The original callable without modification.
+    """
+
+    def wrapper(func: T) -> T:
+        return func
+
+    return wrapper