scrapely-client-python 1.0.0__py3-none-any.whl

scrapely_client/_pagination.py

@@ -0,0 +1,220 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Protocol, TypeVar, overload
+
+from scrapely_client._models import KeyValueStoreKey, ListOfKeys, ListOfRequests, Request
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator, Awaitable, Callable, Iterator
+
+T = TypeVar('T')
+
+DEFAULT_CHUNK_SIZE = 1000
+"""Default per-page size used by the iterate helpers when the caller does not specify one.
+
+The value of 1000 keeps backwards compatibility with the previous fixed cache size.
+"""
+
+
+class HasItems(Protocol[T]):
+    """Structural contract for a single page of results from a paginated API endpoint.
+
+    Implementations must expose `items`. They may optionally expose `count` - the number of items scanned by the API for
+    this page, which can exceed `len(items)` when filters drop items from the response. The iterator helpers consult
+    `count` opportunistically via `getattr` for offset bookkeeping and fall back to `len(items)` when it is absent.
+    """
+
+    items: list[T]
+
+
+def get_items_iterator(
+    callback: Callable[..., HasItems[T]],
+    *,
+    limit: int | None = None,
+    offset: int | None = None,
+    chunk_size: int | None = None,
+) -> Iterator[T]:
+    """Yield individual items from offset-based paginated API responses.
+
+    The `callback` is invoked lazily to fetch each page from the API. It must accept `limit` and `offset` keyword
+    arguments and return an object whose `items` attribute is a list. If the object also exposes a `count` attribute, it
+    is used for offset bookkeeping (the Scrapely API's `count` reflects items scanned, which can exceed items returned when
+    filters are applied).
+
+    Iteration stops when a page scans no items (`count` is `0`, or `items` is empty when `count` is absent) or when the
+    user-requested `limit` is reached. A page can scan items while returning none - filters like `clean` drop items from
+    `items` but still count toward `count` - so terminating on scanned rather than returned items keeps the iterator
+    advancing across fully-filtered pages. The `total` field is intentionally not consulted, because it can change
+    between calls.
+
+    Args:
+        callback: Function returning a single page of items.
+        limit: Maximum total number of items to yield across all pages. `None` or `0` means no limit.
+        offset: Starting offset for the first page.
+        chunk_size: Maximum number of items requested per API call. `None` or `0` lets the API decide.
+    """
+    effective_chunk = chunk_size or 0
+    initial_offset = offset or 0
+    initial_limit = limit or 0
+    fetched_items = 0
+
+    while True:
+        current_page = callback(
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
+            offset=initial_offset + fetched_items,
+        )
+        yield from current_page.items
+
+        page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items))
+        fetched_items += page_scanned
+
+        if not page_scanned or (initial_limit and fetched_items >= initial_limit):
+            break
+
+
+async def get_items_iterator_async(
+    callback: Callable[..., Awaitable[HasItems[T]]],
+    *,
+    limit: int | None = None,
+    offset: int | None = None,
+    chunk_size: int | None = None,
+) -> AsyncIterator[T]:
+    """Async variant of :func:`get_items_iterator`.
+
+    The `callback` must be an awaitable returning a single page of items.
+    """
+    effective_chunk = chunk_size or 0
+    initial_offset = offset or 0
+    initial_limit = limit or 0
+    fetched_items = 0
+
+    while True:
+        current_page = await callback(
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
+            offset=initial_offset + fetched_items,
+        )
+        for item in current_page.items:
+            yield item
+
+        page_scanned = max(getattr(current_page, 'count', 0), len(current_page.items))
+        fetched_items += page_scanned
+
+        if not page_scanned or (initial_limit and fetched_items >= initial_limit):
+            break
+
+
+@overload
+def get_cursor_iterator(
+    callback: Callable[..., ListOfKeys],
+    *,
+    cursor: str | None = None,
+    limit: int | None = None,
+    chunk_size: int | None = None,
+) -> Iterator[KeyValueStoreKey]: ...
+@overload
+def get_cursor_iterator(
+    callback: Callable[..., ListOfRequests],
+    *,
+    cursor: str | None = None,
+    limit: int | None = None,
+    chunk_size: int | None = None,
+) -> Iterator[Request]: ...
+def get_cursor_iterator(
+    callback: Callable[..., ListOfKeys | ListOfRequests],
+    *,
+    cursor: str | None = None,
+    limit: int | None = None,
+    chunk_size: int | None = None,
+) -> Iterator[KeyValueStoreKey] | Iterator[Request]:
+    """Yield individual items from cursor-paginated API responses.
+
+    Cursor pagination is restricted to the two API responses that expose it: `ListOfKeys` (for key-value store keys) and
+    `ListOfRequests` (for request queue requests). Iteration ends when the next cursor is `None` or the user-requested
+    `limit` is reached. Emptiness alone does not stop iteration: server-side filters (such as the request-queue state
+    `filter`) can drop every item on a page while a live cursor still points at more data, so termination relies on the
+    cursor, not on whether a page returned items. Unlike offset responses, cursor responses expose no scanned-item
+    `count`, so `count` cannot be used to detect a fully-filtered page here.
+
+    Args:
+        callback: Function returning a single page of items. Receives `cursor` and `limit` kwargs.
+        cursor: Value of the cursor for the first request, or `None` to start from the beginning.
+        limit: Maximum total number of items to yield across all pages.
+        chunk_size: Maximum number of items requested per API call.
+    """
+    effective_chunk = chunk_size or 0
+    initial_limit = limit or 0
+    fetched_items = 0
+
+    while True:
+        current_page = callback(
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
+            cursor=cursor,
+        )
+        yield from current_page.items
+
+        fetched_items += len(current_page.items)
+        cursor = (
+            current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor
+        )
+
+        if cursor is None or (initial_limit and fetched_items >= initial_limit):
+            break
+
+
+@overload
+def get_cursor_iterator_async(
+    callback: Callable[..., Awaitable[ListOfKeys]],
+    *,
+    cursor: str | None = None,
+    limit: int | None = None,
+    chunk_size: int | None = None,
+) -> AsyncIterator[KeyValueStoreKey]: ...
+@overload
+def get_cursor_iterator_async(
+    callback: Callable[..., Awaitable[ListOfRequests]],
+    *,
+    cursor: str | None = None,
+    limit: int | None = None,
+    chunk_size: int | None = None,
+) -> AsyncIterator[Request]: ...
+async def get_cursor_iterator_async(
+    callback: Callable[..., Awaitable[ListOfKeys | ListOfRequests]],
+    *,
+    cursor: str | None = None,
+    limit: int | None = None,
+    chunk_size: int | None = None,
+) -> AsyncIterator[KeyValueStoreKey] | AsyncIterator[Request]:
+    """Async variant of :func:`get_cursor_iterator`."""
+    effective_chunk = chunk_size or 0
+    initial_limit = limit or 0
+    fetched_items = 0
+
+    while True:
+        current_page = await callback(
+            limit=_next_page_limit(initial_limit, fetched_items, effective_chunk),
+            cursor=cursor,
+        )
+        for item in current_page.items:
+            yield item
+
+        fetched_items += len(current_page.items)
+        cursor = (
+            current_page.next_exclusive_start_key if isinstance(current_page, ListOfKeys) else current_page.next_cursor
+        )
+
+        if cursor is None or (initial_limit and fetched_items >= initial_limit):
+            break
+
+
+def _next_page_limit(initial_limit: int, fetched_items: int, effective_chunk: int) -> int:
+    """Compute the `limit` value for the next API call.
+
+    `0` means no limit on the wire (matches the Scrapely API contract). When both an overall `initial_limit` and a per-page
+    `effective_chunk` are set, the call is clamped to whichever is smaller; if either is unset (`0`), the other wins.
+    """
+    if not initial_limit:
+        return effective_chunk
+    remaining = initial_limit - fetched_items
+    if not effective_chunk:
+        return remaining
+    return min(remaining, effective_chunk)

scrapely_client/_resource_clients/__init__.py

@@ -0,0 +1,84 @@
+from .actor import ActorClient, ActorClientAsync
+from .actor_collection import ActorCollectionClient, ActorCollectionClientAsync
+from .actor_env_var import ActorEnvVarClient, ActorEnvVarClientAsync
+from .actor_env_var_collection import ActorEnvVarCollectionClient, ActorEnvVarCollectionClientAsync
+from .actor_version import ActorVersionClient, ActorVersionClientAsync
+from .actor_version_collection import ActorVersionCollectionClient, ActorVersionCollectionClientAsync
+from .build import BuildClient, BuildClientAsync
+from .build_collection import BuildCollectionClient, BuildCollectionClientAsync
+from .dataset import DatasetClient, DatasetClientAsync
+from .dataset_collection import DatasetCollectionClient, DatasetCollectionClientAsync
+from .key_value_store import KeyValueStoreClient, KeyValueStoreClientAsync
+from .key_value_store_collection import KeyValueStoreCollectionClient, KeyValueStoreCollectionClientAsync
+from .log import LogClient, LogClientAsync
+from .request_queue import RequestQueueClient, RequestQueueClientAsync
+from .request_queue_collection import RequestQueueCollectionClient, RequestQueueCollectionClientAsync
+from .run import RunClient, RunClientAsync
+from .run_collection import RunCollectionClient, RunCollectionClientAsync
+from .schedule import ScheduleClient, ScheduleClientAsync
+from .schedule_collection import ScheduleCollectionClient, ScheduleCollectionClientAsync
+from .store_collection import StoreCollectionClient, StoreCollectionClientAsync
+from .task import TaskClient, TaskClientAsync
+from .task_collection import TaskCollectionClient, TaskCollectionClientAsync
+from .user import UserClient, UserClientAsync
+from .webhook import WebhookClient, WebhookClientAsync
+from .webhook_collection import WebhookCollectionClient, WebhookCollectionClientAsync
+from .webhook_dispatch import WebhookDispatchClient, WebhookDispatchClientAsync
+from .webhook_dispatch_collection import WebhookDispatchCollectionClient, WebhookDispatchCollectionClientAsync
+
+__all__ = [
+    'ActorClient',
+    'ActorClientAsync',
+    'ActorCollectionClient',
+    'ActorCollectionClientAsync',
+    'ActorEnvVarClient',
+    'ActorEnvVarClientAsync',
+    'ActorEnvVarCollectionClient',
+    'ActorEnvVarCollectionClientAsync',
+    'ActorVersionClient',
+    'ActorVersionClientAsync',
+    'ActorVersionCollectionClient',
+    'ActorVersionCollectionClientAsync',
+    'BuildClient',
+    'BuildClientAsync',
+    'BuildCollectionClient',
+    'BuildCollectionClientAsync',
+    'DatasetClient',
+    'DatasetClientAsync',
+    'DatasetCollectionClient',
+    'DatasetCollectionClientAsync',
+    'KeyValueStoreClient',
+    'KeyValueStoreClientAsync',
+    'KeyValueStoreCollectionClient',
+    'KeyValueStoreCollectionClientAsync',
+    'LogClient',
+    'LogClientAsync',
+    'RequestQueueClient',
+    'RequestQueueClientAsync',
+    'RequestQueueCollectionClient',
+    'RequestQueueCollectionClientAsync',
+    'RunClient',
+    'RunClientAsync',
+    'RunCollectionClient',
+    'RunCollectionClientAsync',
+    'ScheduleClient',
+    'ScheduleClientAsync',
+    'ScheduleCollectionClient',
+    'ScheduleCollectionClientAsync',
+    'StoreCollectionClient',
+    'StoreCollectionClientAsync',
+    'TaskClient',
+    'TaskClientAsync',
+    'TaskCollectionClient',
+    'TaskCollectionClientAsync',
+    'UserClient',
+    'UserClientAsync',
+    'WebhookClient',
+    'WebhookClientAsync',
+    'WebhookCollectionClient',
+    'WebhookCollectionClientAsync',
+    'WebhookDispatchClient',
+    'WebhookDispatchClientAsync',
+    'WebhookDispatchCollectionClient',
+    'WebhookDispatchCollectionClientAsync',
+]