apify-client 3.2.2b2__tar.gz → 3.2.2b3__tar.gz

{apify_client-3.2.2b2 → apify_client-3.2.2b3}/CHANGELOG.md

@@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file.
 
 - Stream request bodies from files, iterables, and responses ([#1060](https://github.com/apify/apify-client-python/pull/1060)) ([9be215b](https://github.com/apify/apify-client-python/commit/9be215b23a13de0abc4a255a18450cc764e7a6c7)) by [@vdusek](https://github.com/vdusek), closes [#972](https://github.com/apify/apify-client-python/issues/972)
 - Retry starting a run on memory and concurrent-runs limits ([#1081](https://github.com/apify/apify-client-python/pull/1081)) ([3dd8ca5](https://github.com/apify/apify-client-python/commit/3dd8ca5fe4d2537cb009cb310ba12b39927a9558)) by [@vdusek](https://github.com/vdusek), closes [#1071](https://github.com/apify/apify-client-python/issues/1071)
+- Add live iteration over a run's dataset items ([#1079](https://github.com/apify/apify-client-python/pull/1079)) ([6aa8ead](https://github.com/apify/apify-client-python/commit/6aa8ead676a3520f6a418597eb62f319a3d19d30)) by [@vdusek](https://github.com/vdusek), closes [#1065](https://github.com/apify/apify-client-python/issues/1065)
 
 ### 🐛 Bug Fixes
 

{apify_client-3.2.2b2 → apify_client-3.2.2b3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify_client
-Version: 3.2.2b2
+Version: 3.2.2b3
 Summary: Apify API client for Python
 Keywords: apify,api,client,automation,crawling,scraping
 Author: Apify Technologies s.r.o.

{apify_client-3.2.2b2 → apify_client-3.2.2b3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify_client"
-version = "3.2.2b2"
+version = "3.2.2b3"
 description = "Apify API client for Python"
 license = "Apache-2.0"
 license-files = ["LICENSE"]

{apify_client-3.2.2b2 → apify_client-3.2.2b3}/pyproject.toml.orig

@@ -4,7 +4,7 @@ build-backend = "uv_build"
 
 [project]
 name = "apify_client"
-version = "3.2.2b2"
+version = "3.2.2b3"
 description = "Apify API client for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = "Apache-2.0"

{apify_client-3.2.2b2 → apify_client-3.2.2b3}/src/apify_client/_resource_clients/_resource_client.py

@@ -42,6 +42,7 @@ class ResourceClientBase(metaclass=WithLogDetailsClient):
         client_registry: Any,
         resource_id: str | None = None,
         params: dict | None = None,
+        api_base_url: str | None = None,
     ) -> None:
         """Initialize the resource client.
 
@@ -53,11 +54,13 @@ class ResourceClientBase(metaclass=WithLogDetailsClient):
             client_registry: Bundle of client classes for dependency injection.
             resource_id: Optional resource ID for single-resource clients.
             params: Optional default parameters for all requests.
+            api_base_url: Base URL of the API itself, for clients of top-level resources. Defaults to `base_url`.
         """
         if resource_path.endswith('/'):
             raise ValueError('resource_path must not end with "/"')
 
         self._base_url = base_url
+        self._api_base_url = api_base_url or base_url
         self._public_base_url = public_base_url
         self._http_client = http_client
         self._default_params = params or {}
@@ -82,11 +85,12 @@ class ResourceClientBase(metaclass=WithLogDetailsClient):
     def _base_client_kwargs(self) -> dict[str, Any]:
         """Base kwargs for creating nested/child clients.
 
-        Returns dict with base_url, public_base_url, http_client, and client_registry. Caller adds
+        Returns dict with base_url, api_base_url, public_base_url, http_client, and client_registry. Caller adds
         resource_path, resource_id, and params as needed.
         """
         return {
             'base_url': self._resource_url,
+            'api_base_url': self._api_base_url,
             'public_base_url': self._public_base_url,
             'http_client': self._http_client,
             'client_registry': self._client_registry,
@@ -197,6 +201,7 @@ class ResourceClient(ResourceClientBase):
         client_registry: ClientRegistry,
         resource_id: str | None = None,
         params: dict | None = None,
+        api_base_url: str | None = None,
     ) -> None:
         """Initialize the resource client.
 
@@ -208,6 +213,7 @@ class ResourceClient(ResourceClientBase):
             client_registry: Bundle of client classes for dependency injection.
             resource_id: Optional resource ID for single-resource clients.
             params: Optional default parameters for all requests.
+            api_base_url: Base URL of the API itself, for clients of top-level resources. Defaults to `base_url`.
         """
         super().__init__(
             base_url=base_url,
@@ -217,6 +223,7 @@ class ResourceClient(ResourceClientBase):
             client_registry=client_registry,
             resource_id=resource_id,
             params=params,
+            api_base_url=api_base_url,
         )
 
     def _get(self, *, timeout: Timeout) -> dict | None:
@@ -389,6 +396,7 @@ class ResourceClientAsync(ResourceClientBase):
         client_registry: ClientRegistryAsync,
         resource_id: str | None = None,
         params: dict | None = None,
+        api_base_url: str | None = None,
     ) -> None:
         """Initialize the resource client.
 
@@ -400,6 +408,7 @@ class ResourceClientAsync(ResourceClientBase):
             client_registry: Bundle of client classes for dependency injection.
             resource_id: Optional resource ID for single-resource clients.
             params: Optional default parameters for all requests.
+            api_base_url: Base URL of the API itself, for clients of top-level resources. Defaults to `base_url`.
         """
         super().__init__(
             base_url=base_url,
@@ -409,6 +418,7 @@ class ResourceClientAsync(ResourceClientBase):
             client_registry=client_registry,
             resource_id=resource_id,
             params=params,
+            api_base_url=api_base_url,
         )
 
     async def _get(self, *, timeout: Timeout) -> dict | None:

{apify_client-3.2.2b2 → apify_client-3.2.2b3}/src/apify_client/_resource_clients/run.py

@@ -10,7 +10,8 @@ from typing import TYPE_CHECKING, Any
 from apify_client._docs import docs_group
 from apify_client._logging import create_redirect_logger
 from apify_client._models import Run, RunResponse
-from apify_client._resource_clients._resource_client import ResourceClient, ResourceClientAsync
+from apify_client._pagination import DEFAULT_CHUNK_SIZE
+from apify_client._resource_clients._resource_client import _TERMINAL_STATUSES, ResourceClient, ResourceClientAsync
 from apify_client._status_message_watcher import StatusMessageWatcher, StatusMessageWatcherAsync
 from apify_client._streamed_log import StreamedLog, StreamedLogAsync
 from apify_client._utils.encoding import encode_key_value_store_record_value
@@ -19,6 +20,7 @@ from apify_client._utils.time import to_seconds
 
 if TYPE_CHECKING:
     import logging
+    from collections.abc import AsyncIterator, Iterator
     from decimal import Decimal
 
     from apify_client._literals import GeneralAccess
@@ -32,6 +34,7 @@ if TYPE_CHECKING:
         RequestQueueClient,
         RequestQueueClientAsync,
     )
+    from apify_client._resource_clients.dataset import DatasetItemsPage
     from apify_client.types import Timeout
 
 
@@ -469,6 +472,122 @@ class RunClient(ResourceClient):
 
         return StatusMessageWatcher(run_client=self, to_logger=to_logger, check_period=check_period)
 
+    def iterate_dataset_items(
+        self,
+        *,
+        offset: int | None = None,
+        limit: int | None = None,
+        clean: bool | None = None,
+        fields: list[str] | None = None,
+        omit: list[str] | None = None,
+        unwind: list[str] | None = None,
+        skip_empty: bool | None = None,
+        skip_hidden: bool | None = None,
+        chunk_size: int | None = None,
+        poll_interval: timedelta = timedelta(seconds=5),
+        timeout: Timeout = 'long',
+    ) -> Iterator[dict]:
+        """Iterate over the items of the run's default dataset while the run is still producing them.
+
+        While the run has not finished, each poll yields the rows below the dataset's `item_count` and then waits up to
+        `poll_interval` for the run to finish, so the last rows are read as soon as it does. Each page is requested with
+        a `limit` that ends at `item_count`, so it covers exactly the rows it asks for, whatever the filters or `unwind`
+        do to the items. `item_count` lags a few seconds behind the pushed items, so once the run reaches a terminal
+        status, the rows past it are read a page at a time until none are left, and the iterator returns. On a
+        `last_run()` client, the iterator sticks to the run that its first request resolves to.
+
+        https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items
+
+        Args:
+            offset: Number of items that should be skipped at the start. The default value is 0.
+            limit: Maximum number of dataset rows to scan. Fewer items are yielded when filters drop some, more
+                when `unwind` splits a row into several. By default there is no limit.
+            clean: If True, returns only non-empty items and skips hidden fields (i.e. fields starting with
+                the # character). The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True
+                parameters.
+            fields: A list of fields which should be picked from the items, only these fields will remain in
+                the resulting record objects.
+            omit: A list of fields which should be omitted from the items.
+            unwind: A list of fields which should be unwound, in order which they should be processed. Each field
+                should be either an array or an object. If the field is an array then every element of the array
+                will become a separate record and merged with parent object. If the unwound field is an object then
+                it is merged with the parent object.
+            skip_empty: If True, then empty items are skipped from the output.
+            skip_hidden: If True, then hidden fields are skipped from the output, i.e. fields starting with
+                the # character.
+            chunk_size: Maximum number of dataset rows requested per API call.
+            poll_interval: How long to wait for the run to finish between polls.
+            timeout: Timeout for each API HTTP request.
+
+        Yields:
+            An item from the dataset.
+        """
+        page_size = chunk_size or DEFAULT_CHUNK_SIZE
+        position = offset or 0
+        end = position + limit if limit else None
+
+        run = self.get(timeout=timeout)
+        # A `last_run()` client resolves `runs/last` per request, so a newer run would swap the dataset mid-iteration.
+        run_client = (
+            self._client_registry.run_client(
+                resource_id=run.id,
+                base_url=self._api_base_url,
+                public_base_url=self._public_base_url,
+                http_client=self._http_client,
+                client_registry=self._client_registry,
+            )
+            if run is not None and run.id != self._resource_id
+            else self
+        )
+        dataset_client = run_client.dataset()
+
+        def list_page(page_offset: int, page_limit: int) -> DatasetItemsPage:
+            return dataset_client.list_items(
+                offset=page_offset,
+                limit=page_limit,
+                clean=clean,
+                fields=fields,
+                omit=omit,
+                unwind=unwind,
+                skip_empty=skip_empty,
+                skip_hidden=skip_hidden,
+                timeout=timeout,
+            )
+
+        while True:
+            is_finished = run is None or run.status in _TERMINAL_STATUSES
+            dataset = dataset_client.get(timeout=timeout)
+            item_count = dataset.item_count if dataset else 0
+            if end is not None:
+                item_count = min(item_count, end)
+
+            while position < item_count:
+                page_limit = min(page_size, item_count - position)
+                page = list_page(position, page_limit)
+                yield from page.items
+                position += page_limit
+
+            if end is not None and position >= end:
+                return
+            if is_finished:
+                break
+            run = run_client.wait_for_finish(wait_duration=poll_interval, timeout=timeout)
+
+        while True:
+            page_limit = min(page_size, end - position) if end is not None else page_size
+            page = list_page(position, page_limit)
+            yield from page.items
+            # Only an empty page marks the end, as filters can shorten a full one. A page that `clean`, `skip_empty` or
+            # `unwind` emptied past a lagging `item_count` reports no scanned rows either, so a plain read checks.
+            if not page.count and (
+                not (clean or skip_empty or unwind)
+                or not dataset_client.list_items(offset=position, limit=1, timeout=timeout).items
+            ):
+                return
+            position += page_limit
+            if end is not None and position >= end:
+                return
+
 
 @docs_group('Resource clients')
 class RunClientAsync(ResourceClientAsync):
@@ -903,3 +1022,121 @@ class RunClientAsync(ResourceClientAsync):
             to_logger = create_redirect_logger(f'apify.{name}')
 
         return StatusMessageWatcherAsync(run_client=self, to_logger=to_logger, check_period=check_period)
+
+    async def iterate_dataset_items(
+        self,
+        *,
+        offset: int | None = None,
+        limit: int | None = None,
+        clean: bool | None = None,
+        fields: list[str] | None = None,
+        omit: list[str] | None = None,
+        unwind: list[str] | None = None,
+        skip_empty: bool | None = None,
+        skip_hidden: bool | None = None,
+        chunk_size: int | None = None,
+        poll_interval: timedelta = timedelta(seconds=5),
+        timeout: Timeout = 'long',
+    ) -> AsyncIterator[dict]:
+        """Iterate over the items of the run's default dataset while the run is still producing them.
+
+        While the run has not finished, each poll yields the rows below the dataset's `item_count` and then waits up to
+        `poll_interval` for the run to finish, so the last rows are read as soon as it does. Each page is requested with
+        a `limit` that ends at `item_count`, so it covers exactly the rows it asks for, whatever the filters or `unwind`
+        do to the items. `item_count` lags a few seconds behind the pushed items, so once the run reaches a terminal
+        status, the rows past it are read a page at a time until none are left, and the iterator returns. On a
+        `last_run()` client, the iterator sticks to the run that its first request resolves to.
+
+        https://docs.apify.com/api/v2#/reference/datasets/item-collection/get-items
+
+        Args:
+            offset: Number of items that should be skipped at the start. The default value is 0.
+            limit: Maximum number of dataset rows to scan. Fewer items are yielded when filters drop some, more
+                when `unwind` splits a row into several. By default there is no limit.
+            clean: If True, returns only non-empty items and skips hidden fields (i.e. fields starting with
+                the # character). The clean parameter is just a shortcut for skip_hidden=True and skip_empty=True
+                parameters.
+            fields: A list of fields which should be picked from the items, only these fields will remain in
+                the resulting record objects.
+            omit: A list of fields which should be omitted from the items.
+            unwind: A list of fields which should be unwound, in order which they should be processed. Each field
+                should be either an array or an object. If the field is an array then every element of the array
+                will become a separate record and merged with parent object. If the unwound field is an object then
+                it is merged with the parent object.
+            skip_empty: If True, then empty items are skipped from the output.
+            skip_hidden: If True, then hidden fields are skipped from the output, i.e. fields starting with
+                the # character.
+            chunk_size: Maximum number of dataset rows requested per API call.
+            poll_interval: How long to wait for the run to finish between polls.
+            timeout: Timeout for each API HTTP request.
+
+        Yields:
+            An item from the dataset.
+        """
+        page_size = chunk_size or DEFAULT_CHUNK_SIZE
+        position = offset or 0
+        end = position + limit if limit else None
+
+        run = await self.get(timeout=timeout)
+        # A `last_run()` client resolves `runs/last` per request, so a newer run would swap the dataset mid-iteration.
+        run_client = (
+            self._client_registry.run_client(
+                resource_id=run.id,
+                base_url=self._api_base_url,
+                public_base_url=self._public_base_url,
+                http_client=self._http_client,
+                client_registry=self._client_registry,
+            )
+            if run is not None and run.id != self._resource_id
+            else self
+        )
+        dataset_client = run_client.dataset()
+
+        async def list_page(page_offset: int, page_limit: int) -> DatasetItemsPage:
+            return await dataset_client.list_items(
+                offset=page_offset,
+                limit=page_limit,
+                clean=clean,
+                fields=fields,
+                omit=omit,
+                unwind=unwind,
+                skip_empty=skip_empty,
+                skip_hidden=skip_hidden,
+                timeout=timeout,
+            )
+
+        while True:
+            is_finished = run is None or run.status in _TERMINAL_STATUSES
+            dataset = await dataset_client.get(timeout=timeout)
+            item_count = dataset.item_count if dataset else 0
+            if end is not None:
+                item_count = min(item_count, end)
+
+            while position < item_count:
+                page_limit = min(page_size, item_count - position)
+                page = await list_page(position, page_limit)
+                for item in page.items:
+                    yield item
+                position += page_limit
+
+            if end is not None and position >= end:
+                return
+            if is_finished:
+                break
+            run = await run_client.wait_for_finish(wait_duration=poll_interval, timeout=timeout)
+
+        while True:
+            page_limit = min(page_size, end - position) if end is not None else page_size
+            page = await list_page(position, page_limit)
+            for item in page.items:
+                yield item
+            # Only an empty page marks the end, as filters can shorten a full one. A page that `clean`, `skip_empty` or
+            # `unwind` emptied past a lagging `item_count` reports no scanned rows either, so a plain read checks.
+            if not page.count and (
+                not (clean or skip_empty or unwind)
+                or not (await dataset_client.list_items(offset=position, limit=1, timeout=timeout)).items
+            ):
+                return
+            position += page_limit
+            if end is not None and position >= end:
+                return