apify 2.7.1b19__py3-none-any.whl → 2.7.1b21__py3-none-any.whl

apify/storage_clients/_apify/_request_queue_single_client.py

@@ -0,0 +1,399 @@
+from __future__ import annotations
+
+from collections import deque
+from datetime import datetime, timezone
+from logging import getLogger
+from typing import TYPE_CHECKING, Final
+
+from cachetools import LRUCache
+
+from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
+
+from apify import Request
+from apify.storage_clients._apify._utils import unique_key_to_request_id
+
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
+    from apify_client.clients import RequestQueueClientAsync
+
+
+logger = getLogger(__name__)
+
+
+class _ApifyRequestQueueSingleClient:
+    """An Apify platform implementation of the request queue client with limited capability.
+
+    This client is designed to use as little resources as possible, but has to be used in constrained context.
+    Constraints:
+    - Only one client is consuming the request queue at the time.
+    - Multiple producers can put requests to the queue, but their forefront requests are not guaranteed to be handled
+      so quickly as this client does not aggressively fetch the forefront and relies on local head estimation.
+    - Requests are only added to the queue, never deleted. (Marking as handled is ok.)
+    - Other producers can add new requests, but not modify existing ones (otherwise caching can miss the updates)
+
+    If the constraints are not met, the client might work in an unpredictable way.
+    """
+
+    _MAX_HEAD_ITEMS: Final[int] = 1000
+    """The maximum head items read count limited by API."""
+
+    def __init__(
+        self,
+        *,
+        api_client: RequestQueueClientAsync,
+        metadata: RequestQueueMetadata,
+        cache_size: int,
+    ) -> None:
+        """Initialize a new instance.
+
+        Preferably use the `ApifyRequestQueueClient.open` class method to create a new instance.
+        """
+        self.metadata = metadata
+        """Additional data related to the RequestQueue."""
+
+        self._api_client = api_client
+        """The Apify request queue client for API operations."""
+
+        self._requests_cache: LRUCache[str, Request] = LRUCache(maxsize=cache_size)
+        """A cache to store request objects. Request unique key is used as the cache key."""
+
+        self._head_requests: deque[str] = deque()
+        """Ordered unique keys of requests that represent queue head."""
+
+        self._requests_already_handled: set[str] = set()
+        """Local estimation of requests unique keys that are already present and handled on the platform.
+
+        - To enhance local deduplication.
+        - To reduce the _requests_cache size. Already handled requests are most likely not going to be needed again,
+        so no need to cache more than their unique_key.
+        """
+
+        self._requests_in_progress: set[str] = set()
+        """Set of requests unique keys that are being processed locally.
+
+        - To help decide if the RQ is finished or not. This is the only consumer, so it can be tracked locally.
+        """
+
+        self._initialized_caches = False
+        """This flag indicates whether the local caches were already initialized.
+
+        Initialization is done lazily only if deduplication is needed (When calling add_batch_of_requests).
+        """
+
+    async def add_batch_of_requests(
+        self,
+        requests: Sequence[Request],
+        *,
+        forefront: bool = False,
+    ) -> AddRequestsResponse:
+        """Add a batch of requests to the queue.
+
+        Args:
+            requests: The requests to add.
+            forefront: Whether to add the requests to the beginning of the queue.
+
+        Returns:
+            Response containing information about the added requests.
+        """
+        if not self._initialized_caches:
+            # One time process to initialize local caches for existing request queues.
+            await self._init_caches()
+            self._initialized_caches = True
+
+        new_requests: list[Request] = []
+        already_present_requests: list[ProcessedRequest] = []
+
+        for request in requests:
+            # Check if request is known to be already handled (it has to be present as well.)
+            if request.unique_key in self._requests_already_handled:
+                already_present_requests.append(
+                    ProcessedRequest.model_validate(
+                        {
+                            'uniqueKey': request.unique_key,
+                            'wasAlreadyPresent': True,
+                            'wasAlreadyHandled': True,
+                        }
+                    )
+                )
+            # Check if request is known to be already present, but unhandled
+            elif self._requests_cache.get(request.unique_key):
+                already_present_requests.append(
+                    ProcessedRequest.model_validate(
+                        {
+                            'uniqueKey': request.unique_key,
+                            'wasAlreadyPresent': True,
+                            'wasAlreadyHandled': request.was_already_handled,
+                        }
+                    )
+                )
+            else:
+                # Push the request to the platform. Probably not there, or we are not aware of it
+                new_requests.append(request)
+
+                # Update local caches
+                self._requests_cache[request.unique_key] = request
+                if forefront:
+                    self._head_requests.append(request.unique_key)
+                else:
+                    self._head_requests.appendleft(request.unique_key)
+
+        if new_requests:
+            # Prepare requests for API by converting to dictionaries.
+            requests_dict = [
+                request.model_dump(
+                    by_alias=True,
+                )
+                for request in new_requests
+            ]
+
+            # Send requests to API.
+            api_response = AddRequestsResponse.model_validate(
+                await self._api_client.batch_add_requests(requests=requests_dict, forefront=forefront)
+            )
+            # Add the locally known already present processed requests based on the local cache.
+            api_response.processed_requests.extend(already_present_requests)
+            # Remove unprocessed requests from the cache
+            for unprocessed_request in api_response.unprocessed_requests:
+                self._requests_cache.pop(unprocessed_request.unique_key, None)
+
+        else:
+            api_response = AddRequestsResponse.model_validate(
+                {'unprocessedRequests': [], 'processedRequests': already_present_requests}
+            )
+
+        # Update assumed total count for newly added requests.
+        new_request_count = 0
+        for processed_request in api_response.processed_requests:
+            if not processed_request.was_already_present and not processed_request.was_already_handled:
+                new_request_count += 1
+        self.metadata.total_request_count += new_request_count
+        self.metadata.pending_request_count += new_request_count
+
+        return api_response
+
+    async def get_request(self, unique_key: str) -> Request | None:
+        """Get a request by unique key.
+
+        Args:
+            unique_key: Unique key of the request to get.
+
+        Returns:
+            The request or None if not found.
+        """
+        if unique_key in self._requests_cache:
+            return self._requests_cache[unique_key]
+
+        response = await self._api_client.get_request(unique_key_to_request_id(unique_key))
+
+        if response is None:
+            return None
+
+        return Request.model_validate(response)
+
+    async def fetch_next_request(self) -> Request | None:
+        """Return the next request in the queue to be processed.
+
+        Once you successfully finish processing of the request, you need to call `mark_request_as_handled`
+        to mark the request as handled in the queue. If there was some error in processing the request, call
+        `reclaim_request` instead, so that the queue will give the request to some other consumer
+        in another call to the `fetch_next_request` method.
+
+        Returns:
+            The request or `None` if there are no more pending requests.
+        """
+        await self._ensure_head_is_non_empty()
+
+        while self._head_requests:
+            request_unique_key = self._head_requests.pop()
+            if (
+                request_unique_key not in self._requests_in_progress
+                and request_unique_key not in self._requests_already_handled
+            ):
+                self._requests_in_progress.add(request_unique_key)
+                return await self.get_request(request_unique_key)
+        # No request locally and the ones returned from the platform are already in progress.
+        return None
+
+    async def _ensure_head_is_non_empty(self) -> None:
+        """Ensure that the queue head has requests if they are available in the queue."""
+        if len(self._head_requests) <= 1:
+            await self._list_head()
+
+    async def _list_head(self) -> None:
+        desired_new_head_items = 200
+        # The head will contain in progress requests as well, so we need to fetch more, to get some new ones.
+        requested_head_items = max(self._MAX_HEAD_ITEMS, desired_new_head_items + len(self._requests_in_progress))
+        response = await self._api_client.list_head(limit=requested_head_items)
+
+        # Update metadata
+        # Check if there is another client working with the RequestQueue
+        self.metadata.had_multiple_clients = response.get('hadMultipleClients', False)
+        # Should warn once? This might be outside expected context if the other consumers consumes at the same time
+
+        if modified_at := response.get('queueModifiedAt'):
+            self.metadata.modified_at = max(self.metadata.modified_at, modified_at)
+
+        # Update the cached data
+        for request_data in response.get('items', []):
+            request = Request.model_validate(request_data)
+
+            if request.unique_key in self._requests_in_progress:
+                # Ignore requests that are already in progress, we will not process them again.
+                continue
+            if request.was_already_handled:
+                # Do not cache fully handled requests, we do not need them. Just cache their unique_key.
+                self._requests_already_handled.add(request.unique_key)
+            else:
+                # Only fetch the request if we do not know it yet.
+                if request.unique_key not in self._requests_cache:
+                    request = Request.model_validate(
+                        await self._api_client.get_request(unique_key_to_request_id(request.unique_key))
+                    )
+                    self._requests_cache[request.unique_key] = request
+
+                # Add new requests to the end of the head, unless already present in head
+                if request.unique_key not in self._head_requests:
+                    self._head_requests.appendleft(request.unique_key)
+
+    async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None:
+        """Mark a request as handled after successful processing.
+
+        Handled requests will never again be returned by the `fetch_next_request` method.
+
+        Args:
+            request: The request to mark as handled.
+
+        Returns:
+            Information about the queue operation. `None` if the given request was not in progress.
+        """
+        # Set the handled_at timestamp if not already set
+
+        if request.handled_at is None:
+            request.handled_at = datetime.now(tz=timezone.utc)
+            self.metadata.handled_request_count += 1
+            self.metadata.pending_request_count -= 1
+
+        if cached_request := self._requests_cache.get(request.unique_key):
+            cached_request.handled_at = request.handled_at
+
+        try:
+            # Update the request in the API
+            # Works as upsert - adds the request if it does not exist yet. (Local request that was handled before
+            # adding to the queue.)
+            processed_request = await self._update_request(request)
+            # Remember that we handled this request, to optimize local deduplication.
+            self._requests_already_handled.add(request.unique_key)
+            # Remove request from cache. It will most likely not be needed.
+            self._requests_cache.pop(request.unique_key)
+            self._requests_in_progress.discard(request.unique_key)
+
+        except Exception as exc:
+            logger.debug(f'Error marking request {request.unique_key} as handled: {exc!s}')
+            return None
+        else:
+            return processed_request
+
+    async def reclaim_request(
+        self,
+        request: Request,
+        *,
+        forefront: bool = False,
+    ) -> ProcessedRequest | None:
+        """Reclaim a failed request back to the queue.
+
+        The request will be returned for processing later again by another call to `fetch_next_request`.
+
+        Args:
+            request: The request to return to the queue.
+            forefront: Whether to add the request to the head or the end of the queue.
+
+        Returns:
+            Information about the queue operation. `None` if the given request was not in progress.
+        """
+        # Check if the request was marked as handled and clear it. When reclaiming,
+        # we want to put the request back for processing.
+        if request.was_already_handled:
+            request.handled_at = None
+
+        try:
+            # Make sure request is in the local cache. We might need it.
+            self._requests_cache[request.unique_key] = request
+
+            # No longer in progress
+            self._requests_in_progress.discard(request.unique_key)
+            # No longer handled
+            self._requests_already_handled.discard(request.unique_key)
+
+            if forefront:
+                # Append to top of the local head estimation
+                self._head_requests.append(request.unique_key)
+
+            processed_request = await self._update_request(request, forefront=forefront)
+            processed_request.unique_key = request.unique_key
+            # If the request was previously handled, decrement our handled count since
+            # we're putting it back for processing.
+            if request.was_already_handled and not processed_request.was_already_handled:
+                self.metadata.handled_request_count -= 1
+                self.metadata.pending_request_count += 1
+
+        except Exception as exc:
+            logger.debug(f'Error reclaiming request {request.unique_key}: {exc!s}')
+            return None
+        else:
+            return processed_request
+
+    async def is_empty(self) -> bool:
+        """Check if the queue is empty.
+
+        Returns:
+            True if the queue is empty, False otherwise.
+        """
+        # Without the lock the `is_empty` is prone to falsely report True with some low probability race condition.
+        await self._ensure_head_is_non_empty()
+        return not self._head_requests and not self._requests_in_progress
+
+    async def _update_request(
+        self,
+        request: Request,
+        *,
+        forefront: bool = False,
+    ) -> ProcessedRequest:
+        """Update a request in the queue.
+
+        Args:
+            request: The updated request.
+            forefront: Whether to put the updated request in the beginning or the end of the queue.
+
+        Returns:
+            The updated request
+        """
+        request_dict = request.model_dump(by_alias=True)
+        request_dict['id'] = unique_key_to_request_id(request.unique_key)
+        response = await self._api_client.update_request(
+            request=request_dict,
+            forefront=forefront,
+        )
+
+        return ProcessedRequest.model_validate(
+            {'uniqueKey': request.unique_key} | response,
+        )
+
+    async def _init_caches(self) -> None:
+        """Initialize the local caches by getting requests from the existing queue.
+
+        This is mainly done to improve local deduplication capability. List request can return up to 10k requests, but
+        their order is implementation detail and does not respect head order or insertion order.
+
+        Deduplication on platform is expensive, it takes 1 API call per request and 1 write operation per request.
+        Local deduplication is cheaper, it takes 1 API call for whole cache and 1 read operation per request.
+        """
+        response = await self._api_client.list_requests(limit=10_000)
+        for request_data in response.get('items', []):
+            request = Request.model_validate(request_data)
+            if request.was_already_handled:
+                # Cache just unique_key for deduplication
+                self._requests_already_handled.add(request.unique_key)
+            else:
+                # Cache full request
+                self._requests_cache[request.unique_key] = request

apify/storage_clients/_apify/_storage_client.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Literal
 
 from typing_extensions import override
 
@@ -23,15 +23,35 @@ if TYPE_CHECKING:
 class ApifyStorageClient(StorageClient):
     """Apify storage client."""
 
+    def __init__(self, *, request_queue_access: Literal['single', 'shared'] = 'single') -> None:
+        """Initialize the Apify storage client.
+
+        Args:
+            request_queue_access: Controls the implementation of the request queue client based on expected scenario:
+                - 'single' is suitable for single consumer scenarios. It makes less API calls, is cheaper and faster.
+                - 'shared' is suitable for multiple consumers scenarios at the cost of higher API usage.
+                Detailed constraints for the 'single' access type:
+                - Only one client is consuming the request queue at the time.
+                - Multiple producers can put requests to the queue, but their forefront requests are not guaranteed to
+                  be handled so quickly as this client does not aggressively fetch the forefront and relies on local
+                  head estimation.
+                - Requests are only added to the queue, never deleted by other clients. (Marking as handled is ok.)
+                - Other producers can add new requests, but not modify existing ones.
+                  (Modifications would not be included in local cache)
+        """
+        self._request_queue_access = request_queue_access
+
     # This class breaches Liskov Substitution Principle. It requires specialized Configuration compared to its parent.
     _lsp_violation_error_message_template = (
         'Expected "configuration" to be an instance of "apify.Configuration", but got {} instead.'
     )
 
     @override
-    def get_additional_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
+    def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
         if isinstance(configuration, ApifyConfiguration):
-            return hash_api_base_url_and_token(configuration)
+            # It is not supported to open exactly same queue with 'single' and 'shared' client at the same time.
+            # Whichever client variation gets used first, wins.
+            return super().get_storage_client_cache_key(configuration), hash_api_base_url_and_token(configuration)
 
         config_class = type(configuration)
         raise TypeError(
@@ -79,6 +99,8 @@ class ApifyStorageClient(StorageClient):
     ) -> ApifyRequestQueueClient:
         configuration = configuration or ApifyConfiguration.get_global_configuration()
         if isinstance(configuration, ApifyConfiguration):
-            return await ApifyRequestQueueClient.open(id=id, name=name, alias=alias, configuration=configuration)
+            return await ApifyRequestQueueClient.open(
+                id=id, name=name, alias=alias, configuration=configuration, access=self._request_queue_access
+            )
 
         raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))

apify/storage_clients/_apify/_utils.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
 import logging
+import re
 from asyncio import Lock
+from base64 import b64encode
+from hashlib import sha256
 from logging import getLogger
 from typing import TYPE_CHECKING, ClassVar
 
@@ -76,7 +79,7 @@ class AliasResolver:
         Returns:
             Map of aliases and storage ids.
         """
-        if not cls._alias_map:
+        if not cls._alias_map and Configuration.get_global_configuration().is_at_home:
             default_kvs_client = await _get_default_kvs_client()
 
             record = await default_kvs_client.get_record(cls._ALIAS_MAPPING_KEY)
@@ -107,7 +110,7 @@ class AliasResolver:
         # Update in-memory mapping
         (await self._get_alias_map())[self._storage_key] = storage_id
         if not Configuration.get_global_configuration().is_at_home:
-            logging.getLogger(__name__).warning(
+            logging.getLogger(__name__).debug(
                 'AliasResolver storage limited retention is only supported on Apify platform. Storage is not exported.'
             )
             return
@@ -156,7 +159,8 @@ async def _get_default_kvs_client() -> KeyValueStoreClientAsync:
         min_delay_between_retries_millis=500,
         timeout_secs=360,
     )
-
+    if not configuration.default_key_value_store_id:
+        raise ValueError("'Configuration.default_key_value_store_id' must be set.")
     return apify_client_async.key_value_store(key_value_store_id=configuration.default_key_value_store_id)
 
 
@@ -165,3 +169,26 @@ def hash_api_base_url_and_token(configuration: Configuration) -> str:
     if configuration.api_public_base_url is None or configuration.token is None:
         raise ValueError("'Configuration.api_public_base_url' and 'Configuration.token' must be set.")
     return compute_short_hash(f'{configuration.api_public_base_url}{configuration.token}'.encode())
+
+
+def unique_key_to_request_id(unique_key: str, *, request_id_length: int = 15) -> str:
+    """Generate a deterministic request ID based on a unique key.
+
+    Args:
+        unique_key: The unique key to convert into a request ID.
+        request_id_length: The length of the request ID.
+
+    Returns:
+        A URL-safe, truncated request ID based on the unique key.
+    """
+    # Encode the unique key and compute its SHA-256 hash
+    hashed_key = sha256(unique_key.encode('utf-8')).digest()
+
+    # Encode the hash in base64 and decode it to get a string
+    base64_encoded = b64encode(hashed_key).decode('utf-8')
+
+    # Remove characters that are not URL-safe ('+', '/', or '=')
+    url_safe_key = re.sub(r'(\+|\/|=)', '', base64_encoded)
+
+    # Truncate the key to the desired length
+    return url_safe_key[:request_id_length]

apify/storage_clients/_smart_apify/__init__.py

@@ -0,0 +1 @@
+from ._storage_client import SmartApifyStorageClient

apify/storage_clients/_smart_apify/_storage_client.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from typing_extensions import override
+
+from crawlee.storage_clients._base import DatasetClient, KeyValueStoreClient, RequestQueueClient, StorageClient
+
+from apify._configuration import Configuration as ApifyConfiguration
+from apify._utils import docs_group
+from apify.storage_clients import ApifyStorageClient
+from apify.storage_clients._file_system import ApifyFileSystemStorageClient
+
+if TYPE_CHECKING:
+    from collections.abc import Hashable
+
+    from crawlee.configuration import Configuration as CrawleeConfiguration
+
+
+@docs_group('Storage clients')
+class SmartApifyStorageClient(StorageClient):
+    """SmartApifyStorageClient that delegates to cloud_storage_client or local_storage_client.
+
+    When running on Apify platform use cloud_storage_client, else use local_storage_client. This storage client is
+    designed to work specifically in Actor context.
+    """
+
+    def __init__(
+        self,
+        *,
+        cloud_storage_client: ApifyStorageClient | None = None,
+        local_storage_client: StorageClient | None = None,
+    ) -> None:
+        """Initialize the Apify storage client.
+
+        Args:
+            cloud_storage_client: Client used to communicate with the Apify platform storage. Either through
+                `force_cloud` argument when opening storages or automatically when running on the Apify platform.
+            local_storage_client: Client used to communicate with the storage when not running on the Apify
+                platform and not using `force_cloud` argument when opening storages.
+        """
+        self._cloud_storage_client = cloud_storage_client or ApifyStorageClient(request_queue_access='single')
+        self._local_storage_client = local_storage_client or ApifyFileSystemStorageClient()
+
+    def __str__(self) -> str:
+        return (
+            f'{self.__class__.__name__}(cloud_storage_client={self._cloud_storage_client.__class__.__name__},'
+            f' local_storage_client={self._local_storage_client.__class__.__name__})'
+        )
+
+    def get_suitable_storage_client(self, *, force_cloud: bool = False) -> StorageClient:
+        """Get a suitable storage client based on the global configuration and the value of the force_cloud flag.
+
+        Args:
+            force_cloud: If True, return `cloud_storage_client`.
+        """
+        if ApifyConfiguration.get_global_configuration().is_at_home:
+            return self._cloud_storage_client
+
+        configuration = ApifyConfiguration.get_global_configuration()
+        if force_cloud:
+            if configuration.token is None:
+                raise RuntimeError(
+                    'In order to use the Apify cloud storage from your computer, '
+                    'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
+                )
+            return self._cloud_storage_client
+
+        return self._local_storage_client
+
+    @override
+    def get_storage_client_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
+        if ApifyConfiguration.get_global_configuration().is_at_home:
+            if isinstance(configuration, ApifyConfiguration):
+                return self._cloud_storage_client.get_storage_client_cache_key(configuration)
+            raise TypeError('Expecting ApifyConfiguration')
+
+        return self._local_storage_client.get_storage_client_cache_key(configuration)
+
+    @override
+    async def create_dataset_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: CrawleeConfiguration | None = None,
+    ) -> DatasetClient:
+        return await self.get_suitable_storage_client().create_dataset_client(
+            id=id, name=id, alias=alias, configuration=configuration
+        )
+
+    @override
+    async def create_kvs_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: CrawleeConfiguration | None = None,
+    ) -> KeyValueStoreClient:
+        return await self.get_suitable_storage_client().create_kvs_client(
+            id=id, name=id, alias=alias, configuration=configuration
+        )
+
+    @override
+    async def create_rq_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: CrawleeConfiguration | None = None,
+    ) -> RequestQueueClient:
+        return await self.get_suitable_storage_client().create_rq_client(
+            id=id, name=id, alias=alias, configuration=configuration
+        )

{apify-2.7.1b19.dist-info → apify-2.7.1b21.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 2.7.1b19
+Version: 2.7.1b21
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
@@ -228,7 +228,7 @@ Requires-Python: >=3.10
 Requires-Dist: apify-client<3.0.0,>=2.0.0
 Requires-Dist: apify-shared<3.0.0,>=2.0.0
 Requires-Dist: cachetools>=5.5.0
-Requires-Dist: crawlee==0.6.13b42
+Requires-Dist: crawlee==0.6.13b46
 Requires-Dist: cryptography>=42.0.0
 Requires-Dist: impit>=0.6.1
 Requires-Dist: lazy-object-proxy>=1.11.0