apify 3.0.2b7__tar.gz → 3.0.3__tar.gz

{apify-3.0.2b7 → apify-3.0.3}/CHANGELOG.md

@@ -2,8 +2,14 @@
 
 All notable changes to this project will be documented in this file.
 
-<!-- git-cliff-unreleased-start -->
-## 3.0.2 - **not yet released**
+## [3.0.3](https://github.com/apify/apify-sdk-python/releases/tag/v3.0.3) (2025-10-21)
+
+### 🐛 Bug Fixes
+
+- Cache requests in RQ implementations by `id` ([#633](https://github.com/apify/apify-sdk-python/pull/633)) ([76886ce](https://github.com/apify/apify-sdk-python/commit/76886ce496165346a01f67e018547287c211ea54)) by [@Pijukatel](https://github.com/Pijukatel), closes [#630](https://github.com/apify/apify-sdk-python/issues/630)
+
+
+## [3.0.2](https://github.com/apify/apify-sdk-python/releases/tag/v3.0.2) (2025-10-17)
 
 ### 🐛 Bug Fixes
 
@@ -12,7 +18,6 @@ All notable changes to this project will be documented in this file.
 - Handle truncated `unique_key` in `list_head` by fetching full request data ([#631](https://github.com/apify/apify-sdk-python/pull/631)) ([4238086](https://github.com/apify/apify-sdk-python/commit/423808678d9155a84a266bf50bb09f1a56466174)) by [@vdusek](https://github.com/vdusek), closes [#627](https://github.com/apify/apify-sdk-python/issues/627)
 
 
-<!-- git-cliff-unreleased-end -->
 ## [3.0.1](https://github.com/apify/apify-sdk-python/releases/tag/v3.0.1) (2025-10-08)
 
 ### 🐛 Bug Fixes

{apify-3.0.2b7 → apify-3.0.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.0.2b7
+Version: 3.0.3
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
@@ -225,7 +225,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
-Requires-Dist: apify-client<3.0.0,>=2.0.0
+Requires-Dist: apify-client<3.0.0,>=2.2.0
 Requires-Dist: apify-shared<3.0.0,>=2.0.0
 Requires-Dist: cachetools>=5.5.0
 Requires-Dist: crawlee<2.0.0,>=1.0.2

{apify-3.0.2b7 → apify-3.0.3}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "3.0.2b7"
+version = "3.0.3"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -34,7 +34,7 @@ keywords = [
     "scraping",
 ]
 dependencies = [
-    "apify-client>=2.0.0,<3.0.0",
+    "apify-client>=2.2.0,<3.0.0",
     "apify-shared>=2.0.0,<3.0.0",
     "crawlee>=1.0.2,<2.0.0",
     "cachetools>=5.5.0",

{apify-3.0.2b7 → apify-3.0.3}/src/apify/request_loaders/_apify_request_list.py

@@ -22,8 +22,8 @@ URL_NO_COMMAS_REGEX = re.compile(
 class _RequestDetails(BaseModel):
     method: HttpMethod = 'GET'
     payload: str = ''
-    headers: Annotated[dict[str, str], Field(default_factory=dict)] = {}
-    user_data: Annotated[dict[str, str], Field(default_factory=dict, alias='userData')] = {}
+    headers: Annotated[dict[str, str], Field(default_factory=dict)]
+    user_data: Annotated[dict[str, str], Field(default_factory=dict, alias='userData')]
 
 
 class _RequestsFromUrlInput(_RequestDetails):

{apify-3.0.2b7 → apify-3.0.3}/src/apify/storage_clients/_apify/_dataset_client.py

@@ -1,19 +1,19 @@
 from __future__ import annotations
 
 import asyncio
+import warnings
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
 from typing_extensions import override
 
-from apify_client import ApifyClientAsync
 from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.file import json_dumps
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 from crawlee.storages import Dataset
 
-from ._utils import AliasResolver
+from ._utils import AliasResolver, create_apify_client
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -52,12 +52,17 @@ class ApifyDatasetClient(DatasetClient):
         self._api_client = api_client
         """The Apify dataset client for API operations."""
 
-        self._api_public_base_url = api_public_base_url
-        """The public base URL for accessing the key-value store records."""
-
         self._lock = lock
         """A lock to ensure that only one operation is performed at a time."""
 
+        if api_public_base_url:
+            # Remove in version 4.0, https://github.com/apify/apify-sdk-python/issues/635
+            warnings.warn(
+                'api_public_base_url argument is deprecated and will be removed in version 4.0.0',
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
     @override
     async def get_metadata(self) -> DatasetMetadata:
         metadata = await self._api_client.get()
@@ -99,29 +104,7 @@ class ApifyDatasetClient(DatasetClient):
         if sum(1 for param in [id, name, alias] if param is not None) > 1:
             raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
 
-        token = configuration.token
-        if not token:
-            raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
-
-        api_url = configuration.api_base_url
-        if not api_url:
-            raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
-
-        api_public_base_url = configuration.api_public_base_url
-        if not api_public_base_url:
-            raise ValueError(
-                'Apify storage client requires a valid API public base URL in Configuration '
-                f'(api_public_base_url={api_public_base_url}).'
-            )
-
-        # Create Apify client with the provided token and API URL.
-        apify_client_async = ApifyClientAsync(
-            token=token,
-            api_url=api_url,
-            max_retries=8,
-            min_delay_between_retries_millis=500,
-            timeout_secs=360,
-        )
+        apify_client_async = create_apify_client(configuration)
         apify_datasets_client = apify_client_async.datasets()
 
         # Normalize unnamed default storage in cases where not defined in `configuration.default_dataset_id` to unnamed
@@ -178,7 +161,7 @@ class ApifyDatasetClient(DatasetClient):
 
         return cls(
             api_client=apify_dataset_client,
-            api_public_base_url=api_public_base_url,
+            api_public_base_url='',  # Remove in version 4.0, https://github.com/apify/apify-sdk-python/issues/635
             lock=asyncio.Lock(),
         )
 

{apify-3.0.2b7 → apify-3.0.3}/src/apify/storage_clients/_apify/_key_value_store_client.py

@@ -1,20 +1,18 @@
 from __future__ import annotations
 
 import asyncio
+import warnings
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
 from typing_extensions import override
-from yarl import URL
 
-from apify_client import ApifyClientAsync
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata
 from crawlee.storages import KeyValueStore
 
 from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage
-from ._utils import AliasResolver
-from apify._crypto import create_hmac_signature
+from ._utils import AliasResolver, create_apify_client
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -43,12 +41,17 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         self._api_client = api_client
         """The Apify KVS client for API operations."""
 
-        self._api_public_base_url = api_public_base_url
-        """The public base URL for accessing the key-value store records."""
-
         self._lock = lock
         """A lock to ensure that only one operation is performed at a time."""
 
+        if api_public_base_url:
+            # Remove in version 4.0, https://github.com/apify/apify-sdk-python/issues/635
+            warnings.warn(
+                'api_public_base_url argument is deprecated and will be removed in version 4.0.0',
+                DeprecationWarning,
+                stacklevel=2,
+            )
+
     @override
     async def get_metadata(self) -> ApifyKeyValueStoreMetadata:
         metadata = await self._api_client.get()
@@ -90,29 +93,7 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         if sum(1 for param in [id, name, alias] if param is not None) > 1:
             raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
 
-        token = configuration.token
-        if not token:
-            raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
-
-        api_url = configuration.api_base_url
-        if not api_url:
-            raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
-
-        api_public_base_url = configuration.api_public_base_url
-        if not api_public_base_url:
-            raise ValueError(
-                'Apify storage client requires a valid API public base URL in Configuration '
-                f'(api_public_base_url={api_public_base_url}).'
-            )
-
-        # Create Apify client with the provided token and API URL.
-        apify_client_async = ApifyClientAsync(
-            token=token,
-            api_url=api_url,
-            max_retries=8,
-            min_delay_between_retries_millis=500,
-            timeout_secs=360,
-        )
+        apify_client_async = create_apify_client(configuration)
         apify_kvss_client = apify_client_async.key_value_stores()
 
         # Normalize unnamed default storage in cases where not defined in `configuration.default_key_value_store_id` to
@@ -170,7 +151,7 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
 
         return cls(
             api_client=apify_kvs_client,
-            api_public_base_url=api_public_base_url,
+            api_public_base_url='',  # Remove in version 4.0, https://github.com/apify/apify-sdk-python/issues/635
             lock=asyncio.Lock(),
         )
 
@@ -251,15 +232,4 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         Returns:
             A public URL that can be used to access the value of the given key in the KVS.
         """
-        if self._api_client.resource_id is None:
-            raise ValueError('resource_id cannot be None when generating a public URL')
-
-        public_url = (
-            URL(self._api_public_base_url) / 'v2' / 'key-value-stores' / self._api_client.resource_id / 'records' / key
-        )
-        metadata = await self.get_metadata()
-
-        if metadata.url_signing_secret_key is not None:
-            public_url = public_url.with_query(signature=create_hmac_signature(metadata.url_signing_secret_key, key))
-
-        return str(public_url)
+        return await self._api_client.get_record_public_url(key=key)

{apify-3.0.2b7 → apify-3.0.3}/src/apify/storage_clients/_apify/_models.py

@@ -94,8 +94,8 @@ class CachedRequest(BaseModel):
     Only internal structure.
     """
 
-    unique_key: str
-    """Unique key of the request."""
+    id: str
+    """Id of the request."""
 
     was_already_handled: bool
     """Whether the request was already handled."""

{apify-3.0.2b7 → apify-3.0.3}/src/apify/storage_clients/_apify/_request_queue_client.py

@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Final, Literal
 
 from typing_extensions import override
 
-from apify_client import ApifyClientAsync
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
@@ -14,7 +13,7 @@ from crawlee.storages import RequestQueue
 from ._models import ApifyRequestQueueMetadata, RequestQueueStats
 from ._request_queue_shared_client import ApifyRequestQueueSharedClient
 from ._request_queue_single_client import ApifyRequestQueueSingleClient
-from ._utils import AliasResolver
+from ._utils import AliasResolver, create_apify_client
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -228,29 +227,7 @@ class ApifyRequestQueueClient(RequestQueueClient):
         if sum(1 for param in [id, name, alias] if param is not None) > 1:
             raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
 
-        token = configuration.token
-        if not token:
-            raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
-
-        api_url = configuration.api_base_url
-        if not api_url:
-            raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
-
-        api_public_base_url = configuration.api_public_base_url
-        if not api_public_base_url:
-            raise ValueError(
-                'Apify storage client requires a valid API public base URL in Configuration '
-                f'(api_public_base_url={api_public_base_url}).'
-            )
-
-        # Create Apify client with the provided token and API URL.
-        apify_client_async = ApifyClientAsync(
-            token=token,
-            api_url=api_url,
-            max_retries=8,
-            min_delay_between_retries_millis=500,
-            timeout_secs=360,
-        )
+        apify_client_async = create_apify_client(configuration)
         apify_rqs_client = apify_client_async.request_queues()
 
         # Normalize unnamed default storage in cases where not defined in `configuration.default_request_queue_id` to

{apify-3.0.2b7 → apify-3.0.3}/src/apify/storage_clients/_apify/_request_queue_shared_client.py

@@ -54,10 +54,10 @@ class ApifyRequestQueueSharedClient:
         """The Apify request queue client for API operations."""
 
         self._queue_head = deque[str]()
-        """A deque to store request unique keys in the queue head."""
+        """A deque to store request ids in the queue head."""
 
         self._requests_cache: LRUCache[str, CachedRequest] = LRUCache(maxsize=cache_size)
-        """A cache to store request objects. Request unique key is used as the cache key."""
+        """A cache to store request objects. Request id is used as the cache key."""
 
         self._queue_has_locked_requests: bool | None = None
         """Whether the queue has requests locked by another client."""
@@ -101,12 +101,14 @@ class ApifyRequestQueueSharedClient:
         already_present_requests: list[ProcessedRequest] = []
 
         for request in requests:
-            if self._requests_cache.get(request.unique_key):
+            request_id = unique_key_to_request_id(request.unique_key)
+            if self._requests_cache.get(request_id):
                 # We are not sure if it was already handled at this point, and it is not worth calling API for it.
                 # It could have been handled by another client in the meantime, so cached information about
                 # `request.was_already_handled` is not reliable.
                 already_present_requests.append(
                     ProcessedRequest(
+                        id=request_id,
                         unique_key=request.unique_key,
                         was_already_present=True,
                         was_already_handled=request.was_already_handled,
@@ -116,12 +118,13 @@ class ApifyRequestQueueSharedClient:
             else:
                 # Add new request to the cache.
                 processed_request = ProcessedRequest(
+                    id=request_id,
                     unique_key=request.unique_key,
                     was_already_present=True,
                     was_already_handled=request.was_already_handled,
                 )
                 self._cache_request(
-                    request.unique_key,
+                    request_id,
                     processed_request,
                 )
                 new_requests.append(request)
@@ -131,7 +134,6 @@ class ApifyRequestQueueSharedClient:
             requests_dict = [
                 request.model_dump(
                     by_alias=True,
-                    exclude={'id'},  # Exclude ID fields from requests since the API doesn't accept them.
                 )
                 for request in new_requests
             ]
@@ -146,7 +148,8 @@ class ApifyRequestQueueSharedClient:
 
             # Remove unprocessed requests from the cache
             for unprocessed_request in api_response.unprocessed_requests:
-                self._requests_cache.pop(unprocessed_request.unique_key, None)
+                unprocessed_request_id = unique_key_to_request_id(unprocessed_request.unique_key)
+                self._requests_cache.pop(unprocessed_request_id, None)
 
         else:
             api_response = AddRequestsResponse.model_validate(
@@ -179,7 +182,10 @@ class ApifyRequestQueueSharedClient:
         Returns:
             The request or None if not found.
         """
-        response = await self._api_client.get_request(unique_key_to_request_id(unique_key))
+        return await self._get_request_by_id(unique_key_to_request_id(unique_key))
+
+    async def _get_request_by_id(self, request_id: str) -> Request | None:
+        response = await self._api_client.get_request(request_id)
 
         if response is None:
             return None
@@ -206,15 +212,15 @@ class ApifyRequestQueueSharedClient:
                 return None
 
             # Get the next request ID from the queue head
-            next_unique_key = self._queue_head.popleft()
+            next_request_id = self._queue_head.popleft()
 
-        request = await self._get_or_hydrate_request(next_unique_key)
+        request = await self._get_or_hydrate_request(next_request_id)
 
         # Handle potential inconsistency where request might not be in the main table yet
         if request is None:
             logger.debug(
                 'Cannot find a request from the beginning of queue, will be retried later',
-                extra={'nextRequestUniqueKey': next_unique_key},
+                extra={'next_request_id': next_request_id},
             )
             return None
 
@@ -222,16 +228,16 @@ class ApifyRequestQueueSharedClient:
         if request.handled_at is not None:
             logger.debug(
                 'Request fetched from the beginning of queue was already handled',
-                extra={'nextRequestUniqueKey': next_unique_key},
+                extra={'next_request_id': next_request_id},
             )
             return None
 
         # Use get request to ensure we have the full request object.
-        request = await self.get_request(request.unique_key)
+        request = await self._get_request_by_id(next_request_id)
         if request is None:
             logger.debug(
                 'Request fetched from the beginning of queue was not found in the RQ',
-                extra={'nextRequestUniqueKey': next_unique_key},
+                extra={'next_request_id': next_request_id},
             )
             return None
 
@@ -248,15 +254,17 @@ class ApifyRequestQueueSharedClient:
         Returns:
             Information about the queue operation. `None` if the given request was not in progress.
         """
+        request_id = unique_key_to_request_id(request.unique_key)
         # Set the handled_at timestamp if not already set
         if request.handled_at is None:
             request.handled_at = datetime.now(tz=timezone.utc)
 
-        if cached_request := self._requests_cache[request.unique_key]:
+        if cached_request := self._requests_cache[request_id]:
             cached_request.was_already_handled = request.was_already_handled
         try:
             # Update the request in the API
             processed_request = await self._update_request(request)
+            processed_request.id = request_id
             processed_request.unique_key = request.unique_key
 
             # Update assumed handled count if this wasn't already handled
@@ -265,10 +273,9 @@ class ApifyRequestQueueSharedClient:
                 self.metadata.pending_request_count -= 1
 
             # Update the cache with the handled request
-            cache_key = request.unique_key
             self._cache_request(
-                cache_key,
-                processed_request,
+                cache_key=request_id,
+                processed_request=processed_request,
                 hydrated_request=request,
             )
         except Exception as exc:
@@ -352,17 +359,17 @@ class ApifyRequestQueueSharedClient:
         # Fetch requests from the API and populate the queue head
         await self._list_head()
 
-    async def _get_or_hydrate_request(self, unique_key: str) -> Request | None:
-        """Get a request by unique key, either from cache or by fetching from API.
+    async def _get_or_hydrate_request(self, request_id: str) -> Request | None:
+        """Get a request by id, either from cache or by fetching from API.
 
         Args:
-            unique_key: Unique key of the request to get.
+            request_id: Id of the request to get.
 
         Returns:
             The request if found and valid, otherwise None.
         """
         # First check if the request is in our cache
-        cached_entry = self._requests_cache.get(unique_key)
+        cached_entry = self._requests_cache.get(request_id)
 
         if cached_entry and cached_entry.hydrated:
             # If we have the request hydrated in cache, return it
@@ -371,17 +378,17 @@ class ApifyRequestQueueSharedClient:
         # If not in cache or not hydrated, fetch the request
         try:
             # Fetch the request data
-            request = await self.get_request(unique_key)
+            request = await self._get_request_by_id(request_id)
 
             # If request is not found and return None
             if not request:
                 return None
 
             # Update cache with hydrated request
-            cache_key = request.unique_key
             self._cache_request(
-                cache_key,
-                ProcessedRequest(
+                cache_key=request_id,
+                processed_request=ProcessedRequest(
+                    id=request_id,
                     unique_key=request.unique_key,
                     was_already_present=True,
                     was_already_handled=request.handled_at is not None,
@@ -389,7 +396,7 @@ class ApifyRequestQueueSharedClient:
                 hydrated_request=request,
             )
         except Exception as exc:
-            logger.debug(f'Error fetching request {unique_key}: {exc!s}')
+            logger.debug(f'Error fetching request {request_id}: {exc!s}')
             return None
         else:
             return request
@@ -438,8 +445,8 @@ class ApifyRequestQueueSharedClient:
             logger.debug(f'Using cached queue head with {len(self._queue_head)} requests')
             # Create a list of requests from the cached queue head
             items = []
-            for unique_key in list(self._queue_head)[:limit]:
-                cached_request = self._requests_cache.get(unique_key)
+            for request_id in list(self._queue_head)[:limit]:
+                cached_request = self._requests_cache.get(request_id)
                 if cached_request and cached_request.hydrated:
                     items.append(cached_request.hydrated)
 
@@ -472,32 +479,35 @@ class ApifyRequestQueueSharedClient:
 
         for request_data in response.get('items', []):
             request = Request.model_validate(request_data)
+            request_id = request_data.get('id')
 
             # Skip requests without ID or unique key
-            if not request.unique_key:
+            if not request.unique_key or not request_id:
                 logger.debug(
-                    'Skipping request from queue head, missing unique key',
+                    'Skipping request from queue head, missing unique key or id',
                     extra={
                         'unique_key': request.unique_key,
+                        'id': request_id,
                     },
                 )
                 continue
 
             # Cache the request
             self._cache_request(
-                request.unique_key,
+                request_id,
                 ProcessedRequest(
+                    id=request_id,
                     unique_key=request.unique_key,
                     was_already_present=True,
                     was_already_handled=False,
                 ),
                 hydrated_request=request,
             )
-            self._queue_head.append(request.unique_key)
+            self._queue_head.append(request_id)
 
-        for leftover_unique_key in leftover_buffer:
+        for leftover_id in leftover_buffer:
             # After adding new requests to the forefront, any existing leftover locked request is kept in the end.
-            self._queue_head.append(leftover_unique_key)
+            self._queue_head.append(leftover_id)
         return RequestQueueHead.model_validate(response)
 
     def _cache_request(
@@ -516,7 +526,7 @@ class ApifyRequestQueueSharedClient:
             hydrated_request: The hydrated request object, if available.
         """
         self._requests_cache[cache_key] = CachedRequest(
-            unique_key=processed_request.unique_key,
+            id=processed_request.id,
             was_already_handled=processed_request.was_already_handled,
             hydrated=hydrated_request,
             lock_expires_at=None,