apify 3.4.2b29__tar.gz → 3.4.2b30__tar.gz

{apify-3.4.2b29 → apify-3.4.2b30}/CHANGELOG.md

@@ -32,6 +32,7 @@ All notable changes to this project will be documented in this file.
 - **scrapy:** Avoid mutating request userData during Scrapy-Apify conversion ([#978](https://github.com/apify/apify-sdk-python/pull/978)) ([b0b7df7](https://github.com/apify/apify-sdk-python/commit/b0b7df72eb169778ab88be04d8b30bb0bdc307d3)) by [@vdusek](https://github.com/vdusek)
 - **scrapy:** Async-thread startup race, shutdown lifecycle, and timeout setting ([#979](https://github.com/apify/apify-sdk-python/pull/979)) ([ae12935](https://github.com/apify/apify-sdk-python/commit/ae1293512f5ee781533dab5b1dd1f0af0fcc2497)) by [@vdusek](https://github.com/vdusek)
 - Commit request queue dedup cache only after batch_add_requests succeeds ([#975](https://github.com/apify/apify-sdk-python/pull/975)) ([078ab87](https://github.com/apify/apify-sdk-python/commit/078ab8744c96e61a5226a9d19869b5e0df71ab23)) by [@vdusek](https://github.com/vdusek)
+- Prevent request queue softlock by adding `is_finished` and correcting `is_empty` ([#1008](https://github.com/apify/apify-sdk-python/pull/1008)) ([4ead0c6](https://github.com/apify/apify-sdk-python/commit/4ead0c64d2a95263b2fa970f5a8fff9141db62b2)) by [@Mantisus](https://github.com/Mantisus), closes [#987](https://github.com/apify/apify-sdk-python/issues/987)
 
 ### 🚜 Refactor
 

{apify-3.4.2b29 → apify-3.4.2b30}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 3.4.2b29
+Version: 3.4.2b30
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
@@ -228,7 +228,7 @@ Classifier: Typing :: Typed
 Requires-Python: >=3.11
 Requires-Dist: apify-client<4.0.0,>=3.0.0
 Requires-Dist: cachetools>=5.5.0
-Requires-Dist: crawlee<2.0.0,>=1.0.4
+Requires-Dist: crawlee<2.0.0,>=1.8.0
 Requires-Dist: cryptography>=42.0.0
 Requires-Dist: impit>=0.8.0
 Requires-Dist: lazy-object-proxy>=1.11.0

{apify-3.4.2b29 → apify-3.4.2b30}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "apify"
-version = "3.4.2b29"
+version = "3.4.2b30"
 description = "Apify SDK for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -36,7 +36,7 @@ keywords = [
 ]
 dependencies = [
     "apify-client>=3.0.0,<4.0.0",
-    "crawlee>=1.0.4,<2.0.0",
+    "crawlee>=1.8.0,<2.0.0",
     "cachetools>=5.5.0",
     "cryptography>=42.0.0",
     "impit>=0.8.0",

{apify-3.4.2b29 → apify-3.4.2b30}/src/apify/storage_clients/_apify/_request_queue_client.py

@@ -198,3 +198,7 @@ class ApifyRequestQueueClient(RequestQueueClient):
     @override
     async def is_empty(self) -> bool:
         return await self._implementation.is_empty()
+
+    @override
+    async def is_finished(self) -> bool:
+        return await self._implementation.is_finished()

{apify-3.4.2b29 → apify-3.4.2b30}/src/apify/storage_clients/_apify/_request_queue_shared_client.py

@@ -338,8 +338,18 @@ class ApifyRequestQueueSharedClient:
         # Check _list_head.
         # Without the lock the `is_empty` is prone to falsely report True with some low probability race condition.
         async with self._fetch_lock:
-            head = await self._list_head(limit=1)
-            return len(head.items) == 0 and not self._queue_has_locked_requests
+            return await self._is_empty()
+
+    async def is_finished(self) -> bool:
+        """Specific implementation of this method for the RQ shared access mode."""
+        async with self._fetch_lock:
+            # Order of operations is important here, because affects on `_queue_has_locked_requests`.
+            return await self._is_empty() and not self._queue_has_locked_requests
+
+    async def _is_empty(self) -> bool:
+        """Check whether anything is available to fetch. Lock-free core of `is_empty`, caller must hold the lock."""
+        head = await self._list_head(limit=1)
+        return len(head.items) == 0
 
     async def _get_metadata_estimate(self) -> RequestQueueMetadata:
         """Try to get cached metadata first. If multiple clients, fuse with global metadata.

{apify-3.4.2b29 → apify-3.4.2b30}/src/apify/storage_clients/_apify/_request_queue_single_client.py

@@ -324,9 +324,12 @@ class ApifyRequestQueueSingleClient:
 
     async def is_empty(self) -> bool:
         """Specific implementation of this method for the RQ single access mode."""
-        # Without the lock the `is_empty` is prone to falsely report True with some low probability race condition.
         await self._ensure_head_is_non_empty()
-        return not self._head_requests and not self._requests_in_progress
+        return not self._head_requests
+
+    async def is_finished(self) -> bool:
+        """Specific implementation of this method for the RQ single access mode."""
+        return await self.is_empty() and not self._requests_in_progress
 
     async def _ensure_head_is_non_empty(self) -> None:
         """Ensure that the queue head has requests if they are available in the queue."""