apify 1.4.0b1__tar.gz → 1.4.1__tar.gz

{apify-1.4.0b1 → apify-1.4.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.4.0b1
+Version: 1.4.1
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License
@@ -24,10 +24,10 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aiofiles>=22.1.0
-Requires-Dist: aioshutil>=1.0
 Requires-Dist: apify-client~=1.6.0
 Requires-Dist: apify-shared~=1.1.0
+Requires-Dist: aiofiles>=22.1.0
+Requires-Dist: aioshutil>=1.0
 Requires-Dist: colorama>=0.4.6
 Requires-Dist: cryptography>=39.0.0
 Requires-Dist: httpx>=0.24.1
@@ -51,10 +51,10 @@ Requires-Dist: respx~=0.20.1; extra == "dev"
 Requires-Dist: ruff~=0.1.6; extra == "dev"
 Requires-Dist: twine~=4.0.2; extra == "dev"
 Requires-Dist: types-aiofiles~=23.2.0.0; extra == "dev"
-Requires-Dist: types-colorama~=0.4.15.11; extra == "dev"
-Requires-Dist: types-psutil~=5.9.5.12; extra == "dev"
+Requires-Dist: types-colorama~=0.4.15.12; extra == "dev"
+Requires-Dist: types-psutil~=5.9.5.17; extra == "dev"
 Provides-Extra: scrapy
-Requires-Dist: scrapy~=2.11.0; extra == "scrapy"
+Requires-Dist: scrapy>=2.11.0; extra == "scrapy"
 
 # Apify SDK for Python
 

{apify-1.4.0b1 → apify-1.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "apify"
-version = "1.4.0b1"
+version = "1.4.1"
 description = "Apify SDK for Python"
 readme = "README.md"
 license = { text = "Apache Software License" }
@@ -21,11 +21,15 @@ classifiers = [
 ]
 
 requires-python = ">=3.8"
+
+# We use inclusive ordered comparison clause for non-Apify packages intentionally in order to enhance the Apify SDK's
+# compatibility with a wide range of external packages. This decision was discussed in detail in the following PR:
+# https://github.com/apify/apify-sdk-python/pull/154
 dependencies = [
-    "aiofiles >= 22.1.0",
-    "aioshutil >= 1.0",
     "apify-client ~= 1.6.0",
     "apify-shared ~= 1.1.0",
+    "aiofiles >= 22.1.0",
+    "aioshutil >= 1.0",
     "colorama >= 0.4.6",
     "cryptography >= 39.0.0",
     "httpx >= 0.24.1",
@@ -52,11 +56,11 @@ dev = [
     "ruff ~= 0.1.6",
     "twine ~= 4.0.2",
     "types-aiofiles ~= 23.2.0.0",
-    "types-colorama ~= 0.4.15.11",
-    "types-psutil ~= 5.9.5.12",
+    "types-colorama ~= 0.4.15.12",
+    "types-psutil ~= 5.9.5.17",
 ]
 scrapy = [
-    "scrapy ~= 2.11.0",
+    "scrapy >= 2.11.0",
 ]
 
 [project.urls]

{apify-1.4.0b1 → apify-1.4.1}/src/apify/_memory_storage/resource_clients/base_resource_client.py

@@ -8,6 +8,8 @@ from typing import TYPE_CHECKING
 from apify_shared.utils import ignore_docs
 
 if TYPE_CHECKING:
+    from typing_extensions import Self
+
     from ..memory_storage_client import MemoryStorageClient
 
 
@@ -48,9 +50,9 @@ class BaseResourceClient(ABC):
     @classmethod
     @abstractmethod
     def _get_storage_client_cache(
-        cls: type[BaseResourceClient],
+        cls,  # noqa: ANN102 # type annotated cls does not work with Self as a return type
         memory_storage_client: MemoryStorageClient,
-    ) -> list[BaseResourceClient]:
+    ) -> list[Self]:
         raise NotImplementedError('You must override this method in the subclass!')
 
     @abstractmethod
@@ -60,21 +62,21 @@ class BaseResourceClient(ABC):
     @classmethod
     @abstractmethod
     def _create_from_directory(
-        cls: type[BaseResourceClient],
+        cls,  # noqa: ANN102 # type annotated cls does not work with Self as a return type
         storage_directory: str,
         memory_storage_client: MemoryStorageClient,
         id: str | None = None,  # noqa: A002
         name: str | None = None,
-    ) -> BaseResourceClient:
+    ) -> Self:
         raise NotImplementedError('You must override this method in the subclass!')
 
     @classmethod
     def _find_or_create_client_by_id_or_name(
-        cls: type[BaseResourceClient],
+        cls,  # noqa: ANN102 # type annotated cls does not work with Self as a return type
         memory_storage_client: MemoryStorageClient,
         id: str | None = None,  # noqa: A002
         name: str | None = None,
-    ) -> BaseResourceClient | None:
+    ) -> Self | None:
         assert id is not None or name is not None  # noqa: S101
 
         storage_client_cache = cls._get_storage_client_cache(memory_storage_client)

{apify-1.4.0b1 → apify-1.4.1}/src/apify/_memory_storage/resource_clients/dataset.py

@@ -74,8 +74,8 @@ class DatasetClient(BaseResourceClient):
         found = self._find_or_create_client_by_id_or_name(memory_storage_client=self._memory_storage_client, id=self._id, name=self._name)
 
         if found:
-            async with found._file_operation_lock:  # type: ignore
-                await found._update_timestamps(has_been_modified=False)  # type: ignore
+            async with found._file_operation_lock:
+                await found._update_timestamps(has_been_modified=False)
                 return found._to_resource_info()
 
         return None
@@ -103,7 +103,7 @@ class DatasetClient(BaseResourceClient):
         if name is None:
             return existing_dataset_by_id._to_resource_info()
 
-        async with existing_dataset_by_id._file_operation_lock:  # type: ignore
+        async with existing_dataset_by_id._file_operation_lock:
             # Check that name is not in use already
             existing_dataset_by_name = next(
                 (dataset for dataset in self._memory_storage_client._datasets_handled if dataset._name and dataset._name.lower() == name.lower()),
@@ -122,7 +122,7 @@ class DatasetClient(BaseResourceClient):
             await force_rename(previous_dir, existing_dataset_by_id._resource_directory)
 
             # Update timestamps
-            await existing_dataset_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+            await existing_dataset_by_id._update_timestamps(has_been_modified=True)
 
         return existing_dataset_by_id._to_resource_info()
 
@@ -193,9 +193,9 @@ class DatasetClient(BaseResourceClient):
         if existing_dataset_by_id is None:
             raise_on_non_existing_storage(StorageTypes.DATASET, self._id)
 
-        async with existing_dataset_by_id._file_operation_lock:  # type: ignore
-            start, end = existing_dataset_by_id._get_start_and_end_indexes(  # type: ignore
-                max(existing_dataset_by_id._item_count - (offset or 0) - (limit or LIST_ITEMS_LIMIT), 0) if desc else offset or 0,  # type: ignore
+        async with existing_dataset_by_id._file_operation_lock:
+            start, end = existing_dataset_by_id._get_start_and_end_indexes(
+                max(existing_dataset_by_id._item_count - (offset or 0) - (limit or LIST_ITEMS_LIMIT), 0) if desc else offset or 0,
                 limit,
             )
 
@@ -203,9 +203,9 @@ class DatasetClient(BaseResourceClient):
 
             for idx in range(start, end):
                 entry_number = self._generate_local_entry_name(idx)
-                items.append(existing_dataset_by_id._dataset_entries[entry_number])  # type: ignore
+                items.append(existing_dataset_by_id._dataset_entries[entry_number])
 
-            await existing_dataset_by_id._update_timestamps(has_been_modified=False)  # type: ignore
+            await existing_dataset_by_id._update_timestamps(has_been_modified=False)
 
             if desc:
                 items.reverse()
@@ -217,7 +217,7 @@ class DatasetClient(BaseResourceClient):
                     'items': items,
                     'limit': limit or LIST_ITEMS_LIMIT,
                     'offset': offset or 0,
-                    'total': existing_dataset_by_id._item_count,  # type: ignore
+                    'total': existing_dataset_by_id._item_count,
                 }
             )
 
@@ -308,16 +308,16 @@ class DatasetClient(BaseResourceClient):
 
         added_ids: list[str] = []
         for entry in normalized:
-            existing_dataset_by_id._item_count += 1  # type: ignore
-            idx = self._generate_local_entry_name(existing_dataset_by_id._item_count)  # type: ignore
+            existing_dataset_by_id._item_count += 1
+            idx = self._generate_local_entry_name(existing_dataset_by_id._item_count)
 
-            existing_dataset_by_id._dataset_entries[idx] = entry  # type: ignore
+            existing_dataset_by_id._dataset_entries[idx] = entry
             added_ids.append(idx)
 
-        data_entries = [(id, existing_dataset_by_id._dataset_entries[id]) for id in added_ids]  # type: ignore # noqa: A001
+        data_entries = [(id, existing_dataset_by_id._dataset_entries[id]) for id in added_ids] # noqa: A001
 
-        async with existing_dataset_by_id._file_operation_lock:  # type: ignore
-            await existing_dataset_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+        async with existing_dataset_by_id._file_operation_lock:
+            await existing_dataset_by_id._update_timestamps(has_been_modified=True)
 
             await _update_dataset_items(
                 data=data_entries,
@@ -385,7 +385,7 @@ class DatasetClient(BaseResourceClient):
         return memory_storage_client._datasets_directory
 
     @classmethod
-    def _get_storage_client_cache(  # type: ignore
+    def _get_storage_client_cache(
         cls: type[DatasetClient],
         memory_storage_client: MemoryStorageClient,
     ) -> list[DatasetClient]:

{apify-1.4.0b1 → apify-1.4.1}/src/apify/_memory_storage/resource_clients/key_value_store.py

@@ -100,8 +100,8 @@ class KeyValueStoreClient(BaseResourceClient):
         found = self._find_or_create_client_by_id_or_name(memory_storage_client=self._memory_storage_client, id=self._id, name=self._name)
 
         if found:
-            async with found._file_operation_lock:  # type: ignore
-                await found._update_timestamps(has_been_modified=False)  # type: ignore
+            async with found._file_operation_lock:
+                await found._update_timestamps(has_been_modified=False)
                 return found._to_resource_info()
 
         return None
@@ -127,7 +127,7 @@ class KeyValueStoreClient(BaseResourceClient):
         if name is None:
             return existing_store_by_id._to_resource_info()
 
-        async with existing_store_by_id._file_operation_lock:  # type: ignore
+        async with existing_store_by_id._file_operation_lock:
             # Check that name is not in use already
             existing_store_by_name = next(
                 (store for store in self._memory_storage_client._key_value_stores_handled if store._name and store._name.lower() == name.lower()),
@@ -146,7 +146,7 @@ class KeyValueStoreClient(BaseResourceClient):
             await force_rename(previous_dir, existing_store_by_id._resource_directory)
 
             # Update timestamps
-            await existing_store_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+            await existing_store_by_id._update_timestamps(has_been_modified=True)
 
         return existing_store_by_id._to_resource_info()
 
@@ -187,7 +187,7 @@ class KeyValueStoreClient(BaseResourceClient):
 
         items = []
 
-        for record in existing_store_by_id._records.values():  # type: ignore
+        for record in existing_store_by_id._records.values():
             size = len(record['value'])
             items.append(
                 {
@@ -222,8 +222,8 @@ class KeyValueStoreClient(BaseResourceClient):
         is_last_selected_item_absolutely_last = last_item_in_store == last_selected_item
         next_exclusive_start_key = None if is_last_selected_item_absolutely_last else last_selected_item['key']
 
-        async with existing_store_by_id._file_operation_lock:  # type: ignore
-            await existing_store_by_id._update_timestamps(has_been_modified=False)  # type: ignore
+        async with existing_store_by_id._file_operation_lock:
+            await existing_store_by_id._update_timestamps(has_been_modified=False)
 
         return {
             'count': len(items),
@@ -247,7 +247,7 @@ class KeyValueStoreClient(BaseResourceClient):
         if existing_store_by_id is None:
             raise_on_non_existing_storage(StorageTypes.KEY_VALUE_STORE, self._id)
 
-        stored_record = existing_store_by_id._records.get(key)  # type: ignore
+        stored_record = existing_store_by_id._records.get(key)
 
         if stored_record is None:
             return None
@@ -264,8 +264,8 @@ class KeyValueStoreClient(BaseResourceClient):
             except ValueError:
                 logger.exception('Error parsing key-value store record')
 
-        async with existing_store_by_id._file_operation_lock:  # type: ignore
-            await existing_store_by_id._update_timestamps(has_been_modified=False)  # type: ignore
+        async with existing_store_by_id._file_operation_lock:
+            await existing_store_by_id._update_timestamps(has_been_modified=False)
 
         return record
 
@@ -324,22 +324,22 @@ class KeyValueStoreClient(BaseResourceClient):
         if 'application/json' in content_type and not is_file_or_bytes(value) and not isinstance(value, str):
             value = json_dumps(value).encode('utf-8')
 
-        async with existing_store_by_id._file_operation_lock:  # type: ignore
-            await existing_store_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+        async with existing_store_by_id._file_operation_lock:
+            await existing_store_by_id._update_timestamps(has_been_modified=True)
             record: KeyValueStoreRecord = {
                 'key': key,
                 'value': value,
                 'contentType': content_type,
             }
 
-            old_record = existing_store_by_id._records.get(key)  # type: ignore
-            existing_store_by_id._records[key] = record  # type: ignore
+            old_record = existing_store_by_id._records.get(key)
+            existing_store_by_id._records[key] = record
 
             if self._memory_storage_client._persist_storage:
                 if old_record is not None and _filename_from_record(old_record) != _filename_from_record(record):
-                    await existing_store_by_id._delete_persisted_record(old_record)  # type: ignore
+                    await existing_store_by_id._delete_persisted_record(old_record)
 
-                await existing_store_by_id._persist_record(record)  # type: ignore
+                await existing_store_by_id._persist_record(record)
 
     async def _persist_record(self: KeyValueStoreClient, record: KeyValueStoreRecord) -> None:
         store_directory = self._resource_directory
@@ -385,14 +385,14 @@ class KeyValueStoreClient(BaseResourceClient):
         if existing_store_by_id is None:
             raise_on_non_existing_storage(StorageTypes.KEY_VALUE_STORE, self._id)
 
-        record = existing_store_by_id._records.get(key)  # type: ignore
+        record = existing_store_by_id._records.get(key)
 
         if record is not None:
-            async with existing_store_by_id._file_operation_lock:  # type: ignore
-                del existing_store_by_id._records[key]  # type: ignore
-                await existing_store_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+            async with existing_store_by_id._file_operation_lock:
+                del existing_store_by_id._records[key]
+                await existing_store_by_id._update_timestamps(has_been_modified=True)
                 if self._memory_storage_client._persist_storage:
-                    await existing_store_by_id._delete_persisted_record(record)  # type: ignore
+                    await existing_store_by_id._delete_persisted_record(record)
 
     async def _delete_persisted_record(self: KeyValueStoreClient, record: KeyValueStoreRecord) -> None:
         store_directory = self._resource_directory
@@ -437,7 +437,7 @@ class KeyValueStoreClient(BaseResourceClient):
         return memory_storage_client._key_value_stores_directory
 
     @classmethod
-    def _get_storage_client_cache(  # type: ignore
+    def _get_storage_client_cache(
         cls: type[KeyValueStoreClient],
         memory_storage_client: MemoryStorageClient,
     ) -> list[KeyValueStoreClient]:

{apify-1.4.0b1 → apify-1.4.1}/src/apify/_memory_storage/resource_clients/request_queue.py

@@ -67,8 +67,8 @@ class RequestQueueClient(BaseResourceClient):
         found = self._find_or_create_client_by_id_or_name(memory_storage_client=self._memory_storage_client, id=self._id, name=self._name)
 
         if found:
-            async with found._file_operation_lock:  # type: ignore
-                await found._update_timestamps(has_been_modified=False)  # type: ignore
+            async with found._file_operation_lock:
+                await found._update_timestamps(has_been_modified=False)
                 return found._to_resource_info()
 
         return None
@@ -94,7 +94,7 @@ class RequestQueueClient(BaseResourceClient):
         if name is None:
             return existing_queue_by_id._to_resource_info()
 
-        async with existing_queue_by_id._file_operation_lock:  # type: ignore
+        async with existing_queue_by_id._file_operation_lock:
             # Check that name is not in use already
             existing_queue_by_name = next(
                 (queue for queue in self._memory_storage_client._request_queues_handled if queue._name and queue._name.lower() == name.lower()), None
@@ -112,7 +112,7 @@ class RequestQueueClient(BaseResourceClient):
             await force_rename(previous_dir, existing_queue_by_id._resource_directory)
 
             # Update timestamps
-            await existing_queue_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+            await existing_queue_by_id._update_timestamps(has_been_modified=True)
 
             return existing_queue_by_id._to_resource_info()
 
@@ -146,18 +146,18 @@ class RequestQueueClient(BaseResourceClient):
         if existing_queue_by_id is None:
             raise_on_non_existing_storage(StorageTypes.REQUEST_QUEUE, self._id)
 
-        async with existing_queue_by_id._file_operation_lock:  # type: ignore
-            await existing_queue_by_id._update_timestamps(has_been_modified=False)  # type: ignore
+        async with existing_queue_by_id._file_operation_lock:
+            await existing_queue_by_id._update_timestamps(has_been_modified=False)
 
             items: list[dict] = []
 
             # Iterate all requests in the queue which have sorted key larger than infinity, which means `orderNo` is not `None`
             # This will iterate them in order of `orderNo`
-            for request_key in existing_queue_by_id._requests.irange_key(min_key=-float('inf'), inclusive=(False, True)):  # type: ignore
+            for request_key in existing_queue_by_id._requests.irange_key(min_key=-float('inf'), inclusive=(False, True)):
                 if len(items) == limit:
                     break
 
-                request = existing_queue_by_id._requests.get(request_key)  # type: ignore
+                request = existing_queue_by_id._requests.get(request_key)
 
                 # Check that the request still exists and was not handled,
                 # in case something deleted it or marked it as handled concurrenctly
@@ -167,7 +167,7 @@ class RequestQueueClient(BaseResourceClient):
             return {
                 'limit': limit,
                 'hadMultipleClients': False,
-                'queueModifiedAt': existing_queue_by_id._modified_at,  # type: ignore
+                'queueModifiedAt': existing_queue_by_id._modified_at,
                 'items': [self._json_to_request(item['json']) for item in items],
             }
 
@@ -190,12 +190,12 @@ class RequestQueueClient(BaseResourceClient):
 
         request_model = self._create_internal_request(request, forefront)
 
-        async with existing_queue_by_id._file_operation_lock:  # type: ignore
-            existing_request_with_id = existing_queue_by_id._requests.get(request_model['id'])  # type: ignore
+        async with existing_queue_by_id._file_operation_lock:
+            existing_request_with_id = existing_queue_by_id._requests.get(request_model['id'])
 
             # We already have the request present, so we return information about it
             if existing_request_with_id is not None:
-                await existing_queue_by_id._update_timestamps(has_been_modified=False)  # type: ignore
+                await existing_queue_by_id._update_timestamps(has_been_modified=False)
 
                 return {
                     'requestId': existing_request_with_id['id'],
@@ -203,12 +203,12 @@ class RequestQueueClient(BaseResourceClient):
                     'wasAlreadyPresent': True,
                 }
 
-            existing_queue_by_id._requests[request_model['id']] = request_model  # type: ignore
+            existing_queue_by_id._requests[request_model['id']] = request_model
             if request_model['orderNo'] is None:
-                existing_queue_by_id._handled_request_count += 1  # type: ignore
+                existing_queue_by_id._handled_request_count += 1
             else:
-                existing_queue_by_id._pending_request_count += 1  # type: ignore
-            await existing_queue_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+                existing_queue_by_id._pending_request_count += 1
+            await existing_queue_by_id._update_timestamps(has_been_modified=True)
             await update_request_queue_item(
                 request=request_model,
                 request_id=request_model['id'],
@@ -240,10 +240,10 @@ class RequestQueueClient(BaseResourceClient):
         if existing_queue_by_id is None:
             raise_on_non_existing_storage(StorageTypes.REQUEST_QUEUE, self._id)
 
-        async with existing_queue_by_id._file_operation_lock:  # type: ignore
-            await existing_queue_by_id._update_timestamps(has_been_modified=False)  # type: ignore
+        async with existing_queue_by_id._file_operation_lock:
+            await existing_queue_by_id._update_timestamps(has_been_modified=False)
 
-            request = existing_queue_by_id._requests.get(request_id)  # type: ignore
+            request = existing_queue_by_id._requests.get(request_id)
             return self._json_to_request(request['json'] if request is not None else None)
 
     async def update_request(self: RequestQueueClient, request: dict, *, forefront: bool | None = None) -> dict:
@@ -268,17 +268,17 @@ class RequestQueueClient(BaseResourceClient):
         # First we need to check the existing request to be
         # able to return information about its handled state.
 
-        existing_request = existing_queue_by_id._requests.get(request_model['id'])  # type: ignore
+        existing_request = existing_queue_by_id._requests.get(request_model['id'])
 
         # Undefined means that the request is not present in the queue.
         # We need to insert it, to behave the same as API.
         if existing_request is None:
             return await self.add_request(request, forefront=forefront)
 
-        async with existing_queue_by_id._file_operation_lock:  # type: ignore
+        async with existing_queue_by_id._file_operation_lock:
             # When updating the request, we need to make sure that
             # the handled counts are updated correctly in all cases.
-            existing_queue_by_id._requests[request_model['id']] = request_model  # type: ignore
+            existing_queue_by_id._requests[request_model['id']] = request_model
 
             pending_count_adjustment = 0
             is_request_handled_state_changing = not isinstance(existing_request['orderNo'], type(request_model['orderNo']))
@@ -288,9 +288,9 @@ class RequestQueueClient(BaseResourceClient):
             if is_request_handled_state_changing:
                 pending_count_adjustment = 1 if request_was_handled_before_update else -1
 
-            existing_queue_by_id._pending_request_count += pending_count_adjustment  # type: ignore
-            existing_queue_by_id._handled_request_count -= pending_count_adjustment  # type: ignore
-            await existing_queue_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+            existing_queue_by_id._pending_request_count += pending_count_adjustment
+            existing_queue_by_id._handled_request_count -= pending_count_adjustment
+            await existing_queue_by_id._update_timestamps(has_been_modified=True)
             await update_request_queue_item(
                 request=request_model,
                 request_id=request_model['id'],
@@ -317,16 +317,16 @@ class RequestQueueClient(BaseResourceClient):
         if existing_queue_by_id is None:
             raise_on_non_existing_storage(StorageTypes.REQUEST_QUEUE, self._id)
 
-        async with existing_queue_by_id._file_operation_lock:  # type: ignore
-            request = existing_queue_by_id._requests.get(request_id)  # type: ignore
+        async with existing_queue_by_id._file_operation_lock:
+            request = existing_queue_by_id._requests.get(request_id)
 
             if request:
-                del existing_queue_by_id._requests[request_id]  # type: ignore
+                del existing_queue_by_id._requests[request_id]
                 if request['orderNo'] is None:
-                    existing_queue_by_id._handled_request_count -= 1  # type: ignore
+                    existing_queue_by_id._handled_request_count -= 1
                 else:
-                    existing_queue_by_id._pending_request_count -= 1  # type: ignore
-                await existing_queue_by_id._update_timestamps(has_been_modified=True)  # type: ignore
+                    existing_queue_by_id._pending_request_count -= 1
+                await existing_queue_by_id._update_timestamps(has_been_modified=True)
                 await delete_request(entity_directory=existing_queue_by_id._resource_directory, request_id=request_id)
 
     def _to_resource_info(self: RequestQueueClient) -> dict:
@@ -403,7 +403,7 @@ class RequestQueueClient(BaseResourceClient):
         return memory_storage_client._request_queues_directory
 
     @classmethod
-    def _get_storage_client_cache(  # type: ignore
+    def _get_storage_client_cache(
         cls: type[RequestQueueClient],
         memory_storage_client: MemoryStorageClient,
     ) -> list[RequestQueueClient]:

{apify-1.4.0b1 → apify-1.4.1}/src/apify/scrapy/middlewares.py

@@ -4,10 +4,7 @@ import traceback
 from typing import TYPE_CHECKING, Any
 
 try:
-    from scrapy import Spider  # noqa: TCH002
     from scrapy.downloadermiddlewares.retry import RetryMiddleware
-    from scrapy.exceptions import IgnoreRequest
-    from scrapy.http import Request, Response  # noqa: TCH002
     from scrapy.utils.response import response_status_message
 except ImportError as exc:
     raise ImportError(
@@ -18,6 +15,9 @@ from ..actor import Actor
 from .utils import nested_event_loop, open_queue_with_custom_client, to_apify_request
 
 if TYPE_CHECKING:
+    from scrapy import Spider
+    from scrapy.http import Request, Response
+
     from ..storages import RequestQueue
 
 
@@ -33,11 +33,6 @@ class ApifyRetryMiddleware(RetryMiddleware):
             traceback.print_exc()
             raise
 
-    def __del__(self: ApifyRetryMiddleware) -> None:
-        """Before deleting the instance, close the nested event loop."""
-        nested_event_loop.stop()
-        nested_event_loop.close()
-
     def process_response(
         self: ApifyRetryMiddleware,
         request: Request,
@@ -54,9 +49,11 @@ class ApifyRetryMiddleware(RetryMiddleware):
         Returns:
             The response, or a new request if the request should be retried.
         """
+        if not isinstance(request.url, str):
+            raise TypeError(f'Expected request.url to be a string, got {type(request.url)} instead.')
+
         # Robots requests are bypassed directly, they don't go through a Scrapy Scheduler, and also through our
         # Request Queue. Check the scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware for details.
-        assert isinstance(request.url, str)  # noqa: S101
         if request.url.endswith('robots.txt'):
             return response
 
@@ -72,20 +69,30 @@ class ApifyRetryMiddleware(RetryMiddleware):
         exception: BaseException,
         spider: Spider,
     ) -> Request | Response | None:
-        """Handle the exception and decide whether the request should be retried."""
-        Actor.log.debug(f'ApifyRetryMiddleware.process_exception was called (scrapy_request={request})...')
+        """Handle the exception and decide whether the request should be retried.
+
+        Args:
+            request: The request that encountered an exception.
+            exception: The exception that occurred.
+            spider: The Spider that sent the request.
+
+        Returns:
+            None: The request will not be retried.
+        """
+        Actor.log.debug(f'ApifyRetryMiddleware.process_exception was called (request={request}, exception={exception})...')
         apify_request = to_apify_request(request, spider=spider)
 
-        if isinstance(exception, IgnoreRequest):
-            try:
-                nested_event_loop.run_until_complete(self._rq.mark_request_as_handled(apify_request))
-            except BaseException:
-                traceback.print_exc()
-                raise
-        else:
-            nested_event_loop.run_until_complete(self._rq.reclaim_request(apify_request))
+        # Unlike the default Scrapy RetryMiddleware, we do not attempt to retry requests on exception.
+        # It was causing issues with the Apify request queue, because the request was not marked as handled and was
+        # stucked in the request queue forever - Scrapy crawling never finished. The solution would be to completely
+        # rewrite the retry logic from default RetryMiddleware.
+        try:
+            nested_event_loop.run_until_complete(self._rq.mark_request_as_handled(apify_request))
+        except BaseException:
+            traceback.print_exc()
+            raise
 
-        return super().process_exception(request, exception, spider)
+        return None
 
     async def _handle_retry_logic(
         self: ApifyRetryMiddleware,

{apify-1.4.0b1 → apify-1.4.1}/src/apify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.4.0b1
+Version: 1.4.1
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License
@@ -24,10 +24,10 @@ Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.8
 Description-Content-Type: text/markdown
 License-File: LICENSE
-Requires-Dist: aiofiles>=22.1.0
-Requires-Dist: aioshutil>=1.0
 Requires-Dist: apify-client~=1.6.0
 Requires-Dist: apify-shared~=1.1.0
+Requires-Dist: aiofiles>=22.1.0
+Requires-Dist: aioshutil>=1.0
 Requires-Dist: colorama>=0.4.6
 Requires-Dist: cryptography>=39.0.0
 Requires-Dist: httpx>=0.24.1
@@ -51,10 +51,10 @@ Requires-Dist: respx~=0.20.1; extra == "dev"
 Requires-Dist: ruff~=0.1.6; extra == "dev"
 Requires-Dist: twine~=4.0.2; extra == "dev"
 Requires-Dist: types-aiofiles~=23.2.0.0; extra == "dev"
-Requires-Dist: types-colorama~=0.4.15.11; extra == "dev"
-Requires-Dist: types-psutil~=5.9.5.12; extra == "dev"
+Requires-Dist: types-colorama~=0.4.15.12; extra == "dev"
+Requires-Dist: types-psutil~=5.9.5.17; extra == "dev"
 Provides-Extra: scrapy
-Requires-Dist: scrapy~=2.11.0; extra == "scrapy"
+Requires-Dist: scrapy>=2.11.0; extra == "scrapy"
 
 # Apify SDK for Python
 

{apify-1.4.0b1 → apify-1.4.1}/src/apify.egg-info/requires.txt

@@ -1,7 +1,7 @@
-aiofiles>=22.1.0
-aioshutil>=1.0
 apify-client~=1.6.0
 apify-shared~=1.1.0
+aiofiles>=22.1.0
+aioshutil>=1.0
 colorama>=0.4.6
 cryptography>=39.0.0
 httpx>=0.24.1
@@ -26,8 +26,8 @@ respx~=0.20.1
 ruff~=0.1.6
 twine~=4.0.2
 types-aiofiles~=23.2.0.0
-types-colorama~=0.4.15.11
-types-psutil~=5.9.5.12
+types-colorama~=0.4.15.12
+types-psutil~=5.9.5.17
 
 [scrapy]
-scrapy~=2.11.0
+scrapy>=2.11.0