apify 1.5.1b3__tar.gz → 1.5.2b2__tar.gz

{apify-1.5.1b3 → apify-1.5.2b2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.5.1b3
+Version: 1.5.2b2
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License

{apify-1.5.1b3 → apify-1.5.2b2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "apify"
-version = "1.5.1b3"
+version = "1.5.2b2"
 description = "Apify SDK for Python"
 readme = "README.md"
 license = { text = "Apache Software License" }

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/_memory_storage/resource_clients/base_resource_collection_client.py

@@ -43,7 +43,7 @@ class BaseResourceCollectionClient(ABC, Generic[ResourceClientType]):
         raise NotImplementedError('You must override this method in the subclass!')
 
     @abstractmethod
-    async def list(self: BaseResourceCollectionClient) -> ListPage:  # noqa: A003
+    async def list(self: BaseResourceCollectionClient) -> ListPage:
         """List the available storages.
 
         Returns:

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/_memory_storage/resource_clients/dataset.py

@@ -314,7 +314,7 @@ class DatasetClient(BaseResourceClient):
             existing_dataset_by_id._dataset_entries[idx] = entry
             added_ids.append(idx)
 
-        data_entries = [(id, existing_dataset_by_id._dataset_entries[id]) for id in added_ids] # noqa: A001
+        data_entries = [(id, existing_dataset_by_id._dataset_entries[id]) for id in added_ids]  # noqa: A001
 
         async with existing_dataset_by_id._file_operation_lock:
             await existing_dataset_by_id._update_timestamps(has_been_modified=True)

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/_memory_storage/resource_clients/dataset_collection.py

@@ -21,7 +21,7 @@ class DatasetCollectionClient(BaseResourceCollectionClient):
     def _get_resource_client_class(self: DatasetCollectionClient) -> type[DatasetClient]:
         return DatasetClient
 
-    async def list(self: DatasetCollectionClient) -> ListPage:  # noqa: A003
+    async def list(self: DatasetCollectionClient) -> ListPage:
         """List the available datasets.
 
         Returns:

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/_memory_storage/resource_clients/key_value_store_collection.py

@@ -21,7 +21,7 @@ class KeyValueStoreCollectionClient(BaseResourceCollectionClient):
     def _get_resource_client_class(self: KeyValueStoreCollectionClient) -> type[KeyValueStoreClient]:
         return KeyValueStoreClient
 
-    async def list(self: KeyValueStoreCollectionClient) -> ListPage:  # noqa: A003
+    async def list(self: KeyValueStoreCollectionClient) -> ListPage:
         """List the available key-value stores.
 
         Returns:

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/_memory_storage/resource_clients/request_queue_collection.py

@@ -21,7 +21,7 @@ class RequestQueueCollectionClient(BaseResourceCollectionClient):
     def _get_resource_client_class(self: RequestQueueCollectionClient) -> type[RequestQueueClient]:
         return RequestQueueClient
 
-    async def list(self: RequestQueueCollectionClient) -> ListPage:  # noqa: A003
+    async def list(self: RequestQueueCollectionClient) -> ListPage:
         """List the available request queues.
 
         Returns:

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/actor.py

@@ -295,7 +295,7 @@ class Actor(metaclass=_ActorContextManager):
                 await self._send_system_info_interval_task
 
     @classmethod
-    async def exit(  # noqa: A003
+    async def exit(
         cls: type[Actor],
         *,
         exit_code: int = 0,

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/event_manager.py

@@ -219,7 +219,7 @@ class EventManager:
         try:
             async with websockets.client.connect(self._config.actor_events_ws_url) as websocket:
                 self._platform_events_websocket = websocket
-                self._connected_to_platform_websocket.set_result(True)  # noqa: FBT003
+                self._connected_to_platform_websocket.set_result(True)
                 async for message in websocket:
                     try:
                         parsed_message = json.loads(message)
@@ -234,4 +234,4 @@ class EventManager:
                         logger.exception('Cannot parse actor event', extra={'message': message})
         except Exception:
             logger.exception('Error in websocket connection')
-            self._connected_to_platform_websocket.set_result(False)  # noqa: FBT003
+            self._connected_to_platform_websocket.set_result(False)

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/log.py

@@ -82,7 +82,7 @@ class ActorLogFormatter(logging.Formatter):
         return extra_fields
 
     @ignore_docs
-    def format(self: ActorLogFormatter, record: logging.LogRecord) -> str:  # noqa: A003
+    def format(self: ActorLogFormatter, record: logging.LogRecord) -> str:
         """Format the log record nicely.
 
         This formats the log record so that it:

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/scrapy/__init__.py

@@ -1,3 +1,2 @@
-from .pipelines import ActorDatasetPushPipeline
 from .scheduler import ApifyScheduler
 from .utils import get_basic_auth_header, get_running_event_loop_id, open_queue_with_custom_client, to_apify_request, to_scrapy_request

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/scrapy/middlewares/apify_proxy.py

@@ -1,19 +1,21 @@
 from __future__ import annotations
 
-from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
-from scrapy.core.downloader.handlers.http11 import TunnelError
-from scrapy.exceptions import NotConfigured
+try:
+    from scrapy import Request, Spider  # noqa: TCH002
+    from scrapy.core.downloader.handlers.http11 import TunnelError
+    from scrapy.crawler import Crawler  # noqa: TCH002
+    from scrapy.exceptions import NotConfigured
+except ImportError as exc:
+    raise ImportError(
+        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
+    ) from exc
 
 from ...actor import Actor
 from ...proxy_configuration import ProxyConfiguration
 from ..utils import get_basic_auth_header
 
-if TYPE_CHECKING:
-    from scrapy import Request, Spider
-    from scrapy.crawler import Crawler
-
 
 class ApifyHttpProxyMiddleware:
     """Apify HTTP proxy middleware for Scrapy.

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/scrapy/middlewares/apify_retry.py

@@ -4,7 +4,9 @@ import traceback
 from typing import TYPE_CHECKING, Any
 
 try:
+    from scrapy import Spider  # noqa: TCH002
     from scrapy.downloadermiddlewares.retry import RetryMiddleware
+    from scrapy.http import Request, Response  # noqa: TCH002
     from scrapy.utils.response import response_status_message
 except ImportError as exc:
     raise ImportError(
@@ -15,9 +17,6 @@ from ...actor import Actor
 from ..utils import nested_event_loop, open_queue_with_custom_client, to_apify_request
 
 if TYPE_CHECKING:
-    from scrapy import Spider
-    from scrapy.http import Request, Response
-
     from ...storages import RequestQueue
 
 

apify-1.5.2b2/src/apify/scrapy/pipelines/__init__.py

@@ -0,0 +1 @@
+from .actor_dataset_push import ActorDatasetPushPipeline

apify-1.5.1b3/src/apify/scrapy/pipelines.py → apify-1.5.2b2/src/apify/scrapy/pipelines/actor_dataset_push.py

@@ -9,7 +9,7 @@ except ImportError as exc:
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
     ) from exc
 
-from ..actor import Actor
+from ...actor import Actor
 
 
 class ActorDatasetPushPipeline:

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/scrapy/scheduler.py

@@ -35,7 +35,7 @@ class ApifyScheduler(BaseScheduler):
         self._rq: RequestQueue | None = None
         self.spider: Spider | None = None
 
-    def open(self: ApifyScheduler, spider: Spider) -> None:  # noqa: A003 # this has to be named "open"
+    def open(self: ApifyScheduler, spider: Spider) -> None:  # this has to be named "open"
         """Open the scheduler.
 
         Args:
@@ -55,7 +55,9 @@ class ApifyScheduler(BaseScheduler):
         Returns:
             True if the scheduler has any pending requests, False otherwise.
         """
-        assert isinstance(self._rq, RequestQueue)  # noqa: S101
+        if not isinstance(self._rq, RequestQueue):
+            raise TypeError('self._rq must be an instance of the RequestQueue class')
+
         try:
             is_finished = nested_event_loop.run_until_complete(self._rq.is_finished())
         except BaseException:
@@ -76,10 +78,14 @@ class ApifyScheduler(BaseScheduler):
         call_id = crypto_random_object_id(8)
         Actor.log.debug(f'[{call_id}]: ApifyScheduler.enqueue_request was called (scrapy_request={request})...')
 
-        assert isinstance(self.spider, Spider)  # noqa: S101
+        if not isinstance(self.spider, Spider):
+            raise TypeError('self.spider must be an instance of the Spider class')
+
         apify_request = to_apify_request(request, spider=self.spider)
         Actor.log.debug(f'[{call_id}]: scrapy_request was transformed to apify_request (apify_request={apify_request})')
-        assert isinstance(self._rq, RequestQueue)  # noqa: S101
+
+        if not isinstance(self._rq, RequestQueue):
+            raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
             result = nested_event_loop.run_until_complete(self._rq.add_request(apify_request))
@@ -98,7 +104,9 @@ class ApifyScheduler(BaseScheduler):
         """
         call_id = crypto_random_object_id(8)
         Actor.log.debug(f'[{call_id}]: ApifyScheduler.next_request was called...')
-        assert isinstance(self._rq, RequestQueue)  # noqa: S101
+
+        if not isinstance(self._rq, RequestQueue):
+            raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
             apify_request = nested_event_loop.run_until_complete(self._rq.fetch_next_request())
@@ -111,7 +119,9 @@ class ApifyScheduler(BaseScheduler):
         if apify_request is None:
             return None
 
-        assert isinstance(self.spider, Spider)  # noqa: S101
+        if not isinstance(self.spider, Spider):
+            raise TypeError('self.spider must be an instance of the Spider class')
+
         scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
         Actor.log.debug(
             f'[{call_id}]: apify_request was transformed to the scrapy_request which is gonna be returned (scrapy_request={scrapy_request})',

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/scrapy/utils.py

@@ -6,10 +6,9 @@ import pickle
 from base64 import b64encode
 from urllib.parse import unquote
 
-from scrapy.utils.python import to_bytes
-
 try:
     from scrapy import Request, Spider
+    from scrapy.utils.python import to_bytes
     from scrapy.utils.request import request_from_dict
 except ImportError as exc:
     raise ImportError(
@@ -51,7 +50,8 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> dict:
     Returns:
         The converted Apify request.
     """
-    assert isinstance(scrapy_request, Request)  # noqa: S101
+    if not isinstance(scrapy_request, Request):
+        raise TypeError('scrapy_request must be an instance of the scrapy.Request class')
 
     call_id = crypto_random_object_id(8)
     Actor.log.debug(f'[{call_id}]: to_apify_request was called (scrapy_request={scrapy_request})...')
@@ -91,11 +91,14 @@ def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
     Returns:
         The converted Scrapy request.
     """
-    assert isinstance(apify_request, dict)  # noqa: S101
-    assert 'url' in apify_request  # noqa: S101
-    assert 'method' in apify_request  # noqa: S101
-    assert 'id' in apify_request  # noqa: S101
-    assert 'uniqueKey' in apify_request  # noqa: S101
+    if not isinstance(apify_request, dict):
+        raise TypeError('apify_request must be a dictionary')
+
+    required_keys = ['url', 'method', 'id', 'uniqueKey']
+    missing_keys = [key for key in required_keys if key not in apify_request]
+
+    if missing_keys:
+        raise ValueError(f"apify_request must contain {', '.join(map(repr, missing_keys))} key(s)")
 
     call_id = crypto_random_object_id(8)
     Actor.log.debug(f'[{call_id}]: to_scrapy_request was called (apify_request={apify_request})...')
@@ -106,14 +109,19 @@ def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
         #   - This process involves decoding the base64-encoded request data and reconstructing
         #     the Scrapy Request object from its dictionary representation.
         Actor.log.debug(f'[{call_id}]: Restoring the Scrapy Request from the apify_request...')
+
         scrapy_request_dict_encoded = apify_request['userData']['scrapy_request']
-        assert isinstance(scrapy_request_dict_encoded, str)  # noqa: S101
+        if not isinstance(scrapy_request_dict_encoded, str):
+            raise TypeError('scrapy_request_dict_encoded must be a string')
 
         scrapy_request_dict = pickle.loads(codecs.decode(scrapy_request_dict_encoded.encode(), 'base64'))
-        assert isinstance(scrapy_request_dict, dict)  # noqa: S101
+        if not isinstance(scrapy_request_dict, dict):
+            raise TypeError('scrapy_request_dict must be a dictionary')
 
         scrapy_request = request_from_dict(scrapy_request_dict, spider=spider)
-        assert isinstance(scrapy_request, Request)  # noqa: S101
+        if not isinstance(scrapy_request, Request):
+            raise TypeError('scrapy_request must be an instance of the Request class')
+
         Actor.log.debug(f'[{call_id}]: Scrapy Request successfully reconstructed (scrapy_request={scrapy_request})...')
 
         # Update the meta field with the meta field from the apify_request

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/storages/base_storage.py

@@ -91,7 +91,7 @@ class BaseStorage(ABC, Generic[BaseResourceClientType, BaseResourceCollectionCli
 
     @classmethod
     @abstractmethod
-    async def open(  # noqa: A003
+    async def open(
         cls: type[BaseStorage],
         *,
         id: str | None = None,  # noqa: A002

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/storages/dataset.py

@@ -463,7 +463,7 @@ class Dataset(BaseStorage):
         self._remove_from_cache()
 
     @classmethod
-    async def open(  # noqa: A003
+    async def open(
         cls: type[Dataset],
         *,
         id: str | None = None,  # noqa: A002

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/storages/key_value_store.py

@@ -226,7 +226,7 @@ class KeyValueStore(BaseStorage):
         self._remove_from_cache()
 
     @classmethod
-    async def open(  # noqa: A003
+    async def open(
         cls: type[KeyValueStore],
         *,
         id: str | None = None,  # noqa: A002

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify/storages/request_queue.py

@@ -536,7 +536,7 @@ class RequestQueue(BaseStorage):
         return await self._request_queue_client.get()
 
     @classmethod
-    async def open(  # noqa: A003
+    async def open(
         cls: type[RequestQueue],
         *,
         id: str | None = None,  # noqa: A002

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: apify
-Version: 1.5.1b3
+Version: 1.5.2b2
 Summary: Apify SDK for Python
 Author-email: "Apify Technologies s.r.o." <support@apify.com>
 License: Apache Software License

{apify-1.5.1b3 → apify-1.5.2b2}/src/apify.egg-info/SOURCES.txt

@@ -29,12 +29,13 @@ src/apify/_memory_storage/resource_clients/key_value_store_collection.py
 src/apify/_memory_storage/resource_clients/request_queue.py
 src/apify/_memory_storage/resource_clients/request_queue_collection.py
 src/apify/scrapy/__init__.py
-src/apify/scrapy/pipelines.py
 src/apify/scrapy/scheduler.py
 src/apify/scrapy/utils.py
 src/apify/scrapy/middlewares/__init__.py
 src/apify/scrapy/middlewares/apify_proxy.py
 src/apify/scrapy/middlewares/apify_retry.py
+src/apify/scrapy/pipelines/__init__.py
+src/apify/scrapy/pipelines/actor_dataset_push.py
 src/apify/storages/__init__.py
 src/apify/storages/base_storage.py
 src/apify/storages/dataset.py