apify 2.2.2b1__py3-none-any.whl → 2.3.0__py3-none-any.whl

apify/scrapy/scheduler.py

@@ -1,41 +1,33 @@
 from __future__ import annotations
 
 import traceback
+from logging import getLogger
 from typing import TYPE_CHECKING
 
-from crawlee.storage_clients import MemoryStorageClient
+from scrapy import Spider
+from scrapy.core.scheduler import BaseScheduler
+from scrapy.utils.reactor import is_asyncio_reactor_installed
 
-from apify._configuration import Configuration
+from ._async_thread import AsyncThread
+from .requests import to_apify_request, to_scrapy_request
+from apify import Configuration
 from apify.apify_storage_client import ApifyStorageClient
+from apify.storages import RequestQueue
 
-try:
-    from scrapy import Spider
-    from scrapy.core.scheduler import BaseScheduler
-    from scrapy.utils.reactor import is_asyncio_reactor_installed
-
-    if TYPE_CHECKING:
-        from scrapy.http.request import Request
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
-from crawlee._utils.crypto import crypto_random_object_id
+if TYPE_CHECKING:
+    from scrapy.http.request import Request
+    from twisted.internet.defer import Deferred
 
-from apify import Actor
-from apify.scrapy.requests import to_apify_request, to_scrapy_request
-from apify.scrapy.utils import nested_event_loop
-from apify.storages import RequestQueue
+logger = getLogger(__name__)
 
 
 class ApifyScheduler(BaseScheduler):
-    """A Scrapy scheduler that uses the Apify Request Queue to manage requests.
+    """A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.
 
     This scheduler requires the asyncio Twisted reactor to be installed.
     """
 
     def __init__(self) -> None:
-        """Create a new instance."""
         if not is_asyncio_reactor_installed():
             raise ValueError(
                 f'{ApifyScheduler.__qualname__} requires the asyncio Twisted reactor. '
@@ -45,7 +37,10 @@ class ApifyScheduler(BaseScheduler):
         self._rq: RequestQueue | None = None
         self.spider: Spider | None = None
 
-    def open(self, spider: Spider) -> None:  # this has to be named "open"
+        # A thread with the asyncio event loop to run coroutines on.
+        self._async_thread = AsyncThread()
+
+    def open(self, spider: Spider) -> Deferred[None] | None:
         """Open the scheduler.
 
         Args:
@@ -53,23 +48,42 @@ class ApifyScheduler(BaseScheduler):
         """
         self.spider = spider
 
-        async def open_queue() -> RequestQueue:
+        async def open_rq() -> RequestQueue:
             config = Configuration.get_global_configuration()
-
-            # Use the ApifyStorageClient if the Actor is running on the Apify platform,
-            # otherwise use the MemoryStorageClient.
-            storage_client = (
-                ApifyStorageClient.from_config(config) if config.is_at_home else MemoryStorageClient.from_config(config)
-            )
-
-            return await RequestQueue.open(storage_client=storage_client)
+            if config.is_at_home:
+                storage_client = ApifyStorageClient.from_config(config)
+                return await RequestQueue.open(storage_client=storage_client)
+            return await RequestQueue.open()
 
         try:
-            self._rq = nested_event_loop.run_until_complete(open_queue())
-        except BaseException:
+            self._rq = self._async_thread.run_coro(open_rq())
+        except Exception:
             traceback.print_exc()
             raise
 
+        return None
+
+    def close(self, reason: str) -> None:
+        """Close the scheduler.
+
+        Shut down the event loop and its thread gracefully.
+
+        Args:
+            reason: The reason for closing the spider.
+        """
+        logger.debug(f'Closing {self.__class__.__name__} due to {reason}...')
+        try:
+            self._async_thread.close()
+
+        except KeyboardInterrupt:
+            logger.warning('Shutdown interrupted by KeyboardInterrupt!')
+
+        except Exception:
+            logger.exception('Exception occurred while shutting down.')
+
+        finally:
+            logger.debug(f'{self.__class__.__name__} closed successfully.')
+
     def has_pending_requests(self) -> bool:
         """Check if the scheduler has any pending requests.
 
@@ -80,8 +94,8 @@ class ApifyScheduler(BaseScheduler):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
-            is_finished = nested_event_loop.run_until_complete(self._rq.is_finished())
-        except BaseException:
+            is_finished = self._async_thread.run_coro(self._rq.is_finished())
+        except Exception:
             traceback.print_exc()
             raise
 
@@ -98,29 +112,27 @@ class ApifyScheduler(BaseScheduler):
         Returns:
             True if the request was successfully enqueued, False otherwise.
         """
-        call_id = crypto_random_object_id(8)
-        Actor.log.debug(f'[{call_id}]: ApifyScheduler.enqueue_request was called (scrapy_request={request})...')
+        logger.debug(f'ApifyScheduler.enqueue_request was called (scrapy_request={request})...')
 
         if not isinstance(self.spider, Spider):
             raise TypeError('self.spider must be an instance of the Spider class')
 
         apify_request = to_apify_request(request, spider=self.spider)
         if apify_request is None:
-            Actor.log.error(f'Request {request} was not enqueued because it could not be converted to Apify request.')
+            logger.error(f'Request {request} could not be converted to Apify request.')
             return False
 
-        Actor.log.debug(f'[{call_id}]: scrapy_request was transformed to apify_request (apify_request={apify_request})')
-
+        logger.debug(f'Converted to apify_request: {apify_request}')
         if not isinstance(self._rq, RequestQueue):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
-            result = nested_event_loop.run_until_complete(self._rq.add_request(apify_request))
-        except BaseException:
+            result = self._async_thread.run_coro(self._rq.add_request(apify_request))
+        except Exception:
             traceback.print_exc()
             raise
 
-        Actor.log.debug(f'[{call_id}]: rq.add_request.result={result}...')
+        logger.debug(f'rq.add_request result: {result}')
         return bool(result.was_already_present)
 
     def next_request(self) -> Request | None:
@@ -129,40 +141,31 @@ class ApifyScheduler(BaseScheduler):
         Returns:
             The next request, or None if there are no more requests.
         """
-        call_id = crypto_random_object_id(8)
-        Actor.log.debug(f'[{call_id}]: ApifyScheduler.next_request was called...')
-
+        logger.debug('next_request called...')
         if not isinstance(self._rq, RequestQueue):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
-        # Fetch the next request from the Request Queue
         try:
-            apify_request = nested_event_loop.run_until_complete(self._rq.fetch_next_request())
-        except BaseException:
+            apify_request = self._async_thread.run_coro(self._rq.fetch_next_request())
+        except Exception:
             traceback.print_exc()
             raise
 
-        Actor.log.debug(
-            f'[{call_id}]: a new apify_request from the scheduler was fetched (apify_request={apify_request})'
-        )
-
+        logger.debug(f'Fetched apify_request: {apify_request}')
         if apify_request is None:
             return None
 
         if not isinstance(self.spider, Spider):
             raise TypeError('self.spider must be an instance of the Spider class')
 
-        # Let the Request Queue know that the request is being handled. Every request should be marked as handled,
-        # retrying is handled by the Scrapy's RetryMiddleware.
+        # Let the request queue know that the request is being handled. Every request should
+        # be marked as handled, retrying is handled by the Scrapy's RetryMiddleware.
         try:
-            nested_event_loop.run_until_complete(self._rq.mark_request_as_handled(apify_request))
-        except BaseException:
+            self._async_thread.run_coro(self._rq.mark_request_as_handled(apify_request))
+        except Exception:
             traceback.print_exc()
             raise
 
         scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
-        Actor.log.debug(
-            f'[{call_id}]: apify_request was transformed to the scrapy_request which is gonna be returned '
-            f'(scrapy_request={scrapy_request})',
-        )
+        logger.debug(f'Converted to scrapy_request: {scrapy_request}')
         return scrapy_request

apify/scrapy/utils.py

@@ -1,29 +1,16 @@
 from __future__ import annotations
 
-import asyncio
 from base64 import b64encode
 from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
-from apify_shared.utils import ignore_docs
+from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_bytes
 
-try:
-    from scrapy.utils.project import get_project_settings
-    from scrapy.utils.python import to_bytes
+if TYPE_CHECKING:
+    from scrapy.settings import Settings
 
-    if TYPE_CHECKING:
-        from scrapy.settings import Settings
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. For example, if you use pip, run '
-        '"pip install apify[scrapy]".'
-    ) from exc
 
-
-nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
-
-
-@ignore_docs
 def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'latin-1') -> bytes:
     """Generate a basic authentication header for the given username and password."""
     string = f'{unquote(username)}:{unquote(password)}'
@@ -31,18 +18,6 @@ def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'la
     return b'Basic ' + b64encode(user_pass)
 
 
-@ignore_docs
-def get_running_event_loop_id() -> int:
-    """Get the ID of the currently running event loop.
-
-    It could be useful mainly for debugging purposes.
-
-    Returns:
-        The ID of the event loop.
-    """
-    return id(asyncio.get_running_loop())
-
-
 def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict | None = None) -> Settings:
     """Integrates Apify configuration into a Scrapy project settings.
 
@@ -65,10 +40,6 @@ def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict
     # ensuring it is executed as the final step in the pipeline sequence
     settings['ITEM_PIPELINES']['apify.scrapy.pipelines.ActorDatasetPushPipeline'] = 1000
 
-    # Disable the default AjaxCrawlMiddleware since it can be problematic with Apify. It can return a new request
-    # during process_response, but currently we have no way of detecting it and handling it properly.
-    settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware'] = None
-
     # Replace the default HttpProxyMiddleware with ApifyHttpProxyMiddleware
     settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware'] = None
     settings['DOWNLOADER_MIDDLEWARES']['apify.scrapy.middlewares.ApifyHttpProxyMiddleware'] = 750

{apify-2.2.2b1.dist-info → apify-2.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apify
-Version: 2.2.2b1
+Version: 2.3.0
 Summary: Apify SDK for Python
 License: Apache-2.0
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries
 Provides-Extra: scrapy
-Requires-Dist: apify-client (>=1.8.1)
+Requires-Dist: apify-client (>=1.9.2)
 Requires-Dist: apify-shared (>=1.2.1)
 Requires-Dist: crawlee (>=0.5.1,<0.6.0)
 Requires-Dist: cryptography (>=42.0.0)

{apify-2.2.2b1.dist-info → apify-2.3.0.dist-info}/RECORD

@@ -1,14 +1,15 @@
 apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
-apify/_actor.py,sha256=Pb7HPHIAodQOiGtyb-At45x8GfDItWCusRtQkoz1Pq4,46138
-apify/_configuration.py,sha256=T3Z_o_W98iSyTbrutfb578yW51aexZ_V0FcLwTxFLjI,10878
+apify/_actor.py,sha256=EB3gGjASV0PbPJ6BtgOq45HN23vM-9ceNCNRfeh2BkQ,48821
+apify/_charging.py,sha256=m7hJIQde4M7vS4g_4hsNRP5xHNXjYQ8MyqOEGeNb7VY,12267
+apify/_configuration.py,sha256=yidcWHsu-IJ2mmLmXStKq_HHcdfQxZq7koYjlZfRnQ8,11128
 apify/_consts.py,sha256=_Xq4hOfOA1iZ3n1P967YWdyncKivpbX6RTlp_qanUoE,330
 apify/_crypto.py,sha256=e0_aM3l9_5Osk-jszYOOjrAKK60OggSHbiw5c30QnsU,5638
-apify/_models.py,sha256=Btlz-23obKY5tJ75JnUwkVNC2lmU1IEBbdU3HvWaVhg,5748
+apify/_models.py,sha256=uWazgwDWH3OneftJ3ArhPz9N3oVuam5pdmlOQ1GE0NU,7905
 apify/_platform_event_manager.py,sha256=44xyV0Lpzf4h4VZ0rkyYg_nhbQkEONNor8_Z9gIKO40,7899
 apify/_proxy_configuration.py,sha256=c-O6_PZ9pUD-i4J0RFEKTtfyJPP2rTRJJA1TH8NVsV8,13189
-apify/_utils.py,sha256=CCLkpAsZKp00ykm88Z_Fbck5PNT0j6mJYOuD0RxzZUs,1620
+apify/_utils.py,sha256=92byxeXTpDFwhBq7ZS-obeXKtKWvVzCZMV0Drg3EjhQ,1634
 apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
-apify/apify_storage_client/_apify_storage_client.py,sha256=0rS75JoRHt7stRYS9-oqm3DmaSIZQN5C11N5MZQUvlA,2616
+apify/apify_storage_client/_apify_storage_client.py,sha256=jTX5vd-K9mnFTyZu2V2dUg7oyWogvmNIDUlEXnvIlOw,2766
 apify/apify_storage_client/_dataset_client.py,sha256=UUodnR_MQBg5RkURrfegkGJWR5OmdPPgPfGepvkdQoU,5580
 apify/apify_storage_client/_dataset_collection_client.py,sha256=qCcKZlA0bkO-sL7xED0Yose85NlrRa9AKr4oCSrYX6k,1489
 apify/apify_storage_client/_key_value_store_client.py,sha256=MSuoIeqEHLu92WfUU7kyB3Cc_gKUlm8TghnU3_xkPtE,3363
@@ -18,21 +19,24 @@ apify/apify_storage_client/_request_queue_collection_client.py,sha256=MdzgbQb2D8
 apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/log.py,sha256=j-E4t-WeA93bc1NCQRG8sTntehQCiiN8ia-MdQe3_Ts,1291
 apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/__init__.py,sha256=HE5wCN7-DZKPydLCOvjNyLuL3CvN2fUFweXfrDfe1Ss,348
+apify/scrapy/__init__.py,sha256=m2a0ts_JY9xJkBy4JU5mV8PJqjA3GGKLXBFu4nl-n-A,1048
+apify/scrapy/_actor_runner.py,sha256=rXWSnlQWGskDUH8PtLCv5SkOIx4AiVa4QbCYeCett5c,938
+apify/scrapy/_async_thread.py,sha256=AfeH9ZkSRZXxL11wzwrroDNsTzq4tAvURlinUZBtYMA,4753
+apify/scrapy/_logging_config.py,sha256=hFq90fNtZyjjJA7w2k-mtuEC8xCFiBMTalbwPDcaig4,2022
 apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
-apify/scrapy/middlewares/apify_proxy.py,sha256=H8a3vrA_7S_ucRkE3VDNMU8TY2CdzGTMXbhbJbfLv1c,5755
+apify/scrapy/middlewares/apify_proxy.py,sha256=CDAOXS3bcVDZHM3B0GvhXbxEikMIadLF_0P73WL_nI4,5550
 apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
-apify/scrapy/pipelines/actor_dataset_push.py,sha256=otggoULfUdCqOPJLb9wMROZ9WylnlL-209930tMS2Rg,971
+apify/scrapy/pipelines/actor_dataset_push.py,sha256=XUUyznQTD-E3wYUUFt2WAOnWhbnRrY0WuedlfYfYhDI,846
 apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/scrapy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/requests.py,sha256=yZ9hIsz2YyqOoOwzN9F1h76wG4qwawrI6h_6xq0I7Iw,7599
-apify/scrapy/scheduler.py,sha256=blO333BhFDMu3wAvSQONYdhmAmyiysqsv3YF5FKO_20,6281
-apify/scrapy/utils.py,sha256=758DcHCSAgCTProY0QX74uJ1XrzVsQwvCmFanj2f_3Q,2928
+apify/scrapy/requests.py,sha256=tOiFtG0kyzbBwtmaOisLAcpJENR1eDtpPR1nRH7JJGg,6551
+apify/scrapy/scheduler.py,sha256=-r1wZjMmeRDPxZKGHO-EYDYpGdDgSPAdNgMFViqUK8E,6019
+apify/scrapy/utils.py,sha256=5cka33PWc_at14yjhnLkCvY4h-ySUgVVhhDLxTy39ZI,1965
 apify/storages/__init__.py,sha256=FW-z6ubuPnHGM-Wp15T8mR5q6lnpDGrCW-IkgZd5L30,177
 apify/storages/_request_list.py,sha256=-lZJcE5nq69aJhGFJ7Sh2ctqgAWUDyOwYm5_0y1hdAE,5865
 apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-2.2.2b1.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-2.2.2b1.dist-info/METADATA,sha256=vHXzlhslB1UvfspGitKBpmPXCeqxeRcBgMJ523JtHD8,8696
-apify-2.2.2b1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-apify-2.2.2b1.dist-info/RECORD,,
+apify-2.3.0.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-2.3.0.dist-info/METADATA,sha256=CZJsIK4N-dhBV72efiPK19wTsFzPh9vd8i12FVebJkE,8694
+apify-2.3.0.dist-info/WHEEL,sha256=XbeZDeTWKc1w7CSIyre5aMDU_-PohRwTQceYnisIYYY,88
+apify-2.3.0.dist-info/RECORD,,

{apify-2.2.2b1.dist-info → apify-2.3.0.dist-info}/WHEEL

@@ -1,4 +1,4 @@
 Wheel-Version: 1.0
-Generator: poetry-core 2.0.1
+Generator: poetry-core 2.1.1
 Root-Is-Purelib: true
 Tag: py3-none-any