apify 2.2.2b2__py3-none-any.whl → 2.3.0b1__py3-none-any.whl

apify/_actor.py

@@ -270,8 +270,8 @@ class _ActorType:
             self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in IPython')
         elif os.getenv('PYTEST_CURRENT_TEST', default=False):  # noqa: PLW1508
             self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in an unit test')
-        elif hasattr(asyncio, '_nest_patched'):
-            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in a nested event loop')
+        elif os.getenv('SCRAPY_SETTINGS_MODULE'):
+            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running with Scrapy')
         else:
             sys.exit(exit_code)
 

apify/apify_storage_client/_apify_storage_client.py

@@ -66,3 +66,7 @@ class ApifyStorageClient(BaseStorageClient):
     @override
     async def purge_on_start(self) -> None:
         pass
+
+    @override
+    def get_rate_limit_errors(self) -> dict[int, int]:  # type: ignore[misc]
+        return self._apify_client.stats.rate_limit_errors

apify/scrapy/__init__.py

@@ -1,11 +1,32 @@
-from apify.scrapy.requests import to_apify_request, to_scrapy_request
-from apify.scrapy.scheduler import ApifyScheduler
-from apify.scrapy.utils import get_basic_auth_header, get_running_event_loop_id
+from crawlee._utils.try_import import install_import_hook as _install_import_hook
+from crawlee._utils.try_import import try_import as _try_import
+
+_install_import_hook(__name__)
+
+# The following imports use try_import to handle optional dependencies, as they may not always be available.
+
+with _try_import(__name__, 'run_scrapy_actor'):
+    from ._actor_runner import run_scrapy_actor
+
+with _try_import(__name__, 'initialize_logging'):
+    from ._logging_config import initialize_logging
+
+with _try_import(__name__, 'to_apify_request', 'to_scrapy_request'):
+    from .requests import to_apify_request, to_scrapy_request
+
+with _try_import(__name__, 'ApifyScheduler'):
+    from .scheduler import ApifyScheduler
+
+with _try_import(__name__, 'apply_apify_settings', 'get_basic_auth_header'):
+    from .utils import apply_apify_settings, get_basic_auth_header
+
 
 __all__ = [
     'ApifyScheduler',
+    'apply_apify_settings',
     'get_basic_auth_header',
-    'get_running_event_loop_id',
+    'initialize_logging',
+    'run_scrapy_actor',
     'to_apify_request',
     'to_scrapy_request',
 ]

apify/scrapy/_actor_runner.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING
+
+from twisted.internet.defer import Deferred, ensureDeferred
+from twisted.internet.task import react
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
+
+async def _run_coro_as_deferred(coro: Coroutine) -> None:
+    """Wrap the given asyncio coroutine in a Task and await its result as a Twisted Deferred."""
+    task = asyncio.ensure_future(coro)
+    await Deferred.fromFuture(task)
+
+
+def run_scrapy_actor(coro: Coroutine) -> None:
+    """Start Twisted's reactor and execute the provided Actor coroutine.
+
+    This function initiates the Twisted reactor and runs the given asyncio coroutine (typically the
+    Actor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,
+    enabling the Apify and Scrapy integration to work together.
+    """
+    react(lambda _: ensureDeferred(_run_coro_as_deferred(coro)))

apify/scrapy/_async_thread.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+from concurrent import futures
+from datetime import timedelta
+from logging import getLogger
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
+logger = getLogger(__name__)
+
+
+class AsyncThread:
+    """Class for running an asyncio event loop in a separate thread.
+
+    This allows running asynchronous coroutines from synchronous code by executingthem on an event loop
+    that runs in its own dedicated thread.
+    """
+
+    def __init__(self) -> None:
+        self._eventloop = asyncio.new_event_loop()
+
+        # Start the event loop in a dedicated daemon thread.
+        self._thread = threading.Thread(
+            target=self._start_event_loop,
+            daemon=True,
+        )
+        self._thread.start()
+
+    def run_coro(
+        self,
+        coro: Coroutine,
+        timeout: timedelta = timedelta(seconds=60),
+    ) -> Any:
+        """Run a coroutine on an event loop running in a separate thread.
+
+        This method schedules the coroutine to run on the event loop and blocks until the coroutine completes
+        or the specified timeout is reached.
+
+        Args:
+            coro: The coroutine to run.
+            timeout: The maximum number of seconds to wait for the coroutine to finish.
+
+        Returns:
+            The result returned by the coroutine.
+
+        Raises:
+            RuntimeError: If the event loop is not running.
+            TimeoutError: If the coroutine does not complete within the timeout.
+            Exception: Any exception raised during coroutine execution.
+        """
+        if not self._eventloop.is_running():
+            raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is not running.')
+
+        # Submit the coroutine to the event loop running in the other thread.
+        future = asyncio.run_coroutine_threadsafe(coro, self._eventloop)
+        try:
+            # Wait for the coroutine's result until the specified timeout.
+            return future.result(timeout=timeout.total_seconds())
+        except futures.TimeoutError as exc:
+            logger.exception('Coroutine execution timed out.', exc_info=exc)
+            raise
+        except Exception as exc:
+            logger.exception('Coroutine execution raised an exception.', exc_info=exc)
+            raise
+
+    def close(self, timeout: timedelta = timedelta(seconds=60)) -> None:
+        """Close the event loop and its thread gracefully.
+
+        This method cancels all pending tasks, stops the event loop, and waits for the thread to exit.
+        If the thread does not exit within the given timeout, a forced shutdown is attempted.
+
+        Args:
+            timeout: The maximum number of seconds to wait for the event loop thread to exit.
+        """
+        if self._eventloop.is_running():
+            # Cancel all pending tasks in the event loop.
+            self.run_coro(self._shutdown_tasks())
+
+        # Schedule the event loop to stop.
+        self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+
+        # Wait for the event loop thread to finish execution.
+        self._thread.join(timeout=timeout.total_seconds())
+
+        # If the thread is still running after the timeout, force a shutdown.
+        if self._thread.is_alive():
+            logger.warning('Event loop thread did not exit cleanly! Forcing shutdown...')
+            self._force_exit_event_loop()
+
+    def _start_event_loop(self) -> None:
+        """Set up and run the asyncio event loop in the dedicated thread."""
+        asyncio.set_event_loop(self._eventloop)
+        try:
+            self._eventloop.run_forever()
+        finally:
+            self._eventloop.close()
+            logger.debug('Asyncio event loop has been closed.')
+
+    async def _shutdown_tasks(self) -> None:
+        """Cancel all pending tasks in the event loop."""
+        # Retrieve all tasks for the event loop, excluding the current task.
+        tasks = [task for task in asyncio.all_tasks(self._eventloop) if task is not asyncio.current_task()]
+
+        # Cancel each pending task.
+        for task in tasks:
+            task.cancel()
+
+        # Wait until all tasks have been cancelled or finished.
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    def _force_exit_event_loop(self) -> None:
+        """Forcefully shut down the event loop and its thread."""
+        try:
+            logger.info('Forced shutdown of the event loop and its thread...')
+            self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+            self._thread.join(timeout=5)
+        except Exception as exc:
+            logger.exception('Exception occurred during forced event loop shutdown.', exc_info=exc)

apify/scrapy/_logging_config.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from scrapy.utils import log as scrapy_logging
+from scrapy.utils.project import get_project_settings
+
+from apify.log import ActorLogFormatter
+
+# Define logger names.
+_PRIMARY_LOGGERS = ['apify', 'apify_client', 'scrapy']
+_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'httpx', 'protego', 'twisted']
+_ALL_LOGGERS = _PRIMARY_LOGGERS + _SUPPLEMENTAL_LOGGERS
+
+
+def _configure_logger(name: str | None, logging_level: str, handler: logging.Handler) -> None:
+    """Clear and reconfigure the logger."""
+    logger = logging.getLogger(name)
+    logger.handlers.clear()
+    logger.setLevel(logging_level)
+
+    if name is None:  # Root logger.
+        logger.addHandler(handler)
+        logger.propagate = False
+    else:
+        logger.propagate = True
+
+
+def initialize_logging() -> None:
+    """Configure logging for Apify Actors and adjust Scrapy's logging settings."""
+    # Retrieve Scrapy project settings and determine the logging level.
+    settings = get_project_settings()
+    logging_level = settings.get('LOG_LEVEL', 'INFO')  # Default to INFO.
+
+    # Create a custom handler with the Apify log formatter.
+    handler = logging.StreamHandler()
+    handler.setFormatter(ActorLogFormatter(include_logger_name=True))
+
+    # Configure the root logger and all other defined loggers.
+    for logger_name in [None, *_ALL_LOGGERS]:
+        _configure_logger(logger_name, logging_level, handler)
+
+    # Set the 'httpx' logger to a less verbose level.
+    logging.getLogger('httpx').setLevel('WARNING')
+
+    # Monkey-patch Scrapy's logging configuration to re-apply our settings.
+    original_configure_logging = scrapy_logging.configure_logging
+
+    def new_configure_logging(*args: Any, **kwargs: Any) -> None:
+        original_configure_logging(*args, **kwargs)
+        for logger_name in [None, *_ALL_LOGGERS]:
+            _configure_logger(logger_name, logging_level, handler)
+
+    scrapy_logging.configure_logging = new_configure_logging

apify/scrapy/middlewares/apify_proxy.py

@@ -3,19 +3,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
-try:
-    if TYPE_CHECKING:
-        from scrapy import Request, Spider
-        from scrapy.crawler import Crawler
-    from scrapy.core.downloader.handlers.http11 import TunnelError
-    from scrapy.exceptions import NotConfigured
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
+from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
 
 from apify import Actor, ProxyConfiguration
-from apify.scrapy.utils import get_basic_auth_header
+from apify.scrapy import get_basic_auth_header
+
+if TYPE_CHECKING:
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
 
 
 class ApifyHttpProxyMiddleware:
@@ -51,7 +47,7 @@ class ApifyHttpProxyMiddleware:
         proxy_settings: dict | None = crawler.settings.get('APIFY_PROXY_SETTINGS')
 
         if proxy_settings is None:
-            Actor.log.warning(
+            Actor.log.info(
                 'ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing '
                 ' in the Actor input.'
             )
@@ -60,7 +56,7 @@ class ApifyHttpProxyMiddleware:
         use_apify_proxy = proxy_settings.get('useApifyProxy', False)
 
         if use_apify_proxy is not True:
-            Actor.log.warning(
+            Actor.log.info(
                 'ApifyHttpProxyMiddleware is not going to be used. Actor input field '
                 '"proxyConfiguration.useApifyProxy" is set to False.'
             )

apify/scrapy/pipelines/actor_dataset_push.py

@@ -1,19 +1,17 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import TYPE_CHECKING
 
 from itemadapter.adapter import ItemAdapter
 
-try:
-    if TYPE_CHECKING:
-        from scrapy import Item, Spider
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
 from apify import Actor
 
+if TYPE_CHECKING:
+    from scrapy import Item, Spider
+
+logger = getLogger(__name__)
+
 
 class ActorDatasetPushPipeline:
     """A Scrapy pipeline for pushing items to an Actor's default dataset.
@@ -28,6 +26,6 @@ class ActorDatasetPushPipeline:
     ) -> Item:
         """Pushes the provided Scrapy item to the Actor's default dataset."""
         item_dict = ItemAdapter(item).asdict()
-        Actor.log.debug(f'Pushing item={item_dict} produced by spider={spider} to the dataset.')
+        logger.debug(f'Pushing item={item_dict} produced by spider={spider} to the dataset.')
         await Actor.push_data(item_dict)
         return item

apify/scrapy/requests.py

@@ -2,37 +2,21 @@ from __future__ import annotations
 
 import codecs
 import pickle
+from logging import getLogger
 from typing import Any, cast
 
-from apify_shared.utils import ignore_docs
+from scrapy import Request as ScrapyRequest
+from scrapy import Spider
+from scrapy.http.headers import Headers
+from scrapy.utils.request import request_from_dict
 
-try:
-    from scrapy import Request, Spider
-    from scrapy.http.headers import Headers
-    from scrapy.utils.request import request_from_dict
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
-from crawlee import Request as CrawleeRequest
+from crawlee import Request as ApifyRequest
 from crawlee._types import HttpHeaders
-from crawlee._utils.crypto import crypto_random_object_id
-from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
-
-from apify import Actor
-
 
-def _is_request_produced_by_middleware(scrapy_request: Request) -> bool:
-    """Returns True if the Scrapy request was produced by a downloader middleware, otherwise False.
-
-    Works for RetryMiddleware and RedirectMiddleware.
-    """
-    return bool(scrapy_request.meta.get('redirect_times')) or bool(scrapy_request.meta.get('retry_times'))
+logger = getLogger(__name__)
 
 
-@ignore_docs
-def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest | None:
+def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequest | None:
     """Convert a Scrapy request to an Apify request.
 
     Args:
@@ -42,54 +26,45 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
     Returns:
         The converted Apify request if the conversion was successful, otherwise None.
     """
-    if not isinstance(scrapy_request, Request):
-        Actor.log.warning(  # type: ignore[unreachable]
-            'Failed to convert to Apify request: Scrapy request must be a Request instance.'
-        )
+    if not isinstance(scrapy_request, ScrapyRequest):
+        logger.warning('Failed to convert to Apify request: Scrapy request must be a ScrapyRequest instance.')  # type: ignore[unreachable]
         return None
 
-    call_id = crypto_random_object_id(8)
-    Actor.log.debug(f'[{call_id}]: to_apify_request was called (scrapy_request={scrapy_request})...')
+    logger.debug(f'to_apify_request was called (scrapy_request={scrapy_request})...')
+
+    # Configuration to behave as similarly as possible to Scrapy's default RFPDupeFilter.
+    request_kwargs: dict[str, Any] = {
+        'url': scrapy_request.url,
+        'method': scrapy_request.method,
+        'payload': scrapy_request.body,
+        'use_extended_unique_key': True,
+        'keep_url_fragment': False,
+    }
 
     try:
-        if _is_request_produced_by_middleware(scrapy_request):
-            unique_key = compute_unique_key(
-                url=scrapy_request.url,
-                method=scrapy_request.method,  # type: ignore[arg-type]  # str vs literal
-                payload=scrapy_request.body,
-                use_extended_unique_key=True,
-            )
-        elif scrapy_request.dont_filter:
-            unique_key = crypto_random_object_id(8)
-        elif scrapy_request.meta.get('apify_request_unique_key'):
-            unique_key = scrapy_request.meta['apify_request_unique_key']
+        if scrapy_request.dont_filter:
+            request_kwargs['always_enqueue'] = True
         else:
-            unique_key = crypto_random_object_id(8)
+            if scrapy_request.meta.get('apify_request_unique_key'):
+                request_kwargs['unique_key'] = scrapy_request.meta['apify_request_unique_key']
 
-        if scrapy_request.meta.get('apify_request_id'):
-            request_id = scrapy_request.meta['apify_request_id']
-        else:
-            request_id = unique_key_to_request_id(unique_key)
-
-        apify_request = CrawleeRequest(
-            url=scrapy_request.url,
-            method=scrapy_request.method,
-            payload=scrapy_request.body,
-            user_data=scrapy_request.meta.get('userData', {}),
-            unique_key=unique_key,
-            id=request_id,
-        )
+            if scrapy_request.meta.get('apify_request_id'):
+                request_kwargs['id'] = scrapy_request.meta['apify_request_id']
+
+        request_kwargs['user_data'] = scrapy_request.meta.get('userData', {})
 
         # Convert Scrapy's headers to a HttpHeaders and store them in the apify_request
         if isinstance(scrapy_request.headers, Headers):
-            apify_request.headers = HttpHeaders(dict(scrapy_request.headers.to_unicode_dict()))
+            request_kwargs['headers'] = HttpHeaders(dict(scrapy_request.headers.to_unicode_dict()))
         else:
-            Actor.log.warning(  # type: ignore[unreachable]
+            logger.warning(  # type: ignore[unreachable]
                 f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
             )
 
-        # Serialize the Scrapy Request and store it in the apify_request.
-        #   - This process involves converting the Scrapy Request object into a dictionary, encoding it to base64,
+        apify_request = ApifyRequest.from_url(**request_kwargs)
+
+        # Serialize the Scrapy ScrapyRequest and store it in the apify_request.
+        #   - This process involves converting the Scrapy ScrapyRequest object into a dictionary, encoding it to base64,
         #     and storing it as 'scrapy_request' within the 'userData' dictionary of the apify_request.
         #   - The serialization process can be referenced at: https://stackoverflow.com/questions/30469575/.
         scrapy_request_dict = scrapy_request.to_dict(spider=spider)
@@ -97,15 +72,14 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
         apify_request.user_data['scrapy_request'] = scrapy_request_dict_encoded
 
     except Exception as exc:
-        Actor.log.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
+        logger.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
         return None
 
-    Actor.log.debug(f'[{call_id}]: scrapy_request was converted to the apify_request={apify_request}')
+    logger.debug(f'scrapy_request was converted to the apify_request={apify_request}')
     return apify_request
 
 
-@ignore_docs
-def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
+def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequest:
     """Convert an Apify request to a Scrapy request.
 
     Args:
@@ -113,24 +87,23 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
         spider: The Scrapy spider that the request is associated with.
 
     Raises:
-        TypeError: If the apify_request is not a crawlee request.
-        ValueError: If the apify_request does not contain the required keys.
+        TypeError: If the Apify request is not an instance of the `ApifyRequest` class.
+        ValueError: If the Apify request does not contain the required keys.
 
     Returns:
         The converted Scrapy request.
     """
-    if not isinstance(cast(Any, apify_request), CrawleeRequest):
-        raise TypeError('apify_request must be a crawlee.Request instance')
+    if not isinstance(cast(Any, apify_request), ApifyRequest):
+        raise TypeError('apify_request must be a crawlee.ScrapyRequest instance')
 
-    call_id = crypto_random_object_id(8)
-    Actor.log.debug(f'[{call_id}]: to_scrapy_request was called (apify_request={apify_request})...')
+    logger.debug(f'to_scrapy_request was called (apify_request={apify_request})...')
 
     # If the apify_request comes from the Scrapy
     if 'scrapy_request' in apify_request.user_data:
-        # Deserialize the Scrapy Request from the apify_request.
+        # Deserialize the Scrapy ScrapyRequest from the apify_request.
         #   - This process involves decoding the base64-encoded request data and reconstructing
-        #     the Scrapy Request object from its dictionary representation.
-        Actor.log.debug(f'[{call_id}]: Restoring the Scrapy Request from the apify_request...')
+        #     the Scrapy ScrapyRequest object from its dictionary representation.
+        logger.debug('Restoring the Scrapy ScrapyRequest from the apify_request...')
 
         scrapy_request_dict_encoded = apify_request.user_data['scrapy_request']
         if not isinstance(scrapy_request_dict_encoded, str):
@@ -141,10 +114,10 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
             raise TypeError('scrapy_request_dict must be a dictionary')
 
         scrapy_request = request_from_dict(scrapy_request_dict, spider=spider)
-        if not isinstance(scrapy_request, Request):
-            raise TypeError('scrapy_request must be an instance of the Request class')
+        if not isinstance(scrapy_request, ScrapyRequest):
+            raise TypeError('scrapy_request must be an instance of the ScrapyRequest class')
 
-        Actor.log.debug(f'[{call_id}]: Scrapy Request successfully reconstructed (scrapy_request={scrapy_request})...')
+        logger.debug(f'Scrapy ScrapyRequest successfully reconstructed (scrapy_request={scrapy_request})...')
 
         # Update the meta field with the meta field from the apify_request
         meta = scrapy_request.meta or {}
@@ -152,11 +125,11 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
         # scrapy_request.meta is a property, so we have to set it like this
         scrapy_request._meta = meta  # noqa: SLF001
 
-    # If the apify_request comes directly from the Request Queue, typically start URLs
+    # If the apify_request comes directly from the Scrapy, typically start URLs.
     else:
-        Actor.log.debug(f'[{call_id}]: gonna create a new Scrapy Request (cannot be restored)')
+        logger.debug('Gonna create a new Scrapy ScrapyRequest (cannot be restored)')
 
-        scrapy_request = Request(
+        scrapy_request = ScrapyRequest(
             url=apify_request.url,
             method=apify_request.method,
             meta={
@@ -173,5 +146,5 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
     if apify_request.user_data:
         scrapy_request.meta['userData'] = apify_request.user_data
 
-    Actor.log.debug(f'[{call_id}]: an apify_request was converted to the scrapy_request={scrapy_request}')
+    logger.debug(f'an apify_request was converted to the scrapy_request={scrapy_request}')
     return scrapy_request

apify/scrapy/scheduler.py

@@ -1,41 +1,33 @@
 from __future__ import annotations
 
 import traceback
+from logging import getLogger
 from typing import TYPE_CHECKING
 
-from crawlee.storage_clients import MemoryStorageClient
+from scrapy import Spider
+from scrapy.core.scheduler import BaseScheduler
+from scrapy.utils.reactor import is_asyncio_reactor_installed
 
-from apify._configuration import Configuration
+from ._async_thread import AsyncThread
+from .requests import to_apify_request, to_scrapy_request
+from apify import Configuration
 from apify.apify_storage_client import ApifyStorageClient
+from apify.storages import RequestQueue
 
-try:
-    from scrapy import Spider
-    from scrapy.core.scheduler import BaseScheduler
-    from scrapy.utils.reactor import is_asyncio_reactor_installed
-
-    if TYPE_CHECKING:
-        from scrapy.http.request import Request
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
-from crawlee._utils.crypto import crypto_random_object_id
+if TYPE_CHECKING:
+    from scrapy.http.request import Request
+    from twisted.internet.defer import Deferred
 
-from apify import Actor
-from apify.scrapy.requests import to_apify_request, to_scrapy_request
-from apify.scrapy.utils import nested_event_loop
-from apify.storages import RequestQueue
+logger = getLogger(__name__)
 
 
 class ApifyScheduler(BaseScheduler):
-    """A Scrapy scheduler that uses the Apify Request Queue to manage requests.
+    """A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.
 
     This scheduler requires the asyncio Twisted reactor to be installed.
     """
 
     def __init__(self) -> None:
-        """Create a new instance."""
         if not is_asyncio_reactor_installed():
             raise ValueError(
                 f'{ApifyScheduler.__qualname__} requires the asyncio Twisted reactor. '
@@ -45,7 +37,10 @@ class ApifyScheduler(BaseScheduler):
         self._rq: RequestQueue | None = None
         self.spider: Spider | None = None
 
-    def open(self, spider: Spider) -> None:  # this has to be named "open"
+        # A thread with the asyncio event loop to run coroutines on.
+        self._async_thread = AsyncThread()
+
+    def open(self, spider: Spider) -> Deferred[None] | None:
         """Open the scheduler.
 
         Args:
@@ -53,23 +48,42 @@ class ApifyScheduler(BaseScheduler):
         """
         self.spider = spider
 
-        async def open_queue() -> RequestQueue:
+        async def open_rq() -> RequestQueue:
             config = Configuration.get_global_configuration()
-
-            # Use the ApifyStorageClient if the Actor is running on the Apify platform,
-            # otherwise use the MemoryStorageClient.
-            storage_client = (
-                ApifyStorageClient.from_config(config) if config.is_at_home else MemoryStorageClient.from_config(config)
-            )
-
-            return await RequestQueue.open(storage_client=storage_client)
+            if config.is_at_home:
+                storage_client = ApifyStorageClient.from_config(config)
+                return await RequestQueue.open(storage_client=storage_client)
+            return await RequestQueue.open()
 
         try:
-            self._rq = nested_event_loop.run_until_complete(open_queue())
-        except BaseException:
+            self._rq = self._async_thread.run_coro(open_rq())
+        except Exception:
             traceback.print_exc()
             raise
 
+        return None
+
+    def close(self, reason: str) -> None:
+        """Close the scheduler.
+
+        Shut down the event loop and its thread gracefully.
+
+        Args:
+            reason: The reason for closing the spider.
+        """
+        logger.debug(f'Closing {self.__class__.__name__} due to {reason}...')
+        try:
+            self._async_thread.close()
+
+        except KeyboardInterrupt:
+            logger.warning('Shutdown interrupted by KeyboardInterrupt!')
+
+        except Exception:
+            logger.exception('Exception occurred while shutting down.')
+
+        finally:
+            logger.debug(f'{self.__class__.__name__} closed successfully.')
+
     def has_pending_requests(self) -> bool:
         """Check if the scheduler has any pending requests.
 
@@ -80,8 +94,8 @@ class ApifyScheduler(BaseScheduler):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
-            is_finished = nested_event_loop.run_until_complete(self._rq.is_finished())
-        except BaseException:
+            is_finished = self._async_thread.run_coro(self._rq.is_finished())
+        except Exception:
             traceback.print_exc()
             raise
 
@@ -98,29 +112,27 @@ class ApifyScheduler(BaseScheduler):
         Returns:
             True if the request was successfully enqueued, False otherwise.
         """
-        call_id = crypto_random_object_id(8)
-        Actor.log.debug(f'[{call_id}]: ApifyScheduler.enqueue_request was called (scrapy_request={request})...')
+        logger.debug(f'ApifyScheduler.enqueue_request was called (scrapy_request={request})...')
 
         if not isinstance(self.spider, Spider):
             raise TypeError('self.spider must be an instance of the Spider class')
 
         apify_request = to_apify_request(request, spider=self.spider)
         if apify_request is None:
-            Actor.log.error(f'Request {request} was not enqueued because it could not be converted to Apify request.')
+            logger.error(f'Request {request} could not be converted to Apify request.')
             return False
 
-        Actor.log.debug(f'[{call_id}]: scrapy_request was transformed to apify_request (apify_request={apify_request})')
-
+        logger.debug(f'Converted to apify_request: {apify_request}')
         if not isinstance(self._rq, RequestQueue):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
-            result = nested_event_loop.run_until_complete(self._rq.add_request(apify_request))
-        except BaseException:
+            result = self._async_thread.run_coro(self._rq.add_request(apify_request))
+        except Exception:
             traceback.print_exc()
             raise
 
-        Actor.log.debug(f'[{call_id}]: rq.add_request.result={result}...')
+        logger.debug(f'rq.add_request result: {result}')
         return bool(result.was_already_present)
 
     def next_request(self) -> Request | None:
@@ -129,40 +141,31 @@ class ApifyScheduler(BaseScheduler):
         Returns:
             The next request, or None if there are no more requests.
         """
-        call_id = crypto_random_object_id(8)
-        Actor.log.debug(f'[{call_id}]: ApifyScheduler.next_request was called...')
-
+        logger.debug('next_request called...')
         if not isinstance(self._rq, RequestQueue):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
-        # Fetch the next request from the Request Queue
         try:
-            apify_request = nested_event_loop.run_until_complete(self._rq.fetch_next_request())
-        except BaseException:
+            apify_request = self._async_thread.run_coro(self._rq.fetch_next_request())
+        except Exception:
             traceback.print_exc()
             raise
 
-        Actor.log.debug(
-            f'[{call_id}]: a new apify_request from the scheduler was fetched (apify_request={apify_request})'
-        )
-
+        logger.debug(f'Fetched apify_request: {apify_request}')
         if apify_request is None:
             return None
 
         if not isinstance(self.spider, Spider):
             raise TypeError('self.spider must be an instance of the Spider class')
 
-        # Let the Request Queue know that the request is being handled. Every request should be marked as handled,
-        # retrying is handled by the Scrapy's RetryMiddleware.
+        # Let the request queue know that the request is being handled. Every request should
+        # be marked as handled, retrying is handled by the Scrapy's RetryMiddleware.
         try:
-            nested_event_loop.run_until_complete(self._rq.mark_request_as_handled(apify_request))
-        except BaseException:
+            self._async_thread.run_coro(self._rq.mark_request_as_handled(apify_request))
+        except Exception:
             traceback.print_exc()
             raise
 
         scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
-        Actor.log.debug(
-            f'[{call_id}]: apify_request was transformed to the scrapy_request which is gonna be returned '
-            f'(scrapy_request={scrapy_request})',
-        )
+        logger.debug(f'Converted to scrapy_request: {scrapy_request}')
         return scrapy_request

apify/scrapy/utils.py

@@ -1,29 +1,16 @@
 from __future__ import annotations
 
-import asyncio
 from base64 import b64encode
 from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
-from apify_shared.utils import ignore_docs
+from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_bytes
 
-try:
-    from scrapy.utils.project import get_project_settings
-    from scrapy.utils.python import to_bytes
+if TYPE_CHECKING:
+    from scrapy.settings import Settings
 
-    if TYPE_CHECKING:
-        from scrapy.settings import Settings
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. For example, if you use pip, run '
-        '"pip install apify[scrapy]".'
-    ) from exc
 
-
-nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
-
-
-@ignore_docs
 def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'latin-1') -> bytes:
     """Generate a basic authentication header for the given username and password."""
     string = f'{unquote(username)}:{unquote(password)}'
@@ -31,18 +18,6 @@ def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'la
     return b'Basic ' + b64encode(user_pass)
 
 
-@ignore_docs
-def get_running_event_loop_id() -> int:
-    """Get the ID of the currently running event loop.
-
-    It could be useful mainly for debugging purposes.
-
-    Returns:
-        The ID of the event loop.
-    """
-    return id(asyncio.get_running_loop())
-
-
 def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict | None = None) -> Settings:
     """Integrates Apify configuration into a Scrapy project settings.
 
@@ -65,10 +40,6 @@ def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict
     # ensuring it is executed as the final step in the pipeline sequence
     settings['ITEM_PIPELINES']['apify.scrapy.pipelines.ActorDatasetPushPipeline'] = 1000
 
-    # Disable the default AjaxCrawlMiddleware since it can be problematic with Apify. It can return a new request
-    # during process_response, but currently we have no way of detecting it and handling it properly.
-    settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware'] = None
-
     # Replace the default HttpProxyMiddleware with ApifyHttpProxyMiddleware
     settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware'] = None
     settings['DOWNLOADER_MIDDLEWARES']['apify.scrapy.middlewares.ApifyHttpProxyMiddleware'] = 750

{apify-2.2.2b2.dist-info → apify-2.3.0b1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apify
-Version: 2.2.2b2
+Version: 2.3.0b1
 Summary: Apify SDK for Python
 License: Apache-2.0
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries
 Provides-Extra: scrapy
-Requires-Dist: apify-client (>=1.8.1)
+Requires-Dist: apify-client (>=1.9.1)
 Requires-Dist: apify-shared (>=1.2.1)
 Requires-Dist: crawlee (>=0.5.1,<0.6.0)
 Requires-Dist: cryptography (>=42.0.0)

{apify-2.2.2b2.dist-info → apify-2.3.0b1.dist-info}/RECORD

@@ -1,5 +1,5 @@
 apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
-apify/_actor.py,sha256=Pb7HPHIAodQOiGtyb-At45x8GfDItWCusRtQkoz1Pq4,46138
+apify/_actor.py,sha256=cN74o-yrxwtrvt6RA9SXzp42fEw-xcZcd9ur8KUhRmM,46129
 apify/_configuration.py,sha256=T3Z_o_W98iSyTbrutfb578yW51aexZ_V0FcLwTxFLjI,10878
 apify/_consts.py,sha256=_Xq4hOfOA1iZ3n1P967YWdyncKivpbX6RTlp_qanUoE,330
 apify/_crypto.py,sha256=e0_aM3l9_5Osk-jszYOOjrAKK60OggSHbiw5c30QnsU,5638
@@ -8,7 +8,7 @@ apify/_platform_event_manager.py,sha256=44xyV0Lpzf4h4VZ0rkyYg_nhbQkEONNor8_Z9gIK
 apify/_proxy_configuration.py,sha256=c-O6_PZ9pUD-i4J0RFEKTtfyJPP2rTRJJA1TH8NVsV8,13189
 apify/_utils.py,sha256=CCLkpAsZKp00ykm88Z_Fbck5PNT0j6mJYOuD0RxzZUs,1620
 apify/apify_storage_client/__init__.py,sha256=-UbR68bFsDR6ln8OFs4t50eqcnY36hujO-SeOt-KmcA,114
-apify/apify_storage_client/_apify_storage_client.py,sha256=0rS75JoRHt7stRYS9-oqm3DmaSIZQN5C11N5MZQUvlA,2616
+apify/apify_storage_client/_apify_storage_client.py,sha256=jTX5vd-K9mnFTyZu2V2dUg7oyWogvmNIDUlEXnvIlOw,2766
 apify/apify_storage_client/_dataset_client.py,sha256=UUodnR_MQBg5RkURrfegkGJWR5OmdPPgPfGepvkdQoU,5580
 apify/apify_storage_client/_dataset_collection_client.py,sha256=qCcKZlA0bkO-sL7xED0Yose85NlrRa9AKr4oCSrYX6k,1489
 apify/apify_storage_client/_key_value_store_client.py,sha256=MSuoIeqEHLu92WfUU7kyB3Cc_gKUlm8TghnU3_xkPtE,3363
@@ -18,21 +18,24 @@ apify/apify_storage_client/_request_queue_collection_client.py,sha256=MdzgbQb2D8
 apify/apify_storage_client/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/log.py,sha256=j-E4t-WeA93bc1NCQRG8sTntehQCiiN8ia-MdQe3_Ts,1291
 apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/__init__.py,sha256=HE5wCN7-DZKPydLCOvjNyLuL3CvN2fUFweXfrDfe1Ss,348
+apify/scrapy/__init__.py,sha256=m2a0ts_JY9xJkBy4JU5mV8PJqjA3GGKLXBFu4nl-n-A,1048
+apify/scrapy/_actor_runner.py,sha256=rXWSnlQWGskDUH8PtLCv5SkOIx4AiVa4QbCYeCett5c,938
+apify/scrapy/_async_thread.py,sha256=AfeH9ZkSRZXxL11wzwrroDNsTzq4tAvURlinUZBtYMA,4753
+apify/scrapy/_logging_config.py,sha256=hFq90fNtZyjjJA7w2k-mtuEC8xCFiBMTalbwPDcaig4,2022
 apify/scrapy/middlewares/__init__.py,sha256=tfW-d3WFWLeNEjL8fTmon6NwgD-OXx1Bw2fBdU-wPy4,114
-apify/scrapy/middlewares/apify_proxy.py,sha256=H8a3vrA_7S_ucRkE3VDNMU8TY2CdzGTMXbhbJbfLv1c,5755
+apify/scrapy/middlewares/apify_proxy.py,sha256=CDAOXS3bcVDZHM3B0GvhXbxEikMIadLF_0P73WL_nI4,5550
 apify/scrapy/middlewares/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/scrapy/pipelines/__init__.py,sha256=GWPeLN_Zwj8vRBWtXW6DaxdB7mvyQ7Jw5Tz1ccgWlZI,119
-apify/scrapy/pipelines/actor_dataset_push.py,sha256=otggoULfUdCqOPJLb9wMROZ9WylnlL-209930tMS2Rg,971
+apify/scrapy/pipelines/actor_dataset_push.py,sha256=XUUyznQTD-E3wYUUFt2WAOnWhbnRrY0WuedlfYfYhDI,846
 apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/scrapy/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify/scrapy/requests.py,sha256=yZ9hIsz2YyqOoOwzN9F1h76wG4qwawrI6h_6xq0I7Iw,7599
-apify/scrapy/scheduler.py,sha256=blO333BhFDMu3wAvSQONYdhmAmyiysqsv3YF5FKO_20,6281
-apify/scrapy/utils.py,sha256=758DcHCSAgCTProY0QX74uJ1XrzVsQwvCmFanj2f_3Q,2928
+apify/scrapy/requests.py,sha256=tOiFtG0kyzbBwtmaOisLAcpJENR1eDtpPR1nRH7JJGg,6551
+apify/scrapy/scheduler.py,sha256=-r1wZjMmeRDPxZKGHO-EYDYpGdDgSPAdNgMFViqUK8E,6019
+apify/scrapy/utils.py,sha256=5cka33PWc_at14yjhnLkCvY4h-ySUgVVhhDLxTy39ZI,1965
 apify/storages/__init__.py,sha256=FW-z6ubuPnHGM-Wp15T8mR5q6lnpDGrCW-IkgZd5L30,177
 apify/storages/_request_list.py,sha256=-lZJcE5nq69aJhGFJ7Sh2ctqgAWUDyOwYm5_0y1hdAE,5865
 apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-2.2.2b2.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-2.2.2b2.dist-info/METADATA,sha256=QZkLUfL0yzzoIjc_9Gdst5EG_600Nw044phOKoMhZCw,8696
-apify-2.2.2b2.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
-apify-2.2.2b2.dist-info/RECORD,,
+apify-2.3.0b1.dist-info/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-2.3.0b1.dist-info/METADATA,sha256=IO-6Yvs9P-SeqNnJ04jAWzer0mzqa9hY1AJqKr0X_2w,8696
+apify-2.3.0b1.dist-info/WHEEL,sha256=IYZQI976HJqqOpQU6PHkJ8fb3tMNBFjg-Cn-pwAbaFM,88
+apify-2.3.0b1.dist-info/RECORD,,