apify 2.2.2b2__tar.gz → 2.3.0b1__tar.gz

{apify-2.2.2b2 → apify-2.3.0b1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: apify
-Version: 2.2.2b2
+Version: 2.3.0b1
 Summary: Apify SDK for Python
 License: Apache-2.0
 Keywords: apify,sdk,automation,chrome,crawlee,crawler,headless,scraper,scraping
@@ -19,7 +19,7 @@ Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Software Development :: Libraries
 Provides-Extra: scrapy
-Requires-Dist: apify-client (>=1.8.1)
+Requires-Dist: apify-client (>=1.9.1)
 Requires-Dist: apify-shared (>=1.2.1)
 Requires-Dist: crawlee (>=0.5.1,<0.6.0)
 Requires-Dist: cryptography (>=42.0.0)

{apify-2.2.2b2 → apify-2.3.0b1}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api"
 
 [tool.poetry]
 name = "apify"
-version = "2.2.2b2"
+version = "2.3.0b1"
 description = "Apify SDK for Python"
 authors = ["Apify Technologies s.r.o. <support@apify.com>"]
 license = "Apache-2.0"
@@ -43,7 +43,7 @@ keywords = [
 
 [tool.poetry.dependencies]
 python = "^3.9"
-apify-client = ">=1.8.1"
+apify-client = ">=1.9.1"
 apify-shared = ">=1.2.1"
 crawlee = "~0.5.1"
 cryptography = ">=42.0.0"
@@ -60,7 +60,7 @@ websockets = ">=10.0 <14.0.0"
 build = "~1.2.0"
 filelock = "~3.17.0"
 griffe = "~1.5.0"
-mypy = "~1.14.0"
+mypy = "~1.15.0"
 pre-commit = "~4.1.0"
 pydoc-markdown = "~4.8.0"
 pytest = "~8.3.0"
@@ -136,6 +136,18 @@ indent-style = "space"
     "TRY301",  # Abstract `raise` to an inner function
     "PLW0603", # Using the global statement to update `{name}` is discouraged
 ]
+"**/docs/**/scrapy_project/**/__main__.py" = [
+    # Because of asyncioreactor.install() call.
+    "E402", # Module level import not at top of file
+]
+"**/docs/**/scrapy_project/**" = [
+    # Local imports are mixed up with the Apify SDK.
+    "I001",  # Import block is un-sorted or un-formatted
+    # Class variables are common in Scrapy projects.
+    "RUF012",  # Mutable class attributes should be annotated with `typing.ClassVar`
+    # Local imports in Scrapy project.
+    "TID252",  # Prefer absolute imports over relative imports from parent modules
+]
 
 [tool.ruff.lint.flake8-quotes]
 docstring-quotes = "double"

{apify-2.2.2b2 → apify-2.3.0b1}/src/apify/_actor.py

@@ -270,8 +270,8 @@ class _ActorType:
             self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in IPython')
         elif os.getenv('PYTEST_CURRENT_TEST', default=False):  # noqa: PLW1508
             self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in an unit test')
-        elif hasattr(asyncio, '_nest_patched'):
-            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running in a nested event loop')
+        elif os.getenv('SCRAPY_SETTINGS_MODULE'):
+            self.log.debug(f'Not calling sys.exit({exit_code}) because Actor is running with Scrapy')
         else:
             sys.exit(exit_code)
 

{apify-2.2.2b2 → apify-2.3.0b1}/src/apify/apify_storage_client/_apify_storage_client.py

@@ -66,3 +66,7 @@ class ApifyStorageClient(BaseStorageClient):
     @override
     async def purge_on_start(self) -> None:
         pass
+
+    @override
+    def get_rate_limit_errors(self) -> dict[int, int]:  # type: ignore[misc]
+        return self._apify_client.stats.rate_limit_errors

apify-2.3.0b1/src/apify/scrapy/__init__.py

@@ -0,0 +1,32 @@
+from crawlee._utils.try_import import install_import_hook as _install_import_hook
+from crawlee._utils.try_import import try_import as _try_import
+
+_install_import_hook(__name__)
+
+# The following imports use try_import to handle optional dependencies, as they may not always be available.
+
+with _try_import(__name__, 'run_scrapy_actor'):
+    from ._actor_runner import run_scrapy_actor
+
+with _try_import(__name__, 'initialize_logging'):
+    from ._logging_config import initialize_logging
+
+with _try_import(__name__, 'to_apify_request', 'to_scrapy_request'):
+    from .requests import to_apify_request, to_scrapy_request
+
+with _try_import(__name__, 'ApifyScheduler'):
+    from .scheduler import ApifyScheduler
+
+with _try_import(__name__, 'apply_apify_settings', 'get_basic_auth_header'):
+    from .utils import apply_apify_settings, get_basic_auth_header
+
+
+__all__ = [
+    'ApifyScheduler',
+    'apply_apify_settings',
+    'get_basic_auth_header',
+    'initialize_logging',
+    'run_scrapy_actor',
+    'to_apify_request',
+    'to_scrapy_request',
+]

apify-2.3.0b1/src/apify/scrapy/_actor_runner.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+import asyncio
+from typing import TYPE_CHECKING
+
+from twisted.internet.defer import Deferred, ensureDeferred
+from twisted.internet.task import react
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
+
+async def _run_coro_as_deferred(coro: Coroutine) -> None:
+    """Wrap the given asyncio coroutine in a Task and await its result as a Twisted Deferred."""
+    task = asyncio.ensure_future(coro)
+    await Deferred.fromFuture(task)
+
+
+def run_scrapy_actor(coro: Coroutine) -> None:
+    """Start Twisted's reactor and execute the provided Actor coroutine.
+
+    This function initiates the Twisted reactor and runs the given asyncio coroutine (typically the
+    Actor's main) by converting it to a Deferred. This bridges the asyncio and Twisted event loops,
+    enabling the Apify and Scrapy integration to work together.
+    """
+    react(lambda _: ensureDeferred(_run_coro_as_deferred(coro)))

apify-2.3.0b1/src/apify/scrapy/_async_thread.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+from concurrent import futures
+from datetime import timedelta
+from logging import getLogger
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
+logger = getLogger(__name__)
+
+
+class AsyncThread:
+    """Class for running an asyncio event loop in a separate thread.
+
+    This allows running asynchronous coroutines from synchronous code by executingthem on an event loop
+    that runs in its own dedicated thread.
+    """
+
+    def __init__(self) -> None:
+        self._eventloop = asyncio.new_event_loop()
+
+        # Start the event loop in a dedicated daemon thread.
+        self._thread = threading.Thread(
+            target=self._start_event_loop,
+            daemon=True,
+        )
+        self._thread.start()
+
+    def run_coro(
+        self,
+        coro: Coroutine,
+        timeout: timedelta = timedelta(seconds=60),
+    ) -> Any:
+        """Run a coroutine on an event loop running in a separate thread.
+
+        This method schedules the coroutine to run on the event loop and blocks until the coroutine completes
+        or the specified timeout is reached.
+
+        Args:
+            coro: The coroutine to run.
+            timeout: The maximum number of seconds to wait for the coroutine to finish.
+
+        Returns:
+            The result returned by the coroutine.
+
+        Raises:
+            RuntimeError: If the event loop is not running.
+            TimeoutError: If the coroutine does not complete within the timeout.
+            Exception: Any exception raised during coroutine execution.
+        """
+        if not self._eventloop.is_running():
+            raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is not running.')
+
+        # Submit the coroutine to the event loop running in the other thread.
+        future = asyncio.run_coroutine_threadsafe(coro, self._eventloop)
+        try:
+            # Wait for the coroutine's result until the specified timeout.
+            return future.result(timeout=timeout.total_seconds())
+        except futures.TimeoutError as exc:
+            logger.exception('Coroutine execution timed out.', exc_info=exc)
+            raise
+        except Exception as exc:
+            logger.exception('Coroutine execution raised an exception.', exc_info=exc)
+            raise
+
+    def close(self, timeout: timedelta = timedelta(seconds=60)) -> None:
+        """Close the event loop and its thread gracefully.
+
+        This method cancels all pending tasks, stops the event loop, and waits for the thread to exit.
+        If the thread does not exit within the given timeout, a forced shutdown is attempted.
+
+        Args:
+            timeout: The maximum number of seconds to wait for the event loop thread to exit.
+        """
+        if self._eventloop.is_running():
+            # Cancel all pending tasks in the event loop.
+            self.run_coro(self._shutdown_tasks())
+
+        # Schedule the event loop to stop.
+        self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+
+        # Wait for the event loop thread to finish execution.
+        self._thread.join(timeout=timeout.total_seconds())
+
+        # If the thread is still running after the timeout, force a shutdown.
+        if self._thread.is_alive():
+            logger.warning('Event loop thread did not exit cleanly! Forcing shutdown...')
+            self._force_exit_event_loop()
+
+    def _start_event_loop(self) -> None:
+        """Set up and run the asyncio event loop in the dedicated thread."""
+        asyncio.set_event_loop(self._eventloop)
+        try:
+            self._eventloop.run_forever()
+        finally:
+            self._eventloop.close()
+            logger.debug('Asyncio event loop has been closed.')
+
+    async def _shutdown_tasks(self) -> None:
+        """Cancel all pending tasks in the event loop."""
+        # Retrieve all tasks for the event loop, excluding the current task.
+        tasks = [task for task in asyncio.all_tasks(self._eventloop) if task is not asyncio.current_task()]
+
+        # Cancel each pending task.
+        for task in tasks:
+            task.cancel()
+
+        # Wait until all tasks have been cancelled or finished.
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    def _force_exit_event_loop(self) -> None:
+        """Forcefully shut down the event loop and its thread."""
+        try:
+            logger.info('Forced shutdown of the event loop and its thread...')
+            self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+            self._thread.join(timeout=5)
+        except Exception as exc:
+            logger.exception('Exception occurred during forced event loop shutdown.', exc_info=exc)

apify-2.3.0b1/src/apify/scrapy/_logging_config.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from scrapy.utils import log as scrapy_logging
+from scrapy.utils.project import get_project_settings
+
+from apify.log import ActorLogFormatter
+
+# Define logger names.
+_PRIMARY_LOGGERS = ['apify', 'apify_client', 'scrapy']
+_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'httpx', 'protego', 'twisted']
+_ALL_LOGGERS = _PRIMARY_LOGGERS + _SUPPLEMENTAL_LOGGERS
+
+
+def _configure_logger(name: str | None, logging_level: str, handler: logging.Handler) -> None:
+    """Clear and reconfigure the logger."""
+    logger = logging.getLogger(name)
+    logger.handlers.clear()
+    logger.setLevel(logging_level)
+
+    if name is None:  # Root logger.
+        logger.addHandler(handler)
+        logger.propagate = False
+    else:
+        logger.propagate = True
+
+
+def initialize_logging() -> None:
+    """Configure logging for Apify Actors and adjust Scrapy's logging settings."""
+    # Retrieve Scrapy project settings and determine the logging level.
+    settings = get_project_settings()
+    logging_level = settings.get('LOG_LEVEL', 'INFO')  # Default to INFO.
+
+    # Create a custom handler with the Apify log formatter.
+    handler = logging.StreamHandler()
+    handler.setFormatter(ActorLogFormatter(include_logger_name=True))
+
+    # Configure the root logger and all other defined loggers.
+    for logger_name in [None, *_ALL_LOGGERS]:
+        _configure_logger(logger_name, logging_level, handler)
+
+    # Set the 'httpx' logger to a less verbose level.
+    logging.getLogger('httpx').setLevel('WARNING')
+
+    # Monkey-patch Scrapy's logging configuration to re-apply our settings.
+    original_configure_logging = scrapy_logging.configure_logging
+
+    def new_configure_logging(*args: Any, **kwargs: Any) -> None:
+        original_configure_logging(*args, **kwargs)
+        for logger_name in [None, *_ALL_LOGGERS]:
+            _configure_logger(logger_name, logging_level, handler)
+
+    scrapy_logging.configure_logging = new_configure_logging

{apify-2.2.2b2 → apify-2.3.0b1}/src/apify/scrapy/middlewares/apify_proxy.py

@@ -3,19 +3,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
-try:
-    if TYPE_CHECKING:
-        from scrapy import Request, Spider
-        from scrapy.crawler import Crawler
-    from scrapy.core.downloader.handlers.http11 import TunnelError
-    from scrapy.exceptions import NotConfigured
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
+from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
 
 from apify import Actor, ProxyConfiguration
-from apify.scrapy.utils import get_basic_auth_header
+from apify.scrapy import get_basic_auth_header
+
+if TYPE_CHECKING:
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
 
 
 class ApifyHttpProxyMiddleware:
@@ -51,7 +47,7 @@ class ApifyHttpProxyMiddleware:
         proxy_settings: dict | None = crawler.settings.get('APIFY_PROXY_SETTINGS')
 
         if proxy_settings is None:
-            Actor.log.warning(
+            Actor.log.info(
                 'ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing '
                 ' in the Actor input.'
             )
@@ -60,7 +56,7 @@ class ApifyHttpProxyMiddleware:
         use_apify_proxy = proxy_settings.get('useApifyProxy', False)
 
         if use_apify_proxy is not True:
-            Actor.log.warning(
+            Actor.log.info(
                 'ApifyHttpProxyMiddleware is not going to be used. Actor input field '
                 '"proxyConfiguration.useApifyProxy" is set to False.'
             )

{apify-2.2.2b2 → apify-2.3.0b1}/src/apify/scrapy/pipelines/actor_dataset_push.py

@@ -1,19 +1,17 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import TYPE_CHECKING
 
 from itemadapter.adapter import ItemAdapter
 
-try:
-    if TYPE_CHECKING:
-        from scrapy import Item, Spider
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
 from apify import Actor
 
+if TYPE_CHECKING:
+    from scrapy import Item, Spider
+
+logger = getLogger(__name__)
+
 
 class ActorDatasetPushPipeline:
     """A Scrapy pipeline for pushing items to an Actor's default dataset.
@@ -28,6 +26,6 @@ class ActorDatasetPushPipeline:
     ) -> Item:
         """Pushes the provided Scrapy item to the Actor's default dataset."""
         item_dict = ItemAdapter(item).asdict()
-        Actor.log.debug(f'Pushing item={item_dict} produced by spider={spider} to the dataset.')
+        logger.debug(f'Pushing item={item_dict} produced by spider={spider} to the dataset.')
         await Actor.push_data(item_dict)
         return item

apify-2.3.0b1/src/apify/scrapy/requests.py

@@ -0,0 +1,150 @@
+from __future__ import annotations
+
+import codecs
+import pickle
+from logging import getLogger
+from typing import Any, cast
+
+from scrapy import Request as ScrapyRequest
+from scrapy import Spider
+from scrapy.http.headers import Headers
+from scrapy.utils.request import request_from_dict
+
+from crawlee import Request as ApifyRequest
+from crawlee._types import HttpHeaders
+
+logger = getLogger(__name__)
+
+
+def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequest | None:
+    """Convert a Scrapy request to an Apify request.
+
+    Args:
+        scrapy_request: The Scrapy request to be converted.
+        spider: The Scrapy spider that the request is associated with.
+
+    Returns:
+        The converted Apify request if the conversion was successful, otherwise None.
+    """
+    if not isinstance(scrapy_request, ScrapyRequest):
+        logger.warning('Failed to convert to Apify request: Scrapy request must be a ScrapyRequest instance.')  # type: ignore[unreachable]
+        return None
+
+    logger.debug(f'to_apify_request was called (scrapy_request={scrapy_request})...')
+
+    # Configuration to behave as similarly as possible to Scrapy's default RFPDupeFilter.
+    request_kwargs: dict[str, Any] = {
+        'url': scrapy_request.url,
+        'method': scrapy_request.method,
+        'payload': scrapy_request.body,
+        'use_extended_unique_key': True,
+        'keep_url_fragment': False,
+    }
+
+    try:
+        if scrapy_request.dont_filter:
+            request_kwargs['always_enqueue'] = True
+        else:
+            if scrapy_request.meta.get('apify_request_unique_key'):
+                request_kwargs['unique_key'] = scrapy_request.meta['apify_request_unique_key']
+
+            if scrapy_request.meta.get('apify_request_id'):
+                request_kwargs['id'] = scrapy_request.meta['apify_request_id']
+
+        request_kwargs['user_data'] = scrapy_request.meta.get('userData', {})
+
+        # Convert Scrapy's headers to a HttpHeaders and store them in the apify_request
+        if isinstance(scrapy_request.headers, Headers):
+            request_kwargs['headers'] = HttpHeaders(dict(scrapy_request.headers.to_unicode_dict()))
+        else:
+            logger.warning(  # type: ignore[unreachable]
+                f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
+            )
+
+        apify_request = ApifyRequest.from_url(**request_kwargs)
+
+        # Serialize the Scrapy ScrapyRequest and store it in the apify_request.
+        #   - This process involves converting the Scrapy ScrapyRequest object into a dictionary, encoding it to base64,
+        #     and storing it as 'scrapy_request' within the 'userData' dictionary of the apify_request.
+        #   - The serialization process can be referenced at: https://stackoverflow.com/questions/30469575/.
+        scrapy_request_dict = scrapy_request.to_dict(spider=spider)
+        scrapy_request_dict_encoded = codecs.encode(pickle.dumps(scrapy_request_dict), 'base64').decode()
+        apify_request.user_data['scrapy_request'] = scrapy_request_dict_encoded
+
+    except Exception as exc:
+        logger.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
+        return None
+
+    logger.debug(f'scrapy_request was converted to the apify_request={apify_request}')
+    return apify_request
+
+
+def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequest:
+    """Convert an Apify request to a Scrapy request.
+
+    Args:
+        apify_request: The Apify request to be converted.
+        spider: The Scrapy spider that the request is associated with.
+
+    Raises:
+        TypeError: If the Apify request is not an instance of the `ApifyRequest` class.
+        ValueError: If the Apify request does not contain the required keys.
+
+    Returns:
+        The converted Scrapy request.
+    """
+    if not isinstance(cast(Any, apify_request), ApifyRequest):
+        raise TypeError('apify_request must be a crawlee.ScrapyRequest instance')
+
+    logger.debug(f'to_scrapy_request was called (apify_request={apify_request})...')
+
+    # If the apify_request comes from the Scrapy
+    if 'scrapy_request' in apify_request.user_data:
+        # Deserialize the Scrapy ScrapyRequest from the apify_request.
+        #   - This process involves decoding the base64-encoded request data and reconstructing
+        #     the Scrapy ScrapyRequest object from its dictionary representation.
+        logger.debug('Restoring the Scrapy ScrapyRequest from the apify_request...')
+
+        scrapy_request_dict_encoded = apify_request.user_data['scrapy_request']
+        if not isinstance(scrapy_request_dict_encoded, str):
+            raise TypeError('scrapy_request_dict_encoded must be a string')
+
+        scrapy_request_dict = pickle.loads(codecs.decode(scrapy_request_dict_encoded.encode(), 'base64'))
+        if not isinstance(scrapy_request_dict, dict):
+            raise TypeError('scrapy_request_dict must be a dictionary')
+
+        scrapy_request = request_from_dict(scrapy_request_dict, spider=spider)
+        if not isinstance(scrapy_request, ScrapyRequest):
+            raise TypeError('scrapy_request must be an instance of the ScrapyRequest class')
+
+        logger.debug(f'Scrapy ScrapyRequest successfully reconstructed (scrapy_request={scrapy_request})...')
+
+        # Update the meta field with the meta field from the apify_request
+        meta = scrapy_request.meta or {}
+        meta.update({'apify_request_id': apify_request.id, 'apify_request_unique_key': apify_request.unique_key})
+        # scrapy_request.meta is a property, so we have to set it like this
+        scrapy_request._meta = meta  # noqa: SLF001
+
+    # If the apify_request comes directly from the Scrapy, typically start URLs.
+    else:
+        logger.debug('Gonna create a new Scrapy ScrapyRequest (cannot be restored)')
+
+        scrapy_request = ScrapyRequest(
+            url=apify_request.url,
+            method=apify_request.method,
+            meta={
+                'apify_request_id': apify_request.id,
+                'apify_request_unique_key': apify_request.unique_key,
+            },
+        )
+
+    # Add optional 'headers' field
+    if apify_request.headers:
+        scrapy_request.headers |= Headers(apify_request.headers)
+
+    # Add optional 'userData' field
+    if apify_request.user_data:
+        scrapy_request.meta['userData'] = apify_request.user_data
+
+    logger.debug(f'an apify_request was converted to the scrapy_request={scrapy_request}')
+    return scrapy_request

{apify-2.2.2b2 → apify-2.3.0b1}/src/apify/scrapy/scheduler.py

@@ -1,41 +1,33 @@
 from __future__ import annotations
 
 import traceback
+from logging import getLogger
 from typing import TYPE_CHECKING
 
-from crawlee.storage_clients import MemoryStorageClient
+from scrapy import Spider
+from scrapy.core.scheduler import BaseScheduler
+from scrapy.utils.reactor import is_asyncio_reactor_installed
 
-from apify._configuration import Configuration
+from ._async_thread import AsyncThread
+from .requests import to_apify_request, to_scrapy_request
+from apify import Configuration
 from apify.apify_storage_client import ApifyStorageClient
+from apify.storages import RequestQueue
 
-try:
-    from scrapy import Spider
-    from scrapy.core.scheduler import BaseScheduler
-    from scrapy.utils.reactor import is_asyncio_reactor_installed
-
-    if TYPE_CHECKING:
-        from scrapy.http.request import Request
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
-from crawlee._utils.crypto import crypto_random_object_id
+if TYPE_CHECKING:
+    from scrapy.http.request import Request
+    from twisted.internet.defer import Deferred
 
-from apify import Actor
-from apify.scrapy.requests import to_apify_request, to_scrapy_request
-from apify.scrapy.utils import nested_event_loop
-from apify.storages import RequestQueue
+logger = getLogger(__name__)
 
 
 class ApifyScheduler(BaseScheduler):
-    """A Scrapy scheduler that uses the Apify Request Queue to manage requests.
+    """A Scrapy scheduler that uses the Apify `RequestQueue` to manage requests.
 
     This scheduler requires the asyncio Twisted reactor to be installed.
     """
 
     def __init__(self) -> None:
-        """Create a new instance."""
         if not is_asyncio_reactor_installed():
             raise ValueError(
                 f'{ApifyScheduler.__qualname__} requires the asyncio Twisted reactor. '
@@ -45,7 +37,10 @@ class ApifyScheduler(BaseScheduler):
         self._rq: RequestQueue | None = None
         self.spider: Spider | None = None
 
-    def open(self, spider: Spider) -> None:  # this has to be named "open"
+        # A thread with the asyncio event loop to run coroutines on.
+        self._async_thread = AsyncThread()
+
+    def open(self, spider: Spider) -> Deferred[None] | None:
         """Open the scheduler.
 
         Args:
@@ -53,23 +48,42 @@ class ApifyScheduler(BaseScheduler):
         """
         self.spider = spider
 
-        async def open_queue() -> RequestQueue:
+        async def open_rq() -> RequestQueue:
             config = Configuration.get_global_configuration()
-
-            # Use the ApifyStorageClient if the Actor is running on the Apify platform,
-            # otherwise use the MemoryStorageClient.
-            storage_client = (
-                ApifyStorageClient.from_config(config) if config.is_at_home else MemoryStorageClient.from_config(config)
-            )
-
-            return await RequestQueue.open(storage_client=storage_client)
+            if config.is_at_home:
+                storage_client = ApifyStorageClient.from_config(config)
+                return await RequestQueue.open(storage_client=storage_client)
+            return await RequestQueue.open()
 
         try:
-            self._rq = nested_event_loop.run_until_complete(open_queue())
-        except BaseException:
+            self._rq = self._async_thread.run_coro(open_rq())
+        except Exception:
             traceback.print_exc()
             raise
 
+        return None
+
+    def close(self, reason: str) -> None:
+        """Close the scheduler.
+
+        Shut down the event loop and its thread gracefully.
+
+        Args:
+            reason: The reason for closing the spider.
+        """
+        logger.debug(f'Closing {self.__class__.__name__} due to {reason}...')
+        try:
+            self._async_thread.close()
+
+        except KeyboardInterrupt:
+            logger.warning('Shutdown interrupted by KeyboardInterrupt!')
+
+        except Exception:
+            logger.exception('Exception occurred while shutting down.')
+
+        finally:
+            logger.debug(f'{self.__class__.__name__} closed successfully.')
+
     def has_pending_requests(self) -> bool:
         """Check if the scheduler has any pending requests.
 
@@ -80,8 +94,8 @@ class ApifyScheduler(BaseScheduler):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
-            is_finished = nested_event_loop.run_until_complete(self._rq.is_finished())
-        except BaseException:
+            is_finished = self._async_thread.run_coro(self._rq.is_finished())
+        except Exception:
             traceback.print_exc()
             raise
 
@@ -98,29 +112,27 @@ class ApifyScheduler(BaseScheduler):
         Returns:
             True if the request was successfully enqueued, False otherwise.
         """
-        call_id = crypto_random_object_id(8)
-        Actor.log.debug(f'[{call_id}]: ApifyScheduler.enqueue_request was called (scrapy_request={request})...')
+        logger.debug(f'ApifyScheduler.enqueue_request was called (scrapy_request={request})...')
 
         if not isinstance(self.spider, Spider):
             raise TypeError('self.spider must be an instance of the Spider class')
 
         apify_request = to_apify_request(request, spider=self.spider)
         if apify_request is None:
-            Actor.log.error(f'Request {request} was not enqueued because it could not be converted to Apify request.')
+            logger.error(f'Request {request} could not be converted to Apify request.')
             return False
 
-        Actor.log.debug(f'[{call_id}]: scrapy_request was transformed to apify_request (apify_request={apify_request})')
-
+        logger.debug(f'Converted to apify_request: {apify_request}')
         if not isinstance(self._rq, RequestQueue):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
-            result = nested_event_loop.run_until_complete(self._rq.add_request(apify_request))
-        except BaseException:
+            result = self._async_thread.run_coro(self._rq.add_request(apify_request))
+        except Exception:
             traceback.print_exc()
             raise
 
-        Actor.log.debug(f'[{call_id}]: rq.add_request.result={result}...')
+        logger.debug(f'rq.add_request result: {result}')
         return bool(result.was_already_present)
 
     def next_request(self) -> Request | None:
@@ -129,40 +141,31 @@ class ApifyScheduler(BaseScheduler):
         Returns:
             The next request, or None if there are no more requests.
         """
-        call_id = crypto_random_object_id(8)
-        Actor.log.debug(f'[{call_id}]: ApifyScheduler.next_request was called...')
-
+        logger.debug('next_request called...')
         if not isinstance(self._rq, RequestQueue):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
-        # Fetch the next request from the Request Queue
         try:
-            apify_request = nested_event_loop.run_until_complete(self._rq.fetch_next_request())
-        except BaseException:
+            apify_request = self._async_thread.run_coro(self._rq.fetch_next_request())
+        except Exception:
             traceback.print_exc()
             raise
 
-        Actor.log.debug(
-            f'[{call_id}]: a new apify_request from the scheduler was fetched (apify_request={apify_request})'
-        )
-
+        logger.debug(f'Fetched apify_request: {apify_request}')
         if apify_request is None:
             return None
 
         if not isinstance(self.spider, Spider):
             raise TypeError('self.spider must be an instance of the Spider class')
 
-        # Let the Request Queue know that the request is being handled. Every request should be marked as handled,
-        # retrying is handled by the Scrapy's RetryMiddleware.
+        # Let the request queue know that the request is being handled. Every request should
+        # be marked as handled, retrying is handled by the Scrapy's RetryMiddleware.
         try:
-            nested_event_loop.run_until_complete(self._rq.mark_request_as_handled(apify_request))
-        except BaseException:
+            self._async_thread.run_coro(self._rq.mark_request_as_handled(apify_request))
+        except Exception:
             traceback.print_exc()
             raise
 
         scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
-        Actor.log.debug(
-            f'[{call_id}]: apify_request was transformed to the scrapy_request which is gonna be returned '
-            f'(scrapy_request={scrapy_request})',
-        )
+        logger.debug(f'Converted to scrapy_request: {scrapy_request}')
         return scrapy_request

{apify-2.2.2b2 → apify-2.3.0b1}/src/apify/scrapy/utils.py

@@ -1,29 +1,16 @@
 from __future__ import annotations
 
-import asyncio
 from base64 import b64encode
 from typing import TYPE_CHECKING
 from urllib.parse import unquote
 
-from apify_shared.utils import ignore_docs
+from scrapy.utils.project import get_project_settings
+from scrapy.utils.python import to_bytes
 
-try:
-    from scrapy.utils.project import get_project_settings
-    from scrapy.utils.python import to_bytes
+if TYPE_CHECKING:
+    from scrapy.settings import Settings
 
-    if TYPE_CHECKING:
-        from scrapy.settings import Settings
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. For example, if you use pip, run '
-        '"pip install apify[scrapy]".'
-    ) from exc
 
-
-nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
-
-
-@ignore_docs
 def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'latin-1') -> bytes:
     """Generate a basic authentication header for the given username and password."""
     string = f'{unquote(username)}:{unquote(password)}'
@@ -31,18 +18,6 @@ def get_basic_auth_header(username: str, password: str, auth_encoding: str = 'la
     return b'Basic ' + b64encode(user_pass)
 
 
-@ignore_docs
-def get_running_event_loop_id() -> int:
-    """Get the ID of the currently running event loop.
-
-    It could be useful mainly for debugging purposes.
-
-    Returns:
-        The ID of the event loop.
-    """
-    return id(asyncio.get_running_loop())
-
-
 def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict | None = None) -> Settings:
     """Integrates Apify configuration into a Scrapy project settings.
 
@@ -65,10 +40,6 @@ def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict
     # ensuring it is executed as the final step in the pipeline sequence
     settings['ITEM_PIPELINES']['apify.scrapy.pipelines.ActorDatasetPushPipeline'] = 1000
 
-    # Disable the default AjaxCrawlMiddleware since it can be problematic with Apify. It can return a new request
-    # during process_response, but currently we have no way of detecting it and handling it properly.
-    settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware'] = None
-
     # Replace the default HttpProxyMiddleware with ApifyHttpProxyMiddleware
     settings['DOWNLOADER_MIDDLEWARES']['scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware'] = None
     settings['DOWNLOADER_MIDDLEWARES']['apify.scrapy.middlewares.ApifyHttpProxyMiddleware'] = 750

apify-2.2.2b2/src/apify/scrapy/__init__.py

@@ -1,11 +0,0 @@
-from apify.scrapy.requests import to_apify_request, to_scrapy_request
-from apify.scrapy.scheduler import ApifyScheduler
-from apify.scrapy.utils import get_basic_auth_header, get_running_event_loop_id
-
-__all__ = [
-    'ApifyScheduler',
-    'get_basic_auth_header',
-    'get_running_event_loop_id',
-    'to_apify_request',
-    'to_scrapy_request',
-]

apify-2.2.2b2/src/apify/scrapy/requests.py

@@ -1,177 +0,0 @@
-from __future__ import annotations
-
-import codecs
-import pickle
-from typing import Any, cast
-
-from apify_shared.utils import ignore_docs
-
-try:
-    from scrapy import Request, Spider
-    from scrapy.http.headers import Headers
-    from scrapy.utils.request import request_from_dict
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
-from crawlee import Request as CrawleeRequest
-from crawlee._types import HttpHeaders
-from crawlee._utils.crypto import crypto_random_object_id
-from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
-
-from apify import Actor
-
-
-def _is_request_produced_by_middleware(scrapy_request: Request) -> bool:
-    """Returns True if the Scrapy request was produced by a downloader middleware, otherwise False.
-
-    Works for RetryMiddleware and RedirectMiddleware.
-    """
-    return bool(scrapy_request.meta.get('redirect_times')) or bool(scrapy_request.meta.get('retry_times'))
-
-
-@ignore_docs
-def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest | None:
-    """Convert a Scrapy request to an Apify request.
-
-    Args:
-        scrapy_request: The Scrapy request to be converted.
-        spider: The Scrapy spider that the request is associated with.
-
-    Returns:
-        The converted Apify request if the conversion was successful, otherwise None.
-    """
-    if not isinstance(scrapy_request, Request):
-        Actor.log.warning(  # type: ignore[unreachable]
-            'Failed to convert to Apify request: Scrapy request must be a Request instance.'
-        )
-        return None
-
-    call_id = crypto_random_object_id(8)
-    Actor.log.debug(f'[{call_id}]: to_apify_request was called (scrapy_request={scrapy_request})...')
-
-    try:
-        if _is_request_produced_by_middleware(scrapy_request):
-            unique_key = compute_unique_key(
-                url=scrapy_request.url,
-                method=scrapy_request.method,  # type: ignore[arg-type]  # str vs literal
-                payload=scrapy_request.body,
-                use_extended_unique_key=True,
-            )
-        elif scrapy_request.dont_filter:
-            unique_key = crypto_random_object_id(8)
-        elif scrapy_request.meta.get('apify_request_unique_key'):
-            unique_key = scrapy_request.meta['apify_request_unique_key']
-        else:
-            unique_key = crypto_random_object_id(8)
-
-        if scrapy_request.meta.get('apify_request_id'):
-            request_id = scrapy_request.meta['apify_request_id']
-        else:
-            request_id = unique_key_to_request_id(unique_key)
-
-        apify_request = CrawleeRequest(
-            url=scrapy_request.url,
-            method=scrapy_request.method,
-            payload=scrapy_request.body,
-            user_data=scrapy_request.meta.get('userData', {}),
-            unique_key=unique_key,
-            id=request_id,
-        )
-
-        # Convert Scrapy's headers to a HttpHeaders and store them in the apify_request
-        if isinstance(scrapy_request.headers, Headers):
-            apify_request.headers = HttpHeaders(dict(scrapy_request.headers.to_unicode_dict()))
-        else:
-            Actor.log.warning(  # type: ignore[unreachable]
-                f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
-            )
-
-        # Serialize the Scrapy Request and store it in the apify_request.
-        #   - This process involves converting the Scrapy Request object into a dictionary, encoding it to base64,
-        #     and storing it as 'scrapy_request' within the 'userData' dictionary of the apify_request.
-        #   - The serialization process can be referenced at: https://stackoverflow.com/questions/30469575/.
-        scrapy_request_dict = scrapy_request.to_dict(spider=spider)
-        scrapy_request_dict_encoded = codecs.encode(pickle.dumps(scrapy_request_dict), 'base64').decode()
-        apify_request.user_data['scrapy_request'] = scrapy_request_dict_encoded
-
-    except Exception as exc:
-        Actor.log.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
-        return None
-
-    Actor.log.debug(f'[{call_id}]: scrapy_request was converted to the apify_request={apify_request}')
-    return apify_request
-
-
-@ignore_docs
-def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
-    """Convert an Apify request to a Scrapy request.
-
-    Args:
-        apify_request: The Apify request to be converted.
-        spider: The Scrapy spider that the request is associated with.
-
-    Raises:
-        TypeError: If the apify_request is not a crawlee request.
-        ValueError: If the apify_request does not contain the required keys.
-
-    Returns:
-        The converted Scrapy request.
-    """
-    if not isinstance(cast(Any, apify_request), CrawleeRequest):
-        raise TypeError('apify_request must be a crawlee.Request instance')
-
-    call_id = crypto_random_object_id(8)
-    Actor.log.debug(f'[{call_id}]: to_scrapy_request was called (apify_request={apify_request})...')
-
-    # If the apify_request comes from the Scrapy
-    if 'scrapy_request' in apify_request.user_data:
-        # Deserialize the Scrapy Request from the apify_request.
-        #   - This process involves decoding the base64-encoded request data and reconstructing
-        #     the Scrapy Request object from its dictionary representation.
-        Actor.log.debug(f'[{call_id}]: Restoring the Scrapy Request from the apify_request...')
-
-        scrapy_request_dict_encoded = apify_request.user_data['scrapy_request']
-        if not isinstance(scrapy_request_dict_encoded, str):
-            raise TypeError('scrapy_request_dict_encoded must be a string')
-
-        scrapy_request_dict = pickle.loads(codecs.decode(scrapy_request_dict_encoded.encode(), 'base64'))
-        if not isinstance(scrapy_request_dict, dict):
-            raise TypeError('scrapy_request_dict must be a dictionary')
-
-        scrapy_request = request_from_dict(scrapy_request_dict, spider=spider)
-        if not isinstance(scrapy_request, Request):
-            raise TypeError('scrapy_request must be an instance of the Request class')
-
-        Actor.log.debug(f'[{call_id}]: Scrapy Request successfully reconstructed (scrapy_request={scrapy_request})...')
-
-        # Update the meta field with the meta field from the apify_request
-        meta = scrapy_request.meta or {}
-        meta.update({'apify_request_id': apify_request.id, 'apify_request_unique_key': apify_request.unique_key})
-        # scrapy_request.meta is a property, so we have to set it like this
-        scrapy_request._meta = meta  # noqa: SLF001
-
-    # If the apify_request comes directly from the Request Queue, typically start URLs
-    else:
-        Actor.log.debug(f'[{call_id}]: gonna create a new Scrapy Request (cannot be restored)')
-
-        scrapy_request = Request(
-            url=apify_request.url,
-            method=apify_request.method,
-            meta={
-                'apify_request_id': apify_request.id,
-                'apify_request_unique_key': apify_request.unique_key,
-            },
-        )
-
-    # Add optional 'headers' field
-    if apify_request.headers:
-        scrapy_request.headers |= Headers(apify_request.headers)
-
-    # Add optional 'userData' field
-    if apify_request.user_data:
-        scrapy_request.meta['userData'] = apify_request.user_data
-
-    Actor.log.debug(f'[{call_id}]: an apify_request was converted to the scrapy_request={scrapy_request}')
-    return scrapy_request