apify 2.2.2b2__py3-none-any.whl → 2.3.0__py3-none-any.whl

apify/scrapy/_async_thread.py

@@ -0,0 +1,122 @@
+from __future__ import annotations
+
+import asyncio
+import threading
+from concurrent import futures
+from datetime import timedelta
+from logging import getLogger
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    from collections.abc import Coroutine
+
+logger = getLogger(__name__)
+
+
+class AsyncThread:
+    """Class for running an asyncio event loop in a separate thread.
+
+    This allows running asynchronous coroutines from synchronous code by executingthem on an event loop
+    that runs in its own dedicated thread.
+    """
+
+    def __init__(self) -> None:
+        self._eventloop = asyncio.new_event_loop()
+
+        # Start the event loop in a dedicated daemon thread.
+        self._thread = threading.Thread(
+            target=self._start_event_loop,
+            daemon=True,
+        )
+        self._thread.start()
+
+    def run_coro(
+        self,
+        coro: Coroutine,
+        timeout: timedelta = timedelta(seconds=60),
+    ) -> Any:
+        """Run a coroutine on an event loop running in a separate thread.
+
+        This method schedules the coroutine to run on the event loop and blocks until the coroutine completes
+        or the specified timeout is reached.
+
+        Args:
+            coro: The coroutine to run.
+            timeout: The maximum number of seconds to wait for the coroutine to finish.
+
+        Returns:
+            The result returned by the coroutine.
+
+        Raises:
+            RuntimeError: If the event loop is not running.
+            TimeoutError: If the coroutine does not complete within the timeout.
+            Exception: Any exception raised during coroutine execution.
+        """
+        if not self._eventloop.is_running():
+            raise RuntimeError(f'The coroutine {coro} cannot be executed because the event loop is not running.')
+
+        # Submit the coroutine to the event loop running in the other thread.
+        future = asyncio.run_coroutine_threadsafe(coro, self._eventloop)
+        try:
+            # Wait for the coroutine's result until the specified timeout.
+            return future.result(timeout=timeout.total_seconds())
+        except futures.TimeoutError as exc:
+            logger.exception('Coroutine execution timed out.', exc_info=exc)
+            raise
+        except Exception as exc:
+            logger.exception('Coroutine execution raised an exception.', exc_info=exc)
+            raise
+
+    def close(self, timeout: timedelta = timedelta(seconds=60)) -> None:
+        """Close the event loop and its thread gracefully.
+
+        This method cancels all pending tasks, stops the event loop, and waits for the thread to exit.
+        If the thread does not exit within the given timeout, a forced shutdown is attempted.
+
+        Args:
+            timeout: The maximum number of seconds to wait for the event loop thread to exit.
+        """
+        if self._eventloop.is_running():
+            # Cancel all pending tasks in the event loop.
+            self.run_coro(self._shutdown_tasks())
+
+        # Schedule the event loop to stop.
+        self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+
+        # Wait for the event loop thread to finish execution.
+        self._thread.join(timeout=timeout.total_seconds())
+
+        # If the thread is still running after the timeout, force a shutdown.
+        if self._thread.is_alive():
+            logger.warning('Event loop thread did not exit cleanly! Forcing shutdown...')
+            self._force_exit_event_loop()
+
+    def _start_event_loop(self) -> None:
+        """Set up and run the asyncio event loop in the dedicated thread."""
+        asyncio.set_event_loop(self._eventloop)
+        try:
+            self._eventloop.run_forever()
+        finally:
+            self._eventloop.close()
+            logger.debug('Asyncio event loop has been closed.')
+
+    async def _shutdown_tasks(self) -> None:
+        """Cancel all pending tasks in the event loop."""
+        # Retrieve all tasks for the event loop, excluding the current task.
+        tasks = [task for task in asyncio.all_tasks(self._eventloop) if task is not asyncio.current_task()]
+
+        # Cancel each pending task.
+        for task in tasks:
+            task.cancel()
+
+        # Wait until all tasks have been cancelled or finished.
+        await asyncio.gather(*tasks, return_exceptions=True)
+
+    def _force_exit_event_loop(self) -> None:
+        """Forcefully shut down the event loop and its thread."""
+        try:
+            logger.info('Forced shutdown of the event loop and its thread...')
+            self._eventloop.call_soon_threadsafe(self._eventloop.stop)
+            self._thread.join(timeout=5)
+        except Exception as exc:
+            logger.exception('Exception occurred during forced event loop shutdown.', exc_info=exc)

apify/scrapy/_logging_config.py

@@ -0,0 +1,55 @@
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from scrapy.utils import log as scrapy_logging
+from scrapy.utils.project import get_project_settings
+
+from apify.log import ActorLogFormatter
+
+# Define logger names.
+_PRIMARY_LOGGERS = ['apify', 'apify_client', 'scrapy']
+_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'httpx', 'protego', 'twisted']
+_ALL_LOGGERS = _PRIMARY_LOGGERS + _SUPPLEMENTAL_LOGGERS
+
+
+def _configure_logger(name: str | None, logging_level: str, handler: logging.Handler) -> None:
+    """Clear and reconfigure the logger."""
+    logger = logging.getLogger(name)
+    logger.handlers.clear()
+    logger.setLevel(logging_level)
+
+    if name is None:  # Root logger.
+        logger.addHandler(handler)
+        logger.propagate = False
+    else:
+        logger.propagate = True
+
+
+def initialize_logging() -> None:
+    """Configure logging for Apify Actors and adjust Scrapy's logging settings."""
+    # Retrieve Scrapy project settings and determine the logging level.
+    settings = get_project_settings()
+    logging_level = settings.get('LOG_LEVEL', 'INFO')  # Default to INFO.
+
+    # Create a custom handler with the Apify log formatter.
+    handler = logging.StreamHandler()
+    handler.setFormatter(ActorLogFormatter(include_logger_name=True))
+
+    # Configure the root logger and all other defined loggers.
+    for logger_name in [None, *_ALL_LOGGERS]:
+        _configure_logger(logger_name, logging_level, handler)
+
+    # Set the 'httpx' logger to a less verbose level.
+    logging.getLogger('httpx').setLevel('WARNING')
+
+    # Monkey-patch Scrapy's logging configuration to re-apply our settings.
+    original_configure_logging = scrapy_logging.configure_logging
+
+    def new_configure_logging(*args: Any, **kwargs: Any) -> None:
+        original_configure_logging(*args, **kwargs)
+        for logger_name in [None, *_ALL_LOGGERS]:
+            _configure_logger(logger_name, logging_level, handler)
+
+    scrapy_logging.configure_logging = new_configure_logging

apify/scrapy/middlewares/apify_proxy.py

@@ -3,19 +3,15 @@ from __future__ import annotations
 from typing import TYPE_CHECKING
 from urllib.parse import ParseResult, urlparse
 
-try:
-    if TYPE_CHECKING:
-        from scrapy import Request, Spider
-        from scrapy.crawler import Crawler
-    from scrapy.core.downloader.handlers.http11 import TunnelError
-    from scrapy.exceptions import NotConfigured
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
+from scrapy.core.downloader.handlers.http11 import TunnelError
+from scrapy.exceptions import NotConfigured
 
 from apify import Actor, ProxyConfiguration
-from apify.scrapy.utils import get_basic_auth_header
+from apify.scrapy import get_basic_auth_header
+
+if TYPE_CHECKING:
+    from scrapy import Request, Spider
+    from scrapy.crawler import Crawler
 
 
 class ApifyHttpProxyMiddleware:
@@ -51,7 +47,7 @@ class ApifyHttpProxyMiddleware:
         proxy_settings: dict | None = crawler.settings.get('APIFY_PROXY_SETTINGS')
 
         if proxy_settings is None:
-            Actor.log.warning(
+            Actor.log.info(
                 'ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing '
                 ' in the Actor input.'
             )
@@ -60,7 +56,7 @@ class ApifyHttpProxyMiddleware:
         use_apify_proxy = proxy_settings.get('useApifyProxy', False)
 
         if use_apify_proxy is not True:
-            Actor.log.warning(
+            Actor.log.info(
                 'ApifyHttpProxyMiddleware is not going to be used. Actor input field '
                 '"proxyConfiguration.useApifyProxy" is set to False.'
             )

apify/scrapy/pipelines/actor_dataset_push.py

@@ -1,19 +1,17 @@
 from __future__ import annotations
 
+from logging import getLogger
 from typing import TYPE_CHECKING
 
 from itemadapter.adapter import ItemAdapter
 
-try:
-    if TYPE_CHECKING:
-        from scrapy import Item, Spider
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
 from apify import Actor
 
+if TYPE_CHECKING:
+    from scrapy import Item, Spider
+
+logger = getLogger(__name__)
+
 
 class ActorDatasetPushPipeline:
     """A Scrapy pipeline for pushing items to an Actor's default dataset.
@@ -28,6 +26,6 @@ class ActorDatasetPushPipeline:
     ) -> Item:
         """Pushes the provided Scrapy item to the Actor's default dataset."""
         item_dict = ItemAdapter(item).asdict()
-        Actor.log.debug(f'Pushing item={item_dict} produced by spider={spider} to the dataset.')
+        logger.debug(f'Pushing item={item_dict} produced by spider={spider} to the dataset.')
         await Actor.push_data(item_dict)
         return item

apify/scrapy/requests.py

@@ -2,37 +2,21 @@ from __future__ import annotations
 
 import codecs
 import pickle
+from logging import getLogger
 from typing import Any, cast
 
-from apify_shared.utils import ignore_docs
+from scrapy import Request as ScrapyRequest
+from scrapy import Spider
+from scrapy.http.headers import Headers
+from scrapy.utils.request import request_from_dict
 
-try:
-    from scrapy import Request, Spider
-    from scrapy.http.headers import Headers
-    from scrapy.utils.request import request_from_dict
-except ImportError as exc:
-    raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
-    ) from exc
-
-from crawlee import Request as CrawleeRequest
+from crawlee import Request as ApifyRequest
 from crawlee._types import HttpHeaders
-from crawlee._utils.crypto import crypto_random_object_id
-from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
-
-from apify import Actor
-
 
-def _is_request_produced_by_middleware(scrapy_request: Request) -> bool:
-    """Returns True if the Scrapy request was produced by a downloader middleware, otherwise False.
-
-    Works for RetryMiddleware and RedirectMiddleware.
-    """
-    return bool(scrapy_request.meta.get('redirect_times')) or bool(scrapy_request.meta.get('retry_times'))
+logger = getLogger(__name__)
 
 
-@ignore_docs
-def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest | None:
+def to_apify_request(scrapy_request: ScrapyRequest, spider: Spider) -> ApifyRequest | None:
     """Convert a Scrapy request to an Apify request.
 
     Args:
@@ -42,54 +26,45 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
     Returns:
         The converted Apify request if the conversion was successful, otherwise None.
     """
-    if not isinstance(scrapy_request, Request):
-        Actor.log.warning(  # type: ignore[unreachable]
-            'Failed to convert to Apify request: Scrapy request must be a Request instance.'
-        )
+    if not isinstance(scrapy_request, ScrapyRequest):
+        logger.warning('Failed to convert to Apify request: Scrapy request must be a ScrapyRequest instance.')  # type: ignore[unreachable]
         return None
 
-    call_id = crypto_random_object_id(8)
-    Actor.log.debug(f'[{call_id}]: to_apify_request was called (scrapy_request={scrapy_request})...')
+    logger.debug(f'to_apify_request was called (scrapy_request={scrapy_request})...')
+
+    # Configuration to behave as similarly as possible to Scrapy's default RFPDupeFilter.
+    request_kwargs: dict[str, Any] = {
+        'url': scrapy_request.url,
+        'method': scrapy_request.method,
+        'payload': scrapy_request.body,
+        'use_extended_unique_key': True,
+        'keep_url_fragment': False,
+    }
 
     try:
-        if _is_request_produced_by_middleware(scrapy_request):
-            unique_key = compute_unique_key(
-                url=scrapy_request.url,
-                method=scrapy_request.method,  # type: ignore[arg-type]  # str vs literal
-                payload=scrapy_request.body,
-                use_extended_unique_key=True,
-            )
-        elif scrapy_request.dont_filter:
-            unique_key = crypto_random_object_id(8)
-        elif scrapy_request.meta.get('apify_request_unique_key'):
-            unique_key = scrapy_request.meta['apify_request_unique_key']
+        if scrapy_request.dont_filter:
+            request_kwargs['always_enqueue'] = True
         else:
-            unique_key = crypto_random_object_id(8)
+            if scrapy_request.meta.get('apify_request_unique_key'):
+                request_kwargs['unique_key'] = scrapy_request.meta['apify_request_unique_key']
 
-        if scrapy_request.meta.get('apify_request_id'):
-            request_id = scrapy_request.meta['apify_request_id']
-        else:
-            request_id = unique_key_to_request_id(unique_key)
-
-        apify_request = CrawleeRequest(
-            url=scrapy_request.url,
-            method=scrapy_request.method,
-            payload=scrapy_request.body,
-            user_data=scrapy_request.meta.get('userData', {}),
-            unique_key=unique_key,
-            id=request_id,
-        )
+            if scrapy_request.meta.get('apify_request_id'):
+                request_kwargs['id'] = scrapy_request.meta['apify_request_id']
+
+        request_kwargs['user_data'] = scrapy_request.meta.get('userData', {})
 
         # Convert Scrapy's headers to a HttpHeaders and store them in the apify_request
         if isinstance(scrapy_request.headers, Headers):
-            apify_request.headers = HttpHeaders(dict(scrapy_request.headers.to_unicode_dict()))
+            request_kwargs['headers'] = HttpHeaders(dict(scrapy_request.headers.to_unicode_dict()))
         else:
-            Actor.log.warning(  # type: ignore[unreachable]
+            logger.warning(  # type: ignore[unreachable]
                 f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
             )
 
-        # Serialize the Scrapy Request and store it in the apify_request.
-        #   - This process involves converting the Scrapy Request object into a dictionary, encoding it to base64,
+        apify_request = ApifyRequest.from_url(**request_kwargs)
+
+        # Serialize the Scrapy ScrapyRequest and store it in the apify_request.
+        #   - This process involves converting the Scrapy ScrapyRequest object into a dictionary, encoding it to base64,
         #     and storing it as 'scrapy_request' within the 'userData' dictionary of the apify_request.
         #   - The serialization process can be referenced at: https://stackoverflow.com/questions/30469575/.
         scrapy_request_dict = scrapy_request.to_dict(spider=spider)
@@ -97,15 +72,14 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest
         apify_request.user_data['scrapy_request'] = scrapy_request_dict_encoded
 
     except Exception as exc:
-        Actor.log.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
+        logger.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
         return None
 
-    Actor.log.debug(f'[{call_id}]: scrapy_request was converted to the apify_request={apify_request}')
+    logger.debug(f'scrapy_request was converted to the apify_request={apify_request}')
     return apify_request
 
 
-@ignore_docs
-def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
+def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequest:
     """Convert an Apify request to a Scrapy request.
 
     Args:
@@ -113,24 +87,23 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
         spider: The Scrapy spider that the request is associated with.
 
     Raises:
-        TypeError: If the apify_request is not a crawlee request.
-        ValueError: If the apify_request does not contain the required keys.
+        TypeError: If the Apify request is not an instance of the `ApifyRequest` class.
+        ValueError: If the Apify request does not contain the required keys.
 
     Returns:
         The converted Scrapy request.
     """
-    if not isinstance(cast(Any, apify_request), CrawleeRequest):
-        raise TypeError('apify_request must be a crawlee.Request instance')
+    if not isinstance(cast(Any, apify_request), ApifyRequest):
+        raise TypeError('apify_request must be a crawlee.ScrapyRequest instance')
 
-    call_id = crypto_random_object_id(8)
-    Actor.log.debug(f'[{call_id}]: to_scrapy_request was called (apify_request={apify_request})...')
+    logger.debug(f'to_scrapy_request was called (apify_request={apify_request})...')
 
     # If the apify_request comes from the Scrapy
     if 'scrapy_request' in apify_request.user_data:
-        # Deserialize the Scrapy Request from the apify_request.
+        # Deserialize the Scrapy ScrapyRequest from the apify_request.
         #   - This process involves decoding the base64-encoded request data and reconstructing
-        #     the Scrapy Request object from its dictionary representation.
-        Actor.log.debug(f'[{call_id}]: Restoring the Scrapy Request from the apify_request...')
+        #     the Scrapy ScrapyRequest object from its dictionary representation.
+        logger.debug('Restoring the Scrapy ScrapyRequest from the apify_request...')
 
         scrapy_request_dict_encoded = apify_request.user_data['scrapy_request']
         if not isinstance(scrapy_request_dict_encoded, str):
@@ -141,10 +114,10 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
             raise TypeError('scrapy_request_dict must be a dictionary')
 
         scrapy_request = request_from_dict(scrapy_request_dict, spider=spider)
-        if not isinstance(scrapy_request, Request):
-            raise TypeError('scrapy_request must be an instance of the Request class')
+        if not isinstance(scrapy_request, ScrapyRequest):
+            raise TypeError('scrapy_request must be an instance of the ScrapyRequest class')
 
-        Actor.log.debug(f'[{call_id}]: Scrapy Request successfully reconstructed (scrapy_request={scrapy_request})...')
+        logger.debug(f'Scrapy ScrapyRequest successfully reconstructed (scrapy_request={scrapy_request})...')
 
         # Update the meta field with the meta field from the apify_request
         meta = scrapy_request.meta or {}
@@ -152,11 +125,11 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
         # scrapy_request.meta is a property, so we have to set it like this
         scrapy_request._meta = meta  # noqa: SLF001
 
-    # If the apify_request comes directly from the Request Queue, typically start URLs
+    # If the apify_request comes directly from the Scrapy, typically start URLs.
     else:
-        Actor.log.debug(f'[{call_id}]: gonna create a new Scrapy Request (cannot be restored)')
+        logger.debug('Gonna create a new Scrapy ScrapyRequest (cannot be restored)')
 
-        scrapy_request = Request(
+        scrapy_request = ScrapyRequest(
             url=apify_request.url,
             method=apify_request.method,
             meta={
@@ -173,5 +146,5 @@ def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
     if apify_request.user_data:
         scrapy_request.meta['userData'] = apify_request.user_data
 
-    Actor.log.debug(f'[{call_id}]: an apify_request was converted to the scrapy_request={scrapy_request}')
+    logger.debug(f'an apify_request was converted to the scrapy_request={scrapy_request}')
     return scrapy_request