apify 1.7.3b3__py3-none-any.whl → 2.0.0__py3-none-any.whl

apify/log.py

@@ -1,16 +1,12 @@
 from __future__ import annotations
 
-import json
 import logging
-import textwrap
-import traceback
-from typing import Any
+from typing import TYPE_CHECKING
 
-from apify_shared.utils import ignore_docs
-from colorama import Fore, Style, just_fix_windows_console
-
-just_fix_windows_console()
+from crawlee._log_config import CrawleeLogFormatter, configure_logger, get_configured_log_level
 
+if TYPE_CHECKING:
+    from apify import Configuration
 
 # Name of the logger used throughout the library (resolves to 'apify')
 logger_name = __name__.split('.')[0]
@@ -18,107 +14,30 @@ logger_name = __name__.split('.')[0]
 # Logger used throughout the library
 logger = logging.getLogger(logger_name)
 
-_LOG_NAME_COLOR = Fore.LIGHTBLACK_EX
-
-_LOG_LEVEL_COLOR = {
-    logging.DEBUG: Fore.BLUE,
-    logging.INFO: Fore.GREEN,
-    logging.WARNING: Fore.YELLOW,
-    logging.ERROR: Fore.RED,
-    logging.CRITICAL: Fore.RED,
-}
-
-_LOG_LEVEL_SHORT_ALIAS = {
-    logging.DEBUG: 'DEBUG',
-    logging.INFO: 'INFO ',
-    logging.WARNING: 'WARN ',
-    logging.ERROR: 'ERROR',
-}
-
-# So that all the log messages have the same alignment
-_LOG_MESSAGE_INDENT = ' ' * 6
-
-
-class ActorLogFormatter(logging.Formatter):
-    """Log formatter that prints out the log message nicely formatted, with colored level and stringified extra fields.
-
-    It formats the log records so that they:
-    - start with the level (colorized, and padded to 5 chars so that it is nicely aligned)
-    - then have the actual log message, if it's multiline then it's nicely indented
-    - then have the stringified extra log fields
-    - then, if an exception is a part of the log record, prints the formatted exception.
-    """
-
-    # The fields that are added to the log record with `logger.log(..., extra={...})`
-    # are just merged in the log record with the other log record properties, and you can't get them in some nice, isolated way.
-    # So, to get the extra fields, we just compare all the properties present in the log record
-    # with properties present in an empty log record,
-    # and extract all the extra ones not present in the empty log record
-    empty_record = logging.LogRecord('dummy', 0, 'dummy', 0, 'dummy', None, None)
-
-    def __init__(
-        self: ActorLogFormatter,
-        include_logger_name: bool = False,  # noqa: FBT001, FBT002
-        *args: Any,
-        **kwargs: Any,
-    ) -> None:
-        """Create an instance of the ActorLogFormatter.
-
-        Args:
-            include_logger_name: Include logger name at the beginning of the log line. Defaults to False.
-            args: Arguments passed to the parent class.
-            kwargs: Keyword arguments passed to the parent class.
-        """
-        super().__init__(*args, **kwargs)
-        self.include_logger_name = include_logger_name
-
-    def _get_extra_fields(self: ActorLogFormatter, record: logging.LogRecord) -> dict[str, Any]:
-        extra_fields: dict[str, Any] = {}
-        for key, value in record.__dict__.items():
-            if key not in self.empty_record.__dict__:
-                extra_fields[key] = value  # noqa: PERF403
-
-        return extra_fields
-
-    @ignore_docs
-    def format(self: ActorLogFormatter, record: logging.LogRecord) -> str:
-        """Format the log record nicely.
 
-        This formats the log record so that it:
-        - starts with the level (colorized, and padded to 5 chars so that it is nicely aligned)
-        - then has the actual log message, if it's multiline then it's nicely indented
-        - then has the stringified extra log fields
-        - then, if an exception is a part of the log record, prints the formatted exception.
-        """
-        logger_name_string = f'{_LOG_NAME_COLOR}[{record.name}]{Style.RESET_ALL} '
+class ActorLogFormatter(CrawleeLogFormatter):  # noqa: D101 Inherited from parent class
+    pass
 
-        # Colorize the log level, and shorten it to 6 chars tops
-        level_color_code = _LOG_LEVEL_COLOR.get(record.levelno, '')
-        level_short_alias = _LOG_LEVEL_SHORT_ALIAS.get(record.levelno, record.levelname)
-        level_string = f'{level_color_code}{level_short_alias}{Style.RESET_ALL} '
 
-        # Format the exception, if there is some
-        # Basically just print the traceback and indent it a bit
-        exception_string = ''
-        if record.exc_info:
-            exc_info = record.exc_info
-            record.exc_info = None
-            exception_string = ''.join(traceback.format_exception(*exc_info)).rstrip()
-            exception_string = '\n' + textwrap.indent(exception_string, _LOG_MESSAGE_INDENT)
+def _configure_logging(configuration: Configuration) -> None:
+    apify_client_logger = logging.getLogger('apify_client')
+    configure_logger(apify_client_logger, configuration, remove_old_handlers=True)
 
-        # Format the extra log record fields, if there were some
-        # Just stringify them to JSON and color them gray
-        extra_string = ''
-        extra = self._get_extra_fields(record)
-        if extra:
-            extra_string = f' {Fore.LIGHTBLACK_EX}({json.dumps(extra, ensure_ascii=False, default=str)}){Style.RESET_ALL}'
+    level = get_configured_log_level(configuration)
 
-        # Format the actual log message, and indent everything but the first line
-        log_string = super().format(record)
-        log_string = textwrap.indent(log_string, _LOG_MESSAGE_INDENT).lstrip()
+    # Keep apify_client logger quiet unless debug logging is requested
+    if level > logging.DEBUG:
+        apify_client_logger.setLevel(logging.INFO)
+    else:
+        apify_client_logger.setLevel(level)
 
-        if self.include_logger_name:
-            # Include logger name at the beginning of the log line
-            return f'{logger_name_string}{level_string}{log_string}{extra_string}{exception_string}'
+    # Silence HTTPX logger unless debug logging is requested
+    httpx_logger = logging.getLogger('httpx')
+    if level > logging.DEBUG:
+        httpx_logger.setLevel(logging.WARNING)
+    else:
+        httpx_logger.setLevel(level)
 
-        return f'{level_string}{log_string}{extra_string}{exception_string}'
+    # Use configured log level for apify logger
+    apify_logger = logging.getLogger('apify')
+    configure_logger(apify_logger, configuration, remove_old_handlers=True)

apify/scrapy/__init__.py

@@ -1,3 +1,11 @@
-from .requests import to_apify_request, to_scrapy_request
-from .scheduler import ApifyScheduler
-from .utils import get_basic_auth_header, get_running_event_loop_id, open_queue_with_custom_client
+from apify.scrapy.requests import to_apify_request, to_scrapy_request
+from apify.scrapy.scheduler import ApifyScheduler
+from apify.scrapy.utils import get_basic_auth_header, get_running_event_loop_id
+
+__all__ = [
+    'to_apify_request',
+    'to_scrapy_request',
+    'ApifyScheduler',
+    'get_basic_auth_header',
+    'get_running_event_loop_id',
+]

apify/scrapy/middlewares/__init__.py

@@ -1 +1,3 @@
-from .apify_proxy import ApifyHttpProxyMiddleware
+from apify.scrapy.middlewares.apify_proxy import ApifyHttpProxyMiddleware
+
+__all__ = ['ApifyHttpProxyMiddleware']

apify/scrapy/middlewares/apify_proxy.py

@@ -12,8 +12,7 @@ except ImportError as exc:
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
     ) from exc
 
-from apify.actor import Actor
-from apify.proxy_configuration import ProxyConfiguration
+from apify import Actor, ProxyConfiguration
 from apify.scrapy.utils import get_basic_auth_header
 
 
@@ -43,23 +42,25 @@ class ApifyHttpProxyMiddleware:
         """Create an instance of ApifyHttpProxyMiddleware from a Scrapy Crawler.
 
         Args:
-            cls: Class type.
             crawler: Scrapy Crawler object.
 
-        Returns:
-            ApifyHttpProxyMiddleware: Instance of the class.
+        Returns: Instance of the class.
         """
         proxy_settings: dict | None = crawler.settings.get('APIFY_PROXY_SETTINGS')
 
         if proxy_settings is None:
-            Actor.log.warning('ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing in the Actor input.')
+            Actor.log.warning(
+                'ApifyHttpProxyMiddleware is not going to be used. Object "proxyConfiguration" is probably missing '
+                ' in the Actor input.'
+            )
             raise NotConfigured
 
         use_apify_proxy = proxy_settings.get('useApifyProxy', False)
 
         if use_apify_proxy is not True:
             Actor.log.warning(
-                'ApifyHttpProxyMiddleware is not going to be used. Actor input field "proxyConfiguration.useApifyProxy" is probably set to False.'
+                'ApifyHttpProxyMiddleware is not going to be used. Actor input field '
+                '"proxyConfiguration.useApifyProxy" is probably set to False.'
             )
             raise NotConfigured
 
@@ -74,9 +75,6 @@ class ApifyHttpProxyMiddleware:
 
         Raises:
             ValueError: If username and password are not provided in the proxy URL.
-
-        Returns:
-            None: The request is processed and middleware pipeline can continue.
         """
         Actor.log.debug(f'ApifyHttpProxyMiddleware.process_request: request={request}, spider={spider}')
         url = await self._get_new_proxy_url()
@@ -95,7 +93,7 @@ class ApifyHttpProxyMiddleware:
         request: Request,
         exception: Exception,
         spider: Spider,
-    ) -> None | Request:
+    ) -> None:
         """Process an exception that occurs during request processing.
 
         Args:
@@ -104,18 +102,19 @@ class ApifyHttpProxyMiddleware:
             spider: Scrapy Spider object.
 
         Returns:
-            If a TunnelError occurs, return the request object to halt its processing in the middleware pipeline.
-            Return None otherwise to allow the continuation of request processing.
+            Returning None, meaning Scrapy will continue processing this exception, executing any other
+            process_exception() methods of installed middleware, until no middleware is left and the default
+            exception handling kicks in.
         """
         Actor.log.debug(
             f'ApifyHttpProxyMiddleware.process_exception: request={request}, exception={exception}, spider={spider}',
         )
 
         if isinstance(exception, TunnelError):
-            Actor.log.warning(f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", reason="{exception}", skipping...')
-            return request
-
-        return None
+            Actor.log.warning(
+                f'ApifyHttpProxyMiddleware: TunnelError occurred for request="{request}", '
+                'reason="{exception}", skipping...'
+            )
 
     async def _get_new_proxy_url(self: ApifyHttpProxyMiddleware) -> ParseResult:
         """Get a new proxy URL.
@@ -123,8 +122,7 @@ class ApifyHttpProxyMiddleware:
         Raises:
             NotConfigured: If creation of the proxy configuration fails.
 
-        Returns:
-            ParseResult: New proxy URL.
+        Returns: New proxy URL.
         """
         # Get proxy configuration, creating it if necessary
         proxy_cfg = (
@@ -136,7 +134,9 @@ class ApifyHttpProxyMiddleware:
         # If the proxy configuration is still not available, raise an error. However, this should not happen due
         # to the checks in the `from_crawler` method.
         if proxy_cfg is None:
-            Actor.log.error('Creation of proxy configuration failed. Check the field "proxyConfiguration" in the Actor input.')
+            Actor.log.error(
+                'Creation of proxy configuration failed. Check the field "proxyConfiguration" in the Actor input.'
+            )
             raise NotConfigured
 
         # Store the proxy configuration for future use
@@ -144,4 +144,4 @@ class ApifyHttpProxyMiddleware:
 
         # Get a new proxy URL and return it
         new_url = await proxy_cfg.new_url()
-        return urlparse(new_url)
+        return urlparse(str(new_url))

apify/scrapy/pipelines/__init__.py

@@ -1 +1,3 @@
-from .actor_dataset_push import ActorDatasetPushPipeline
+from apify.scrapy.pipelines.actor_dataset_push import ActorDatasetPushPipeline
+
+__all__ = ['ActorDatasetPushPipeline']

apify/scrapy/pipelines/actor_dataset_push.py

@@ -9,7 +9,7 @@ except ImportError as exc:
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
     ) from exc
 
-from apify.actor import Actor
+from apify import Actor
 
 
 class ActorDatasetPushPipeline:

apify/scrapy/requests.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import codecs
 import pickle
+from typing import Any, cast
 
 try:
     from scrapy import Request, Spider
@@ -12,9 +13,11 @@ except ImportError as exc:
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
     ) from exc
 
-from apify._crypto import crypto_random_object_id
-from apify._utils import compute_unique_key
-from apify.actor import Actor
+from crawlee import Request as CrawleeRequest
+from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.requests import compute_unique_key, unique_key_to_request_id
+
+from apify import Actor
 
 
 def _is_request_produced_by_middleware(scrapy_request: Request) -> bool:
@@ -25,7 +28,7 @@ def _is_request_produced_by_middleware(scrapy_request: Request) -> bool:
     return bool(scrapy_request.meta.get('redirect_times')) or bool(scrapy_request.meta.get('retry_times'))
 
 
-def to_apify_request(scrapy_request: Request, spider: Spider) -> dict | None:
+def to_apify_request(scrapy_request: Request, spider: Spider) -> CrawleeRequest | None:
     """Convert a Scrapy request to an Apify request.
 
     Args:
@@ -35,7 +38,7 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> dict | None:
     Returns:
         The converted Apify request if the conversion was successful, otherwise None.
     """
-    if not isinstance(scrapy_request, Request):
+    if not isinstance(cast(Any, scrapy_request), Request):
         Actor.log.warning('Failed to convert to Apify request: Scrapy request must be a Request instance.')
         return None
 
@@ -43,39 +46,41 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> dict | None:
     Actor.log.debug(f'[{call_id}]: to_apify_request was called (scrapy_request={scrapy_request})...')
 
     try:
-        apify_request = {
-            'url': scrapy_request.url,
-            'method': scrapy_request.method,
-            'payload': scrapy_request.body,
-            'userData': scrapy_request.meta.get('userData', {}),
-        }
-
-        # Convert Scrapy's headers to a dictionary and store them in the apify_request
-        if isinstance(scrapy_request.headers, Headers):
-            apify_request['headers'] = dict(scrapy_request.headers.to_unicode_dict())
-        else:
-            Actor.log.warning(f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}')
-
-        # If the request was produced by the middleware (e.g. retry or redirect), we must compute the unique key here
         if _is_request_produced_by_middleware(scrapy_request):
-            apify_request['uniqueKey'] = compute_unique_key(
+            unique_key = compute_unique_key(
                 url=scrapy_request.url,
                 method=scrapy_request.method,
                 payload=scrapy_request.body,
                 use_extended_unique_key=True,
             )
-        # Othwerwise, we can use the unique key (also the id) from the meta
+        elif scrapy_request.dont_filter:
+            unique_key = crypto_random_object_id(8)
+        elif scrapy_request.meta.get('apify_request_unique_key'):
+            unique_key = scrapy_request.meta['apify_request_unique_key']
         else:
-            if scrapy_request.meta.get('apify_request_id'):
-                apify_request['id'] = scrapy_request.meta['apify_request_id']
+            unique_key = crypto_random_object_id(8)
 
-            if scrapy_request.meta.get('apify_request_unique_key'):
-                apify_request['uniqueKey'] = scrapy_request.meta['apify_request_unique_key']
+        if scrapy_request.meta.get('apify_request_id'):
+            request_id = scrapy_request.meta['apify_request_id']
+        else:
+            request_id = unique_key_to_request_id(unique_key)
+
+        apify_request = CrawleeRequest(
+            url=scrapy_request.url,
+            method=scrapy_request.method,
+            payload=scrapy_request.body,
+            user_data=scrapy_request.meta.get('userData', {}),
+            unique_key=unique_key,
+            id=request_id,
+        )
 
-        # If the request's dont_filter field is set, we must generate a random `uniqueKey` to avoid deduplication
-        # of the request in the Request Queue.
-        if scrapy_request.dont_filter:
-            apify_request['uniqueKey'] = crypto_random_object_id(8)
+        # Convert Scrapy's headers to a dictionary and store them in the apify_request
+        if isinstance(scrapy_request.headers, Headers):
+            apify_request.headers = dict(scrapy_request.headers.to_unicode_dict())
+        else:
+            Actor.log.warning(
+                f'Invalid scrapy_request.headers type, not scrapy.http.headers.Headers: {scrapy_request.headers}'
+            )
 
         # Serialize the Scrapy Request and store it in the apify_request.
         #   - This process involves converting the Scrapy Request object into a dictionary, encoding it to base64,
@@ -83,7 +88,7 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> dict | None:
         #   - The serialization process can be referenced at: https://stackoverflow.com/questions/30469575/.
         scrapy_request_dict = scrapy_request.to_dict(spider=spider)
         scrapy_request_dict_encoded = codecs.encode(pickle.dumps(scrapy_request_dict), 'base64').decode()
-        apify_request['userData']['scrapy_request'] = scrapy_request_dict_encoded
+        apify_request.user_data['scrapy_request'] = scrapy_request_dict_encoded
 
     except Exception as exc:
         Actor.log.warning(f'Conversion of Scrapy request {scrapy_request} to Apify request failed; {exc}')
@@ -93,7 +98,7 @@ def to_apify_request(scrapy_request: Request, spider: Spider) -> dict | None:
     return apify_request
 
 
-def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
+def to_scrapy_request(apify_request: CrawleeRequest, spider: Spider) -> Request:
     """Convert an Apify request to a Scrapy request.
 
     Args:
@@ -101,32 +106,26 @@ def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
         spider: The Scrapy spider that the request is associated with.
 
     Raises:
-        TypeError: If the apify_request is not a dictionary.
+        TypeError: If the apify_request is not a crawlee request.
         ValueError: If the apify_request does not contain the required keys.
 
     Returns:
         The converted Scrapy request.
     """
-    if not isinstance(apify_request, dict):
-        raise TypeError('apify_request must be a dictionary')
-
-    required_keys = ['url', 'method', 'id', 'uniqueKey']
-    missing_keys = [key for key in required_keys if key not in apify_request]
-
-    if missing_keys:
-        raise ValueError(f'apify_request must contain {", ".join(map(repr, missing_keys))} key(s)')
+    if not isinstance(cast(Any, apify_request), CrawleeRequest):
+        raise TypeError('apify_request must be a crawlee.Request instance')
 
     call_id = crypto_random_object_id(8)
     Actor.log.debug(f'[{call_id}]: to_scrapy_request was called (apify_request={apify_request})...')
 
     # If the apify_request comes from the Scrapy
-    if 'userData' in apify_request and 'scrapy_request' in apify_request['userData']:
+    if 'scrapy_request' in apify_request.user_data:
         # Deserialize the Scrapy Request from the apify_request.
         #   - This process involves decoding the base64-encoded request data and reconstructing
         #     the Scrapy Request object from its dictionary representation.
         Actor.log.debug(f'[{call_id}]: Restoring the Scrapy Request from the apify_request...')
 
-        scrapy_request_dict_encoded = apify_request['userData']['scrapy_request']
+        scrapy_request_dict_encoded = apify_request.user_data['scrapy_request']
         if not isinstance(scrapy_request_dict_encoded, str):
             raise TypeError('scrapy_request_dict_encoded must be a string')
 
@@ -142,34 +141,36 @@ def to_scrapy_request(apify_request: dict, spider: Spider) -> Request:
 
         # Update the meta field with the meta field from the apify_request
         meta = scrapy_request.meta or {}
-        meta.update({'apify_request_id': apify_request['id'], 'apify_request_unique_key': apify_request['uniqueKey']})
-        scrapy_request._meta = meta  # scrapy_request.meta is a property, so we have to set it like this
+        meta.update({'apify_request_id': apify_request.id, 'apify_request_unique_key': apify_request.unique_key})
+        # scrapy_request.meta is a property, so we have to set it like this
+        scrapy_request._meta = meta  # noqa: SLF001
 
     # If the apify_request comes directly from the Request Queue, typically start URLs
     else:
         Actor.log.debug(f'[{call_id}]: gonna create a new Scrapy Request (cannot be restored)')
 
         scrapy_request = Request(
-            url=apify_request['url'],
-            method=apify_request['method'],
+            url=apify_request.url,
+            method=apify_request.method,
             meta={
-                'apify_request_id': apify_request['id'],
-                'apify_request_unique_key': apify_request['uniqueKey'],
+                'apify_request_id': apify_request.id,
+                'apify_request_unique_key': apify_request.unique_key,
             },
         )
 
     # Add optional 'headers' field
-    if 'headers' in apify_request:
-        if isinstance(apify_request['headers'], dict):
-            scrapy_request.headers = Headers(apify_request['headers'])
+    if apify_request.headers:
+        if isinstance(cast(Any, apify_request.headers), dict):
+            scrapy_request.headers = Headers(apify_request.headers)
         else:
             Actor.log.warning(
-                f'apify_request[headers] is not an instance of the dict class, apify_request[headers] = {apify_request["headers"]}',
+                'apify_request[headers] is not an instance of the dict class, '
+                f'apify_request[headers] = {apify_request.headers}',
             )
 
     # Add optional 'userData' field
-    if 'userData' in apify_request:
-        scrapy_request.meta['userData'] = apify_request['userData']
+    if apify_request.user_data:
+        scrapy_request.meta['userData'] = apify_request.user_data
 
     Actor.log.debug(f'[{call_id}]: an apify_request was converted to the scrapy_request={scrapy_request}')
     return scrapy_request

apify/scrapy/scheduler.py

@@ -2,6 +2,9 @@ from __future__ import annotations
 
 import traceback
 
+from apify._configuration import Configuration
+from apify.apify_storage_client import ApifyStorageClient
+
 try:
     from scrapy import Spider
     from scrapy.core.scheduler import BaseScheduler
@@ -12,10 +15,11 @@ except ImportError as exc:
         'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
     ) from exc
 
-from apify._crypto import crypto_random_object_id
-from apify.actor import Actor
+from crawlee._utils.crypto import crypto_random_object_id
+
+from apify import Actor
 from apify.scrapy.requests import to_apify_request, to_scrapy_request
-from apify.scrapy.utils import nested_event_loop, open_queue_with_custom_client
+from apify.scrapy.utils import nested_event_loop
 from apify.storages import RequestQueue
 
 
@@ -44,8 +48,12 @@ class ApifyScheduler(BaseScheduler):
         """
         self.spider = spider
 
+        async def open_queue() -> RequestQueue:
+            custom_loop_apify_client = ApifyStorageClient(configuration=Configuration.get_global_configuration())
+            return await RequestQueue.open(storage_client=custom_loop_apify_client)
+
         try:
-            self._rq = nested_event_loop.run_until_complete(open_queue_with_custom_client())
+            self._rq = nested_event_loop.run_until_complete(open_queue())
         except BaseException:
             traceback.print_exc()
             raise
@@ -95,18 +103,13 @@ class ApifyScheduler(BaseScheduler):
             raise TypeError('self._rq must be an instance of the RequestQueue class')
 
         try:
-            result = nested_event_loop.run_until_complete(
-                self._rq.add_request(
-                    apify_request,
-                    use_extended_unique_key=True,
-                )
-            )
+            result = nested_event_loop.run_until_complete(self._rq.add_request(apify_request))
         except BaseException:
             traceback.print_exc()
             raise
 
         Actor.log.debug(f'[{call_id}]: rq.add_request.result={result}...')
-        return bool(result['wasAlreadyPresent'])
+        return bool(result.was_already_present)
 
     def next_request(self: ApifyScheduler) -> Request | None:
         """Fetch the next request from the scheduler.
@@ -127,7 +130,9 @@ class ApifyScheduler(BaseScheduler):
             traceback.print_exc()
             raise
 
-        Actor.log.debug(f'[{call_id}]: a new apify_request from the scheduler was fetched (apify_request={apify_request})')
+        Actor.log.debug(
+            f'[{call_id}]: a new apify_request from the scheduler was fetched (apify_request={apify_request})'
+        )
 
         if apify_request is None:
             return None
@@ -145,6 +150,7 @@ class ApifyScheduler(BaseScheduler):
 
         scrapy_request = to_scrapy_request(apify_request, spider=self.spider)
         Actor.log.debug(
-            f'[{call_id}]: apify_request was transformed to the scrapy_request which is gonna be returned (scrapy_request={scrapy_request})',
+            f'[{call_id}]: apify_request was transformed to the scrapy_request which is gonna be returned '
+            f'(scrapy_request={scrapy_request})',
         )
         return scrapy_request

apify/scrapy/utils.py

@@ -10,11 +10,10 @@ try:
     from scrapy.utils.python import to_bytes
 except ImportError as exc:
     raise ImportError(
-        'To use this module, you need to install the "scrapy" extra. Run "pip install apify[scrapy]".',
+        'To use this module, you need to install the "scrapy" extra. For example, if you use pip, run '
+        '"pip install apify[scrapy]".'
     ) from exc
 
-from apify.actor import Actor
-from apify.storages import RequestQueue, StorageClientManager
 
 nested_event_loop: asyncio.AbstractEventLoop = asyncio.new_event_loop()
 
@@ -71,31 +70,3 @@ def apply_apify_settings(*, settings: Settings | None = None, proxy_config: dict
     settings['APIFY_PROXY_SETTINGS'] = proxy_config
 
     return settings
-
-
-async def open_queue_with_custom_client() -> RequestQueue:
-    """Open a Request Queue with custom Apify Client.
-
-    TODO: add support for custom client to Actor.open_request_queue(), so that
-    we don't have to do this hacky workaround
-    """
-    # Create a new Apify Client with its httpx client in the custom event loop
-    custom_loop_apify_client = Actor.new_client()
-
-    # Set the new Apify Client as the default client, back up the old client
-    old_client = Actor.apify_client
-    StorageClientManager.set_cloud_client(custom_loop_apify_client)
-
-    # Create a new Request Queue in the custom event loop,
-    # replace its Apify client with the custom loop's Apify client
-    rq = await Actor.open_request_queue()
-
-    if Actor.config.is_at_home:
-        rq._request_queue_client = custom_loop_apify_client.request_queue(
-            rq._id,
-            client_key=rq._client_key,
-        )
-
-    # Restore the old Apify Client as the default client
-    StorageClientManager.set_cloud_client(old_client)
-    return rq

apify/storages/__init__.py

@@ -1,11 +1,3 @@
-from .dataset import Dataset
-from .key_value_store import KeyValueStore
-from .request_queue import RequestQueue
-from .storage_client_manager import StorageClientManager
+from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
-__all__ = [
-    'Dataset',
-    'KeyValueStore',
-    'RequestQueue',
-    'StorageClientManager',
-]
+__all__ = ['Dataset', 'KeyValueStore', 'RequestQueue']