apify 2.7.1b6__py3-none-any.whl → 2.7.1b8__py3-none-any.whl

apify/_actor.py

@@ -13,7 +13,6 @@ from pydantic import AliasChoices
 
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
-from apify_shared.utils import maybe_extract_enum_member_value
 from crawlee import service_locator
 from crawlee.events import (
     Event,
@@ -30,11 +29,11 @@ from apify._configuration import Configuration
 from apify._consts import EVENT_LISTENERS_TIMEOUT
 from apify._crypto import decrypt_input_secrets, load_private_key
 from apify._models import ActorRun
-from apify._platform_event_manager import EventManager, LocalEventManager, PlatformEventManager
 from apify._proxy_configuration import ProxyConfiguration
-from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython
-from apify.apify_storage_client import ApifyStorageClient
+from apify._utils import docs_group, docs_name, get_system_info, is_running_in_ipython, maybe_extract_enum_member_value
+from apify.events import ApifyEventManager, EventManager, LocalEventManager
 from apify.log import _configure_logging, logger
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
@@ -126,12 +125,12 @@ class _ActorType:
 
         # Create an instance of the cloud storage client, the local storage client is obtained
         # from the service locator.
-        self._cloud_storage_client = ApifyStorageClient.from_config(config=self._configuration)
+        self._cloud_storage_client = ApifyStorageClient()
 
         # Set the event manager based on whether the Actor is running on the platform or locally.
         self._event_manager = (
-            PlatformEventManager(
-                config=self._configuration,
+            ApifyEventManager(
+                configuration=self._configuration,
                 persist_state_interval=self._configuration.persist_state_interval,
             )
             if self.is_at_home()

apify/_configuration.py

@@ -140,6 +140,39 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    default_dataset_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_dataset_id',
+                'apify_default_dataset_id',
+            ),
+            description='Default dataset ID used by the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
+    default_key_value_store_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_key_value_store_id',
+                'apify_default_key_value_store_id',
+            ),
+            description='Default key-value store ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
+    default_request_queue_id: Annotated[
+        str,
+        Field(
+            validation_alias=AliasChoices(
+                'actor_default_request_queue_id',
+                'apify_default_request_queue_id',
+            ),
+            description='Default request queue ID for the Apify storage client when no ID or name is provided.',
+        ),
+    ] = 'default'
+
     disable_outdated_warning: Annotated[
         bool,
         Field(
@@ -334,6 +367,15 @@ class Configuration(CrawleeConfiguration):
         ),
     ] = None
 
+    user_is_paying: Annotated[
+        bool,
+        Field(
+            alias='apify_user_is_paying',
+            description='True if the user calling the Actor is paying user',
+        ),
+        BeforeValidator(lambda val: False if val == '' else val),
+    ] = False
+
     web_server_port: Annotated[
         int,
         Field(

apify/_proxy_configuration.py

@@ -1,13 +1,15 @@
 from __future__ import annotations
 
 import ipaddress
+import json
 import re
 from dataclasses import dataclass, field
 from re import Pattern
 from typing import TYPE_CHECKING, Any
 from urllib.parse import urljoin, urlparse
 
-import httpx
+import impit
+from yarl import URL
 
 from apify_shared.consts import ApifyEnvVars
 from crawlee.proxy_configuration import ProxyConfiguration as CrawleeProxyConfiguration
@@ -20,7 +22,8 @@ from apify.log import logger
 
 if TYPE_CHECKING:
     from apify_client import ApifyClientAsync
-    from crawlee import Request
+
+    from apify import Request
 
 APIFY_PROXY_VALUE_REGEX = re.compile(r'^[\w._~]+$')
 COUNTRY_CODE_REGEX = re.compile(r'^[A-Z]{2}$')
@@ -230,7 +233,7 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
             return None
 
         if self._uses_apify_proxy:
-            parsed_url = httpx.URL(proxy_info.url)
+            parsed_url = URL(proxy_info.url)
             username = self._get_username(session_id)
 
             return ProxyInfo(
@@ -274,11 +277,11 @@ class ProxyConfiguration(CrawleeProxyConfiguration):
             return
 
         status = None
-        async with httpx.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
+        async with impit.AsyncClient(proxy=proxy_info.url, timeout=10) as client:
             for _ in range(2):
                 try:
                     response = await client.get(proxy_status_url)
-                    status = response.json()
+                    status = json.loads(response.text)
                     break
                 except Exception:  # noqa: S110
                     # retry on connection errors

apify/_utils.py

@@ -2,8 +2,9 @@ from __future__ import annotations
 
 import builtins
 import sys
+from enum import Enum
 from importlib import metadata
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, Any, Literal
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -81,3 +82,10 @@ def docs_name(symbol_name: str) -> Callable:  # noqa: ARG001
         return func
 
     return wrapper
+
+
+def maybe_extract_enum_member_value(maybe_enum_member: Any) -> Any:
+    """Extract the value of an enumeration member if it is an Enum, otherwise return the original value."""
+    if isinstance(maybe_enum_member, Enum):
+        return maybe_enum_member.value
+    return maybe_enum_member

apify/events/__init__.py

@@ -0,0 +1,5 @@
+from crawlee.events import EventManager, LocalEventManager
+
+from ._apify_event_manager import ApifyEventManager
+
+__all__ = ['ApifyEventManager', 'EventManager', 'LocalEventManager']

apify/events/_apify_event_manager.py

@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import asyncio
+import contextlib
+from typing import TYPE_CHECKING, Annotated
+
+import websockets.asyncio.client
+from pydantic import Discriminator, TypeAdapter
+from typing_extensions import Self, Unpack, override
+
+from crawlee.events import EventManager
+from crawlee.events._types import Event, EventPersistStateData
+
+from apify._utils import docs_group
+from apify.events._types import DeprecatedEvent, EventMessage, SystemInfoEventData, UnknownEvent
+from apify.log import logger
+
+if TYPE_CHECKING:
+    from types import TracebackType
+
+    from crawlee.events._event_manager import EventManagerOptions
+
+    from apify._configuration import Configuration
+
+
+event_data_adapter = TypeAdapter[EventMessage | DeprecatedEvent | UnknownEvent](
+    Annotated[EventMessage, Discriminator('name')] | DeprecatedEvent | UnknownEvent
+)
+
+
+@docs_group('Event managers')
+class ApifyEventManager(EventManager):
+    """Event manager for the Apify platform.
+
+    This class extends Crawlee's `EventManager` to provide Apify-specific functionality, including websocket
+    connectivity to the Apify platform for receiving platform events.
+
+    The event manager handles:
+    - Registration and emission of events and their listeners.
+    - Websocket connection to Apify platform events.
+    - Processing and validation of platform messages.
+    - Automatic event forwarding from the platform to local event listeners.
+
+    This class should not be used directly. Use the `Actor.on` and `Actor.off` methods to interact
+    with the event system.
+    """
+
+    def __init__(self, configuration: Configuration, **kwargs: Unpack[EventManagerOptions]) -> None:
+        """Initialize a new instance.
+
+        Args:
+            configuration: The Actor configuration for the event manager.
+            **kwargs: Additional event manager options passed to the parent class.
+        """
+        super().__init__(**kwargs)
+
+        self._configuration = configuration
+        """The Actor configuration for the event manager."""
+
+        self._platform_events_websocket: websockets.asyncio.client.ClientConnection | None = None
+        """WebSocket connection to the platform events."""
+
+        self._process_platform_messages_task: asyncio.Task | None = None
+        """Task for processing messages from the platform websocket."""
+
+        self._connected_to_platform_websocket: asyncio.Future[bool] | None = None
+        """Future that resolves when the connection to the platform websocket is established."""
+
+    @override
+    async def __aenter__(self) -> Self:
+        await super().__aenter__()
+        self._connected_to_platform_websocket = asyncio.Future()
+
+        # Run tasks but don't await them
+        if self._configuration.actor_events_ws_url:
+            self._process_platform_messages_task = asyncio.create_task(
+                self._process_platform_messages(self._configuration.actor_events_ws_url)
+            )
+            is_connected = await self._connected_to_platform_websocket
+            if not is_connected:
+                raise RuntimeError('Error connecting to platform events websocket!')
+        else:
+            logger.debug('APIFY_ACTOR_EVENTS_WS_URL env var not set, no events from Apify platform will be emitted.')
+
+        return self
+
+    @override
+    async def __aexit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc_value: BaseException | None,
+        exc_traceback: TracebackType | None,
+    ) -> None:
+        if self._platform_events_websocket:
+            await self._platform_events_websocket.close()
+
+        if self._process_platform_messages_task and not self._process_platform_messages_task.done():
+            self._process_platform_messages_task.cancel()
+            with contextlib.suppress(asyncio.CancelledError):
+                await self._process_platform_messages_task
+
+        await super().__aexit__(exc_type, exc_value, exc_traceback)
+
+    async def _process_platform_messages(self, ws_url: str) -> None:
+        try:
+            async with websockets.asyncio.client.connect(ws_url) as websocket:
+                self._platform_events_websocket = websocket
+                if self._connected_to_platform_websocket is not None:
+                    self._connected_to_platform_websocket.set_result(True)
+
+                async for message in websocket:
+                    try:
+                        parsed_message = event_data_adapter.validate_json(message)
+
+                        if isinstance(parsed_message, DeprecatedEvent):
+                            continue
+
+                        if isinstance(parsed_message, UnknownEvent):
+                            logger.info(
+                                f'Unknown message received: event_name={parsed_message.name}, '
+                                f'event_data={parsed_message.data}'
+                            )
+                            continue
+
+                        self.emit(
+                            event=parsed_message.name,
+                            event_data=parsed_message.data
+                            if not isinstance(parsed_message.data, SystemInfoEventData)
+                            else parsed_message.data.to_crawlee_format(self._configuration.dedicated_cpus or 1),
+                        )
+
+                        if parsed_message.name == Event.MIGRATING:
+                            await self._emit_persist_state_event_rec_task.stop()
+                            self.emit(event=Event.PERSIST_STATE, event_data=EventPersistStateData(is_migrating=True))
+                    except Exception:
+                        logger.exception('Cannot parse Actor event', extra={'message': message})
+        except Exception:
+            logger.exception('Error in websocket connection')
+            if self._connected_to_platform_websocket is not None:
+                self._connected_to_platform_websocket.set_result(False)

apify/events/_types.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Literal
+
+from pydantic import BaseModel, Field
+
+from crawlee.events._types import (
+    Event,
+    EventAbortingData,
+    EventExitData,
+    EventMigratingData,
+    EventPersistStateData,
+    EventSystemInfoData,
+)
+
+from apify._utils import docs_group
+
+
+@docs_group('Event data')
+class SystemInfoEventData(BaseModel):
+    mem_avg_bytes: Annotated[float, Field(alias='memAvgBytes')]
+    mem_current_bytes: Annotated[float, Field(alias='memCurrentBytes')]
+    mem_max_bytes: Annotated[float, Field(alias='memMaxBytes')]
+    cpu_avg_usage: Annotated[float, Field(alias='cpuAvgUsage')]
+    cpu_max_usage: Annotated[float, Field(alias='cpuMaxUsage')]
+    cpu_current_usage: Annotated[float, Field(alias='cpuCurrentUsage')]
+    is_cpu_overloaded: Annotated[bool, Field(alias='isCpuOverloaded')]
+    created_at: Annotated[datetime, Field(alias='createdAt')]
+
+    def to_crawlee_format(self, dedicated_cpus: float) -> EventSystemInfoData:
+        return EventSystemInfoData.model_validate(
+            {
+                'cpu_info': {
+                    'used_ratio': (self.cpu_current_usage / 100) / dedicated_cpus,
+                    'created_at': self.created_at,
+                },
+                'memory_info': {
+                    'total_size': self.mem_max_bytes,
+                    'current_size': self.mem_current_bytes,
+                    'created_at': self.created_at,
+                },
+            }
+        )
+
+
+@docs_group('Events')
+class PersistStateEvent(BaseModel):
+    name: Literal[Event.PERSIST_STATE]
+    data: Annotated[EventPersistStateData, Field(default_factory=lambda: EventPersistStateData(is_migrating=False))]
+
+
+@docs_group('Events')
+class SystemInfoEvent(BaseModel):
+    name: Literal[Event.SYSTEM_INFO]
+    data: SystemInfoEventData
+
+
+@docs_group('Events')
+class MigratingEvent(BaseModel):
+    name: Literal[Event.MIGRATING]
+    data: Annotated[EventMigratingData, Field(default_factory=EventMigratingData)]
+
+
+@docs_group('Events')
+class AbortingEvent(BaseModel):
+    name: Literal[Event.ABORTING]
+    data: Annotated[EventAbortingData, Field(default_factory=EventAbortingData)]
+
+
+@docs_group('Events')
+class ExitEvent(BaseModel):
+    name: Literal[Event.EXIT]
+    data: Annotated[EventExitData, Field(default_factory=EventExitData)]
+
+
+@docs_group('Events')
+class EventWithoutData(BaseModel):
+    name: Literal[
+        Event.SESSION_RETIRED,
+        Event.BROWSER_LAUNCHED,
+        Event.BROWSER_RETIRED,
+        Event.BROWSER_CLOSED,
+        Event.PAGE_CREATED,
+        Event.PAGE_CLOSED,
+    ]
+    data: Any = None
+
+
+@docs_group('Events')
+class DeprecatedEvent(BaseModel):
+    name: Literal['cpuInfo']
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+
+
+@docs_group('Events')
+class UnknownEvent(BaseModel):
+    name: str
+    data: Annotated[dict[str, Any], Field(default_factory=dict)]
+
+
+EventMessage = PersistStateEvent | SystemInfoEvent | MigratingEvent | AbortingEvent | ExitEvent | EventWithoutData

apify/log.py

@@ -27,13 +27,6 @@ def _configure_logging() -> None:
     else:
         apify_client_logger.setLevel(level)
 
-    # Silence HTTPX logger unless debug logging is requested
-    httpx_logger = logging.getLogger('httpx')
-    if level > logging.DEBUG:
-        httpx_logger.setLevel(logging.WARNING)
-    else:
-        httpx_logger.setLevel(level)
-
     # Use configured log level for apify logger
     apify_logger = logging.getLogger('apify')
     configure_logger(apify_logger, remove_old_handlers=True)

apify/request_loaders/__init__.py

@@ -0,0 +1,18 @@
+from crawlee.request_loaders import (
+    RequestList,
+    RequestLoader,
+    RequestManager,
+    RequestManagerTandem,
+    SitemapRequestLoader,
+)
+
+from ._apify_request_list import ApifyRequestList
+
+__all__ = [
+    'ApifyRequestList',
+    'RequestList',
+    'RequestLoader',
+    'RequestManager',
+    'RequestManagerTandem',
+    'SitemapRequestLoader',
+]

apify/{storages/_request_list.py → request_loaders/_apify_request_list.py}

@@ -3,16 +3,15 @@ from __future__ import annotations
 import asyncio
 import re
 from asyncio import Task
-from functools import partial
 from typing import Annotated, Any
 
 from pydantic import BaseModel, Field, TypeAdapter
 
-from crawlee import Request
 from crawlee._types import HttpMethod
-from crawlee.http_clients import HttpClient, HttpxHttpClient
-from crawlee.request_loaders import RequestList as CrawleeRequestList
+from crawlee.http_clients import HttpClient, ImpitHttpClient
+from crawlee.request_loaders import RequestList
 
+from apify import Request
 from apify._utils import docs_group
 
 URL_NO_COMMAS_REGEX = re.compile(
@@ -39,7 +38,7 @@ url_input_adapter = TypeAdapter(list[_RequestsFromUrlInput | _SimpleUrlInput])
 
 
 @docs_group('Request loaders')
-class RequestList(CrawleeRequestList):
+class ApifyRequestList(RequestList):
     """Extends crawlee RequestList.
 
     Method open is used to create RequestList from actor's requestListSources input.
@@ -50,7 +49,7 @@ class RequestList(CrawleeRequestList):
         name: str | None = None,
         request_list_sources_input: list[dict[str, Any]] | None = None,
         http_client: HttpClient | None = None,
-    ) -> RequestList:
+    ) -> ApifyRequestList:
         """Initialize a new instance from request list source input.
 
         Args:
@@ -74,24 +73,26 @@ class RequestList(CrawleeRequestList):
         ```
         """
         request_list_sources_input = request_list_sources_input or []
-        return await RequestList._create_request_list(name, request_list_sources_input, http_client)
+        return await ApifyRequestList._create_request_list(name, request_list_sources_input, http_client)
 
     @staticmethod
     async def _create_request_list(
         name: str | None, request_list_sources_input: list[dict[str, Any]], http_client: HttpClient | None
-    ) -> RequestList:
+    ) -> ApifyRequestList:
         if not http_client:
-            http_client = HttpxHttpClient()
+            http_client = ImpitHttpClient()
 
         url_inputs = url_input_adapter.validate_python(request_list_sources_input)
 
         simple_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _SimpleUrlInput)]
         remote_url_inputs = [url_input for url_input in url_inputs if isinstance(url_input, _RequestsFromUrlInput)]
 
-        simple_url_requests = RequestList._create_requests_from_input(simple_url_inputs)
-        remote_url_requests = await RequestList._fetch_requests_from_url(remote_url_inputs, http_client=http_client)
+        simple_url_requests = ApifyRequestList._create_requests_from_input(simple_url_inputs)
+        remote_url_requests = await ApifyRequestList._fetch_requests_from_url(
+            remote_url_inputs, http_client=http_client
+        )
 
-        return RequestList(name=name, requests=simple_url_requests + remote_url_requests)
+        return ApifyRequestList(name=name, requests=simple_url_requests + remote_url_requests)
 
     @staticmethod
     def _create_requests_from_input(simple_url_inputs: list[_SimpleUrlInput]) -> list[Request]:
@@ -119,13 +120,15 @@ class RequestList(CrawleeRequestList):
         """
         created_requests: list[Request] = []
 
-        def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Task) -> None:
+        async def create_requests_from_response(request_input: _RequestsFromUrlInput, task: Task) -> None:
             """Extract links from response body and use them to create `Request` objects.
 
             Use the regular expression to find all matching links in the response body, then create `Request`
             objects from these links and the provided input attributes.
             """
-            matches = re.finditer(URL_NO_COMMAS_REGEX, task.result().read().decode('utf-8'))
+            response = await (task.result()).read()
+            matches = re.finditer(URL_NO_COMMAS_REGEX, response.decode('utf-8'))
+
             created_requests.extend(
                 [
                     Request.from_url(
@@ -148,7 +151,11 @@ class RequestList(CrawleeRequestList):
                 )
             )
 
-            get_response_task.add_done_callback(partial(create_requests_from_response, remote_url_requests_input))
+            get_response_task.add_done_callback(
+                lambda task, inp=remote_url_requests_input: asyncio.create_task(  # type: ignore[misc]
+                    create_requests_from_response(inp, task)
+                )
+            )
             remote_url_requests.append(get_response_task)
 
         await asyncio.gather(*remote_url_requests)

apify/scrapy/_logging_config.py

@@ -10,7 +10,7 @@ from apify.log import ActorLogFormatter
 
 # Define logger names.
 _PRIMARY_LOGGERS = ['apify', 'apify_client', 'scrapy']
-_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'httpx', 'protego', 'twisted']
+_SUPPLEMENTAL_LOGGERS = ['filelock', 'hpack', 'httpcore', 'protego', 'twisted']
 _ALL_LOGGERS = _PRIMARY_LOGGERS + _SUPPLEMENTAL_LOGGERS
 
 
@@ -37,9 +37,6 @@ def initialize_logging() -> None:
     for logger_name in [None, *_ALL_LOGGERS]:
         _configure_logger(logger_name, logging_level, handler)
 
-    # Set the 'httpx' logger to a less verbose level.
-    logging.getLogger('httpx').setLevel('WARNING')
-
     # Monkey-patch Scrapy's logging configuration to re-apply our settings.
     original_configure_logging = scrapy_logging.configure_logging
 

apify/scrapy/extensions/_httpcache.py

@@ -13,8 +13,8 @@ from scrapy.http.headers import Headers
 from scrapy.responsetypes import responsetypes
 
 from apify import Configuration
-from apify.apify_storage_client import ApifyStorageClient
 from apify.scrapy._async_thread import AsyncThread
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import KeyValueStore
 
 if TYPE_CHECKING:
@@ -51,10 +51,14 @@ class ApifyCacheStorage:
         kvs_name = get_kvs_name(spider.name)
 
         async def open_kvs() -> KeyValueStore:
-            config = Configuration.get_global_configuration()
-            if config.is_at_home:
-                storage_client = ApifyStorageClient.from_config(config)
-                return await KeyValueStore.open(name=kvs_name, storage_client=storage_client)
+            configuration = Configuration.get_global_configuration()
+            if configuration.is_at_home:
+                storage_client = ApifyStorageClient()
+                return await KeyValueStore.open(
+                    name=kvs_name,
+                    configuration=configuration,
+                    storage_client=storage_client,
+                )
             return await KeyValueStore.open(name=kvs_name)
 
         logger.debug("Starting background thread for cache storage's event loop")

apify/scrapy/requests.py

@@ -10,9 +10,10 @@ from scrapy import Spider
 from scrapy.http.headers import Headers
 from scrapy.utils.request import request_from_dict
 
-from crawlee import Request as ApifyRequest
 from crawlee._types import HttpHeaders
 
+from apify import Request as ApifyRequest
+
 logger = getLogger(__name__)
 
 
@@ -121,7 +122,7 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
 
         # Update the meta field with the meta field from the apify_request
         meta = scrapy_request.meta or {}
-        meta.update({'apify_request_id': apify_request.id, 'apify_request_unique_key': apify_request.unique_key})
+        meta.update({'apify_request_unique_key': apify_request.unique_key})
         # scrapy_request.meta is a property, so we have to set it like this
         scrapy_request._meta = meta  # noqa: SLF001
 
@@ -133,7 +134,6 @@ def to_scrapy_request(apify_request: ApifyRequest, spider: Spider) -> ScrapyRequ
             url=apify_request.url,
             method=apify_request.method,
             meta={
-                'apify_request_id': apify_request.id,
                 'apify_request_unique_key': apify_request.unique_key,
             },
         )

apify/scrapy/scheduler.py

@@ -11,7 +11,7 @@ from scrapy.utils.reactor import is_asyncio_reactor_installed
 from ._async_thread import AsyncThread
 from .requests import to_apify_request, to_scrapy_request
 from apify import Configuration
-from apify.apify_storage_client import ApifyStorageClient
+from apify.storage_clients import ApifyStorageClient
 from apify.storages import RequestQueue
 
 if TYPE_CHECKING:
@@ -49,10 +49,13 @@ class ApifyScheduler(BaseScheduler):
         self.spider = spider
 
         async def open_rq() -> RequestQueue:
-            config = Configuration.get_global_configuration()
-            if config.is_at_home:
-                storage_client = ApifyStorageClient.from_config(config)
-                return await RequestQueue.open(storage_client=storage_client)
+            configuration = Configuration.get_global_configuration()
+            if configuration.is_at_home:
+                storage_client = ApifyStorageClient()
+                return await RequestQueue.open(
+                    configuration=configuration,
+                    storage_client=storage_client,
+                )
             return await RequestQueue.open()
 
         try:

apify/storage_clients/__init__.py

@@ -0,0 +1,10 @@
+from crawlee.storage_clients import MemoryStorageClient
+
+from ._apify import ApifyStorageClient
+from ._file_system import ApifyFileSystemStorageClient as FileSystemStorageClient
+
+__all__ = [
+    'ApifyStorageClient',
+    'FileSystemStorageClient',
+    'MemoryStorageClient',
+]