apify 2.7.1b19__py3-none-any.whl → 2.7.1b21__py3-none-any.whl

apify/_actor.py

@@ -25,7 +25,6 @@ from crawlee.events import (
     EventPersistStateData,
     EventSystemInfoData,
 )
-from crawlee.storage_clients import FileSystemStorageClient
 
 from apify._charging import ChargeResult, ChargingManager, ChargingManagerImplementation
 from apify._configuration import Configuration
@@ -38,6 +37,7 @@ from apify.events import ApifyEventManager, EventManager, LocalEventManager
 from apify.log import _configure_logging, logger
 from apify.storage_clients import ApifyStorageClient
 from apify.storage_clients._file_system import ApifyFileSystemStorageClient
+from apify.storage_clients._smart_apify._storage_client import SmartApifyStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
@@ -48,7 +48,6 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from crawlee.proxy_configuration import _NewUrlFunction
-    from crawlee.storage_clients import StorageClient
 
     from apify._models import Webhook
 
@@ -131,7 +130,6 @@ class _ActorType:
         self._configuration = configuration
         self._configure_logging = configure_logging
         self._apify_client: ApifyClientAsync | None = None
-        self._local_storage_client: StorageClient | None = None
 
         self._is_initialized = False
 
@@ -234,45 +232,42 @@ class _ActorType:
         """The logging.Logger instance the Actor uses."""
         return logger
 
-    def _get_local_storage_client(self) -> StorageClient:
-        """Get the local storage client the Actor instance uses."""
-        if self._local_storage_client:
-            return self._local_storage_client
+    def _raise_if_not_initialized(self) -> None:
+        if not self._is_initialized:
+            raise RuntimeError('The Actor was not initialized!')
+
+    @cached_property
+    def _storage_client(self) -> SmartApifyStorageClient:
+        """Storage client used by the actor.
 
+        Depending on the initialization of the service locator the client can be created in different ways.
+        """
         try:
-            # Set implicit default local storage client, unless local storage client was already set.
-            implicit_storage_client = ApifyFileSystemStorageClient()
+            # Nothing was set by the user.
+            implicit_storage_client = SmartApifyStorageClient(
+                local_storage_client=ApifyFileSystemStorageClient(), cloud_storage_client=ApifyStorageClient()
+            )
             service_locator.set_storage_client(implicit_storage_client)
-            self._local_storage_client = implicit_storage_client
         except ServiceConflictError:
             self.log.debug(
                 'Storage client in service locator was set explicitly before Actor.init was called.'
                 'Using the existing storage client as implicit storage client for the Actor.'
             )
-
-        self._local_storage_client = service_locator.get_storage_client()
-        if type(self._local_storage_client) is FileSystemStorageClient:
-            self.log.warning(
-                f'Using {FileSystemStorageClient.__module__}.{FileSystemStorageClient.__name__} in Actor context is not'
-                f' recommended and can lead to problems with reading the input file. Use '
-                f'`apify.storage_clients.FileSystemStorageClient` instead.'
-            )
-
-        return self._local_storage_client
-
-    def _raise_if_not_initialized(self) -> None:
-        if not self._is_initialized:
-            raise RuntimeError('The Actor was not initialized!')
-
-    def _raise_if_cloud_requested_but_not_configured(self, *, force_cloud: bool) -> None:
-        if not force_cloud:
-            return
-
-        if not self.is_at_home() and self.configuration.token is None:
-            raise RuntimeError(
-                'In order to use the Apify cloud storage from your computer, '
-                'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
-            )
+        else:
+            return implicit_storage_client
+
+        # User set something in the service locator.
+        explicit_storage_client = service_locator.get_storage_client()
+        if isinstance(explicit_storage_client, SmartApifyStorageClient):
+            # The client was manually set to the right type in the service locator. This is the explicit way.
+            return explicit_storage_client
+
+        raise RuntimeError(
+            'The storage client in the service locator has to be instance of SmartApifyStorageClient. If you want to '
+            'set the storage client manually you have to call '
+            '`service_locator.set_storage_client(SmartApifyStorageClient(...))` before entering Actor context or '
+            'awaiting `Actor.init`.'
+        )
 
     async def init(self) -> None:
         """Initialize the Actor instance.
@@ -285,6 +280,7 @@ class _ActorType:
         This method should be called immediately before performing any additional Actor actions, and it should be
         called only once.
         """
+        self.log.info('Initializing Actor...')
         if self._configuration:
             # Set explicitly the configuration in the service locator
             service_locator.set_configuration(self.configuration)
@@ -298,22 +294,13 @@ class _ActorType:
         if _ActorType._is_any_instance_initialized:
             self.log.warning('Repeated Actor initialization detected - this is non-standard usage, proceed with care')
 
-        # Create an instance of the cloud storage client, the local storage client is obtained
-        # from the service locator
-        self._cloud_storage_client = ApifyStorageClient()
-
         # Make sure that the currently initialized instance is also available through the global `Actor` proxy
         cast('Proxy', Actor).__wrapped__ = self
 
         self._is_exiting = False
         self._was_final_persist_state_emitted = False
 
-        # If the Actor is running on the Apify platform, we set the cloud storage client.
-        if self.is_at_home():
-            service_locator.set_storage_client(self._cloud_storage_client)
-            self._local_storage_client = self._cloud_storage_client
-        else:
-            self._get_local_storage_client()
+        self.log.debug(f'Storage client set to {self._storage_client}')
 
         service_locator.set_event_manager(self.event_manager)
 
@@ -321,12 +308,8 @@ class _ActorType:
         if self._configure_logging:
             _configure_logging()
 
-        self.log.info('Initializing Actor...')
         self.log.info('System info', extra=get_system_info())
 
-        # TODO: Print outdated SDK version warning (we need a new env var for this)
-        # https://github.com/apify/apify-sdk-python/issues/146
-
         await self.event_manager.__aenter__()
         self.log.debug('Event manager initialized')
 
@@ -473,16 +456,11 @@ class _ActorType:
             An instance of the `Dataset` class for the given ID or name.
         """
         self._raise_if_not_initialized()
-        self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
-
-        storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
-
         return await Dataset.open(
             id=id,
-            alias=alias,
             name=name,
-            configuration=self.configuration,
-            storage_client=storage_client,
+            alias=alias,
+            storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
         )
 
     async def open_key_value_store(
@@ -512,16 +490,11 @@ class _ActorType:
             An instance of the `KeyValueStore` class for the given ID or name.
         """
         self._raise_if_not_initialized()
-        self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
-
-        storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
-
         return await KeyValueStore.open(
             id=id,
-            alias=alias,
             name=name,
-            configuration=self.configuration,
-            storage_client=storage_client,
+            alias=alias,
+            storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
         )
 
     async def open_request_queue(
@@ -553,16 +526,11 @@ class _ActorType:
             An instance of the `RequestQueue` class for the given ID or name.
         """
         self._raise_if_not_initialized()
-        self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
-
-        storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
-
         return await RequestQueue.open(
             id=id,
-            alias=alias,
             name=name,
-            configuration=self.configuration,
-            storage_client=storage_client,
+            alias=alias,
+            storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
         )
 
     @overload

apify/_configuration.py

@@ -142,7 +142,7 @@ class Configuration(CrawleeConfiguration):
     ] = None
 
     default_dataset_id: Annotated[
-        str,
+        str | None,
         Field(
             validation_alias=AliasChoices(
                 'actor_default_dataset_id',
@@ -150,10 +150,10 @@ class Configuration(CrawleeConfiguration):
             ),
             description='Default dataset ID used by the Apify storage client when no ID or name is provided.',
         ),
-    ] = 'default'
+    ] = None
 
     default_key_value_store_id: Annotated[
-        str,
+        str | None,
         Field(
             validation_alias=AliasChoices(
                 'actor_default_key_value_store_id',
@@ -161,10 +161,10 @@ class Configuration(CrawleeConfiguration):
             ),
             description='Default key-value store ID for the Apify storage client when no ID or name is provided.',
         ),
-    ] = 'default'
+    ] = None
 
     default_request_queue_id: Annotated[
-        str,
+        str | None,
         Field(
             validation_alias=AliasChoices(
                 'actor_default_request_queue_id',
@@ -172,7 +172,7 @@ class Configuration(CrawleeConfiguration):
             ),
             description='Default request queue ID for the Apify storage client when no ID or name is provided.',
         ),
-    ] = 'default'
+    ] = None
 
     disable_outdated_warning: Annotated[
         bool,

apify/storage_clients/__init__.py

@@ -2,9 +2,11 @@ from crawlee.storage_clients import MemoryStorageClient
 
 from ._apify import ApifyStorageClient
 from ._file_system import ApifyFileSystemStorageClient as FileSystemStorageClient
+from ._smart_apify import SmartApifyStorageClient
 
 __all__ = [
     'ApifyStorageClient',
     'FileSystemStorageClient',
     'MemoryStorageClient',
+    'SmartApifyStorageClient',
 ]

apify/storage_clients/_apify/_dataset_client.py

@@ -124,8 +124,10 @@ class ApifyDatasetClient(DatasetClient):
         )
         apify_datasets_client = apify_client_async.datasets()
 
-        # Normalize 'default' alias to None
-        alias = None if alias == 'default' else alias
+        # Normalize unnamed default storage in cases where not defined in `configuration.default_dataset_id` to unnamed
+        # storage aliased as `__default__`
+        if not any([alias, name, id, configuration.default_dataset_id]):
+            alias = '__default__'
 
         if alias:
             # Check if there is pre-existing alias mapping in the default KVS.
@@ -150,6 +152,11 @@ class ApifyDatasetClient(DatasetClient):
         # If none are provided, try to get the default storage ID from environment variables.
         elif id is None:
             id = configuration.default_dataset_id
+            if not id:
+                raise ValueError(
+                    'Dataset "id", "name", or "alias" must be specified, '
+                    'or a default dataset ID must be set in the configuration.'
+                )
 
         # Now create the client for the determined ID
         apify_dataset_client = apify_client_async.dataset(dataset_id=id)

apify/storage_clients/_apify/_key_value_store_client.py

@@ -115,8 +115,10 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         )
         apify_kvss_client = apify_client_async.key_value_stores()
 
-        # Normalize 'default' alias to None
-        alias = None if alias == 'default' else alias
+        # Normalize unnamed default storage in cases where not defined in `configuration.default_key_value_store_id` to
+        # unnamed storage aliased as `__default__`
+        if not any([alias, name, id, configuration.default_key_value_store_id]):
+            alias = '__default__'
 
         if alias:
             # Check if there is pre-existing alias mapping in the default KVS.
@@ -142,6 +144,11 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         # If none are provided, try to get the default storage ID from environment variables.
         elif id is None:
             id = configuration.default_key_value_store_id
+            if not id:
+                raise ValueError(
+                    'KeyValueStore "id", "name", or "alias" must be specified, '
+                    'or a default KeyValueStore ID must be set in the configuration.'
+                )
 
         # Now create the client for the determined ID
         apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)