apify 3.0.0rc1__py3-none-any.whl → 3.0.1b2__py3-none-any.whl

apify/_actor.py

@@ -5,6 +5,7 @@ import os
 import sys
 from contextlib import suppress
 from datetime import datetime, timedelta, timezone
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast, overload
 
 from lazy_object_proxy import Proxy
@@ -14,6 +15,7 @@ from pydantic import AliasChoices
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
 from crawlee import service_locator
+from crawlee.errors import ServiceConflictError
 from crawlee.events import (
     Event,
     EventAbortingData,
@@ -34,6 +36,8 @@ from apify._utils import docs_group, docs_name, get_system_info, is_running_in_i
 from apify.events import ApifyEventManager, EventManager, LocalEventManager
 from apify.log import _configure_logging, logger
 from apify.storage_clients import ApifyStorageClient
+from apify.storage_clients._file_system import ApifyFileSystemStorageClient
+from apify.storage_clients._smart_apify._storage_client import SmartApifyStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
@@ -44,7 +48,6 @@ if TYPE_CHECKING:
     from typing_extensions import Self
 
     from crawlee.proxy_configuration import _NewUrlFunction
-    from crawlee.storage_clients import StorageClient
 
     from apify._models import Webhook
 
@@ -119,28 +122,14 @@ class _ActorType:
         self._exit_process = self._get_default_exit_process() if exit_process is None else exit_process
         self._is_exiting = False
 
-        self._configuration = configuration or Configuration.get_global_configuration()
-        self._configure_logging = configure_logging
-        self._apify_client = self.new_client()
-
-        # Create an instance of the cloud storage client, the local storage client is obtained
-        # from the service locator.
-        self._cloud_storage_client = ApifyStorageClient()
+        # Actor state when this method is being executed is unpredictable.
+        # Actor can be initialized by lazy object proxy or by user directly, or by both.
+        # Until `init` method is run, this state of uncertainty remains. This is the reason why any setting done here in
+        # `__init__` method should not be considered final.
 
-        # Set the event manager based on whether the Actor is running on the platform or locally.
-        self._event_manager = (
-            ApifyEventManager(
-                configuration=self._configuration,
-                persist_state_interval=self._configuration.persist_state_interval,
-            )
-            if self.is_at_home()
-            else LocalEventManager(
-                system_info_interval=self._configuration.system_info_interval,
-                persist_state_interval=self._configuration.persist_state_interval,
-            )
-        )
-
-        self._charging_manager = ChargingManagerImplementation(self._configuration, self._apify_client)
+        self._configuration = configuration
+        self._configure_logging = configure_logging
+        self._apify_client: ApifyClientAsync | None = None
 
         self._is_initialized = False
 
@@ -198,46 +187,87 @@ class _ActorType:
     @property
     def apify_client(self) -> ApifyClientAsync:
         """The ApifyClientAsync instance the Actor instance uses."""
+        if not self._apify_client:
+            self._apify_client = self.new_client()
         return self._apify_client
 
-    @property
+    @cached_property
     def configuration(self) -> Configuration:
         """The Configuration instance the Actor instance uses."""
-        return self._configuration
+        if self._configuration:
+            return self._configuration
+
+        try:
+            # Set implicit default Apify configuration, unless configuration was already set.
+            implicit_configuration = Configuration()
+            service_locator.set_configuration(implicit_configuration)
+            self._configuration = implicit_configuration
+        except ServiceConflictError:
+            self.log.debug(
+                'Configuration in service locator was set explicitly before Actor.init was called.'
+                'Using the existing configuration as implicit configuration for the Actor.'
+            )
 
-    @property
-    def config(self) -> Configuration:
-        """The Configuration instance the Actor instance uses."""
+        # Use the configuration from the service locator
+        self._configuration = Configuration.get_global_configuration()
         return self._configuration
 
-    @property
+    @cached_property
     def event_manager(self) -> EventManager:
         """The EventManager instance the Actor instance uses."""
-        return self._event_manager
+        return (
+            ApifyEventManager(
+                configuration=self.configuration,
+                persist_state_interval=self.configuration.persist_state_interval,
+            )
+            if self.is_at_home()
+            else LocalEventManager(
+                system_info_interval=self.configuration.system_info_interval,
+                persist_state_interval=self.configuration.persist_state_interval,
+            )
+        )
 
     @property
     def log(self) -> logging.Logger:
         """The logging.Logger instance the Actor uses."""
         return logger
 
-    @property
-    def _local_storage_client(self) -> StorageClient:
-        """The local storage client the Actor instance uses."""
-        return service_locator.get_storage_client()
-
     def _raise_if_not_initialized(self) -> None:
         if not self._is_initialized:
             raise RuntimeError('The Actor was not initialized!')
 
-    def _raise_if_cloud_requested_but_not_configured(self, *, force_cloud: bool) -> None:
-        if not force_cloud:
-            return
+    @cached_property
+    def _storage_client(self) -> SmartApifyStorageClient:
+        """Storage client used by the actor.
 
-        if not self.is_at_home() and self.config.token is None:
-            raise RuntimeError(
-                'In order to use the Apify cloud storage from your computer, '
-                'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
+        Depending on the initialization of the service locator the client can be created in different ways.
+        """
+        try:
+            # Nothing was set by the user.
+            implicit_storage_client = SmartApifyStorageClient(
+                local_storage_client=ApifyFileSystemStorageClient(), cloud_storage_client=ApifyStorageClient()
             )
+            service_locator.set_storage_client(implicit_storage_client)
+        except ServiceConflictError:
+            self.log.debug(
+                'Storage client in service locator was set explicitly before Actor.init was called.'
+                'Using the existing storage client as implicit storage client for the Actor.'
+            )
+        else:
+            return implicit_storage_client
+
+        # User set something in the service locator.
+        explicit_storage_client = service_locator.get_storage_client()
+        if isinstance(explicit_storage_client, SmartApifyStorageClient):
+            # The client was manually set to the right type in the service locator. This is the explicit way.
+            return explicit_storage_client
+
+        raise RuntimeError(
+            'The storage client in the service locator has to be instance of SmartApifyStorageClient. If you want to '
+            'set the storage client manually you have to call '
+            '`service_locator.set_storage_client(SmartApifyStorageClient(...))` before entering Actor context or '
+            'awaiting `Actor.init`.'
+        )
 
     async def init(self) -> None:
         """Initialize the Actor instance.
@@ -250,6 +280,14 @@ class _ActorType:
         This method should be called immediately before performing any additional Actor actions, and it should be
         called only once.
         """
+        self.log.info('Initializing Actor...')
+        if self._configuration:
+            # Set explicitly the configuration in the service locator
+            service_locator.set_configuration(self.configuration)
+        else:
+            # Ensure that the configuration (cached property) is set
+            _ = self.configuration
+
         if self._is_initialized:
             raise RuntimeError('The Actor was already initialized!')
 
@@ -262,27 +300,20 @@ class _ActorType:
         self._is_exiting = False
         self._was_final_persist_state_emitted = False
 
-        # If the Actor is running on the Apify platform, we set the cloud storage client.
-        if self.is_at_home():
-            service_locator.set_storage_client(self._cloud_storage_client)
+        self.log.debug(f'Storage client set to {self._storage_client}')
 
         service_locator.set_event_manager(self.event_manager)
-        service_locator.set_configuration(self.configuration)
 
         # The logging configuration has to be called after all service_locator set methods.
         if self._configure_logging:
             _configure_logging()
 
-        self.log.info('Initializing Actor...')
         self.log.info('System info', extra=get_system_info())
 
-        # TODO: Print outdated SDK version warning (we need a new env var for this)
-        # https://github.com/apify/apify-sdk-python/issues/146
-
-        await self._event_manager.__aenter__()
+        await self.event_manager.__aenter__()
         self.log.debug('Event manager initialized')
 
-        await self._charging_manager.__aenter__()
+        await self._charging_manager_implementation.__aenter__()
         self.log.debug('Charging manager initialized')
 
         self._is_initialized = True
@@ -323,10 +354,10 @@ class _ActorType:
             await asyncio.sleep(0.1)
 
             if event_listeners_timeout:
-                await self._event_manager.wait_for_all_listeners_to_complete(timeout=event_listeners_timeout)
+                await self.event_manager.wait_for_all_listeners_to_complete(timeout=event_listeners_timeout)
 
-            await self._event_manager.__aexit__(None, None, None)
-            await self._charging_manager.__aexit__(None, None, None)
+            await self.event_manager.__aexit__(None, None, None)
+            await self._charging_manager_implementation.__aexit__(None, None, None)
 
         await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds())
         self._is_initialized = False
@@ -385,8 +416,8 @@ class _ActorType:
                 (increases exponentially from this value).
             timeout: The socket timeout of the HTTP requests sent to the Apify API.
         """
-        token = token or self._configuration.token
-        api_url = api_url or self._configuration.api_base_url
+        token = token or self.configuration.token
+        api_url = api_url or self.configuration.api_base_url
         return ApifyClientAsync(
             token=token,
             api_url=api_url,
@@ -401,6 +432,7 @@ class _ActorType:
         self,
         *,
         id: str | None = None,
+        alias: str | None = None,
         name: str | None = None,
         force_cloud: bool = False,
     ) -> Dataset:
@@ -411,10 +443,12 @@ class _ActorType:
         the Apify cloud.
 
         Args:
-            id: ID of the dataset to be opened. If neither `id` nor `name` are provided, the method returns
-                the default dataset associated with the Actor run.
-            name: Name of the dataset to be opened. If neither `id` nor `name` are provided, the method returns
-                the default dataset associated with the Actor run.
+            id: The ID of the dataset to open. If provided, searches for existing dataset by ID.
+                Mutually exclusive with name and alias.
+            name: The name of the dataset to open (global scope, persists across runs).
+                Mutually exclusive with id and alias.
+            alias: The alias of the dataset to open (run scope, creates unnamed storage).
+                Mutually exclusive with id and name.
             force_cloud: If set to `True` then the Apify cloud storage is always used. This way it is possible
                 to combine local and cloud storage.
 
@@ -422,21 +456,18 @@ class _ActorType:
             An instance of the `Dataset` class for the given ID or name.
         """
         self._raise_if_not_initialized()
-        self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
-
-        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
-
         return await Dataset.open(
             id=id,
             name=name,
-            configuration=self._configuration,
-            storage_client=storage_client,
+            alias=alias,
+            storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
         )
 
     async def open_key_value_store(
         self,
         *,
         id: str | None = None,
+        alias: str | None = None,
         name: str | None = None,
         force_cloud: bool = False,
     ) -> KeyValueStore:
@@ -446,10 +477,12 @@ class _ActorType:
         and retrieved using a unique key. The actual data is stored either on a local filesystem or in the Apify cloud.
 
         Args:
-            id: ID of the key-value store to be opened. If neither `id` nor `name` are provided, the method returns
-                the default key-value store associated with the Actor run.
-            name: Name of the key-value store to be opened. If neither `id` nor `name` are provided, the method
-                returns the default key-value store associated with the Actor run.
+            id: The ID of the KVS to open. If provided, searches for existing KVS by ID.
+                Mutually exclusive with name and alias.
+            name: The name of the KVS to open (global scope, persists across runs).
+                Mutually exclusive with id and alias.
+            alias: The alias of the KVS to open (run scope, creates unnamed storage).
+                Mutually exclusive with id and name.
             force_cloud: If set to `True` then the Apify cloud storage is always used. This way it is possible
                 to combine local and cloud storage.
 
@@ -457,20 +490,18 @@ class _ActorType:
             An instance of the `KeyValueStore` class for the given ID or name.
         """
         self._raise_if_not_initialized()
-        self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
-        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
-
         return await KeyValueStore.open(
             id=id,
             name=name,
-            configuration=self._configuration,
-            storage_client=storage_client,
+            alias=alias,
+            storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
         )
 
     async def open_request_queue(
         self,
         *,
         id: str | None = None,
+        alias: str | None = None,
         name: str | None = None,
         force_cloud: bool = False,
     ) -> RequestQueue:
@@ -482,10 +513,12 @@ class _ActorType:
         crawling orders.
 
         Args:
-            id: ID of the request queue to be opened. If neither `id` nor `name` are provided, the method returns
-                the default request queue associated with the Actor run.
-            name: Name of the request queue to be opened. If neither `id` nor `name` are provided, the method returns
-                the default request queue associated with the Actor run.
+            id: The ID of the RQ to open. If provided, searches for existing RQ by ID.
+                Mutually exclusive with name and alias.
+            name: The name of the RQ to open (global scope, persists across runs).
+                Mutually exclusive with id and alias.
+            alias: The alias of the RQ to open (run scope, creates unnamed storage).
+                Mutually exclusive with id and name.
             force_cloud: If set to `True` then the Apify cloud storage is always used. This way it is possible
                 to combine local and cloud storage.
 
@@ -493,15 +526,11 @@ class _ActorType:
             An instance of the `RequestQueue` class for the given ID or name.
         """
         self._raise_if_not_initialized()
-        self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
-
-        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
-
         return await RequestQueue.open(
             id=id,
             name=name,
-            configuration=self._configuration,
-            storage_client=storage_client,
+            alias=alias,
+            storage_client=self._storage_client.get_suitable_storage_client(force_cloud=force_cloud),
         )
 
     @overload
@@ -524,7 +553,7 @@ class _ActorType:
         data = data if isinstance(data, list) else [data]
 
         max_charged_count = (
-            self._charging_manager.calculate_max_event_charge_count_within_limit(charged_event_name)
+            self.get_charging_manager().calculate_max_event_charge_count_within_limit(charged_event_name)
             if charged_event_name is not None
             else None
         )
@@ -538,7 +567,7 @@ class _ActorType:
             await dataset.push_data(data)
 
         if charged_event_name:
-            return await self._charging_manager.charge(
+            return await self.get_charging_manager().charge(
                 event_name=charged_event_name,
                 count=min(max_charged_count, len(data)) if max_charged_count is not None else len(data),
             )
@@ -549,9 +578,9 @@ class _ActorType:
         """Get the Actor input value from the default key-value store associated with the current Actor run."""
         self._raise_if_not_initialized()
 
-        input_value = await self.get_value(self._configuration.input_key)
-        input_secrets_private_key = self._configuration.input_secrets_private_key_file
-        input_secrets_key_passphrase = self._configuration.input_secrets_private_key_passphrase
+        input_value = await self.get_value(self.configuration.input_key)
+        input_secrets_private_key = self.configuration.input_secrets_private_key_file
+        input_secrets_key_passphrase = self.configuration.input_secrets_private_key_passphrase
         if input_secrets_private_key and input_secrets_key_passphrase:
             private_key = load_private_key(
                 input_secrets_private_key,
@@ -595,7 +624,11 @@ class _ActorType:
     def get_charging_manager(self) -> ChargingManager:
         """Retrieve the charging manager to access granular pricing information."""
         self._raise_if_not_initialized()
-        return self._charging_manager
+        return self._charging_manager_implementation
+
+    @cached_property
+    def _charging_manager_implementation(self) -> ChargingManagerImplementation:
+        return ChargingManagerImplementation(self.configuration, self.apify_client)
 
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Charge for a specified number of events - sub-operations of the Actor.
@@ -607,7 +640,7 @@ class _ActorType:
             count: Number of events to charge for.
         """
         self._raise_if_not_initialized()
-        return await self._charging_manager.charge(event_name, count)
+        return await self.get_charging_manager().charge(event_name, count)
 
     @overload
     def on(
@@ -658,7 +691,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        self._event_manager.on(event=event_name, listener=listener)
+        self.event_manager.on(event=event_name, listener=listener)
         return listener
 
     @overload
@@ -684,11 +717,11 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        self._event_manager.off(event=event_name, listener=listener)
+        self.event_manager.off(event=event_name, listener=listener)
 
     def is_at_home(self) -> bool:
         """Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)."""
-        return self._configuration.is_at_home
+        return self.configuration.is_at_home
 
     def get_env(self) -> dict:
         """Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.
@@ -714,7 +747,7 @@ class _ActorType:
                 aliases = [field_name]
 
             for alias in aliases:
-                config[alias] = getattr(self._configuration, field_name)
+                config[alias] = getattr(self.configuration, field_name)
 
         env_vars = {env_var.value.lower(): env_var.name.lower() for env_var in [*ActorEnvVars, *ApifyEnvVars]}
         return {option_name: config[env_var] for env_var, option_name in env_vars.items() if env_var in config}
@@ -759,7 +792,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if webhooks:
             serialized_webhooks = [
@@ -790,7 +823,7 @@ class _ActorType:
         return ActorRun.model_validate(api_result)
 
     def _get_remaining_time(self) -> timedelta | None:
-        """Get time remaining from the actor timeout. Returns `None` if not on an Apify platform."""
+        """Get time remaining from the Actor timeout. Returns `None` if not on an Apify platform."""
         if self.is_at_home() and self.configuration.timeout_at:
             return self.configuration.timeout_at - datetime.now(tz=timezone.utc)
 
@@ -826,7 +859,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if status_message:
             await client.run(run_id).update(status_message=status_message)
@@ -879,7 +912,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if webhooks:
             serialized_webhooks = [
@@ -951,7 +984,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if webhooks:
             serialized_webhooks = [
@@ -1002,13 +1035,13 @@ class _ActorType:
             return
 
         if not custom_after_sleep:
-            custom_after_sleep = self._configuration.metamorph_after_sleep
+            custom_after_sleep = self.configuration.metamorph_after_sleep
 
-        # If is_at_home() is True, config.actor_run_id is always set
-        if not self._configuration.actor_run_id:
+        # If is_at_home() is True, configuration.actor_run_id is always set
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        await self._apify_client.run(self._configuration.actor_run_id).metamorph(
+        await self.apify_client.run(self.configuration.actor_run_id).metamorph(
             target_actor_id=target_actor_id,
             run_input=run_input,
             target_actor_build=target_actor_build,
@@ -1045,7 +1078,7 @@ class _ActorType:
         _ActorType._is_rebooting = True
 
         if not custom_after_sleep:
-            custom_after_sleep = self._configuration.metamorph_after_sleep
+            custom_after_sleep = self.configuration.metamorph_after_sleep
 
         # Call all the listeners for the PERSIST_STATE and MIGRATING events, and wait for them to finish.
         # PERSIST_STATE listeners are called to allow the Actor to persist its state before the reboot.
@@ -1054,10 +1087,10 @@ class _ActorType:
         # We can't just emit the events and wait for all listeners to finish,
         # because this method might be called from an event listener itself, and we would deadlock.
         persist_state_listeners = flatten(
-            (self._event_manager._listeners_to_wrappers[Event.PERSIST_STATE] or {}).values()  # noqa: SLF001
+            (self.event_manager._listeners_to_wrappers[Event.PERSIST_STATE] or {}).values()  # noqa: SLF001
         )
         migrating_listeners = flatten(
-            (self._event_manager._listeners_to_wrappers[Event.MIGRATING] or {}).values()  # noqa: SLF001
+            (self.event_manager._listeners_to_wrappers[Event.MIGRATING] or {}).values()  # noqa: SLF001
         )
 
         await asyncio.gather(
@@ -1065,10 +1098,10 @@ class _ActorType:
             *[listener(EventMigratingData()) for listener in migrating_listeners],
         )
 
-        if not self._configuration.actor_run_id:
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        await self._apify_client.run(self._configuration.actor_run_id).reboot()
+        await self.apify_client.run(self.configuration.actor_run_id).reboot()
 
         if custom_after_sleep:
             await asyncio.sleep(custom_after_sleep.total_seconds())
@@ -1107,11 +1140,11 @@ class _ActorType:
             return
 
         # If is_at_home() is True, config.actor_run_id is always set
-        if not self._configuration.actor_run_id:
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        await self._apify_client.webhooks().create(
-            actor_run_id=self._configuration.actor_run_id,
+        await self.apify_client.webhooks().create(
+            actor_run_id=self.configuration.actor_run_id,
             event_types=webhook.event_types,
             request_url=webhook.request_url,
             payload_template=webhook.payload_template,
@@ -1143,10 +1176,10 @@ class _ActorType:
             return None
 
         # If is_at_home() is True, config.actor_run_id is always set
-        if not self._configuration.actor_run_id:
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        api_result = await self._apify_client.run(self._configuration.actor_run_id).update(
+        api_result = await self.apify_client.run(self.configuration.actor_run_id).update(
             status_message=status_message, is_status_message_terminal=is_terminal
         )
 
@@ -1156,7 +1189,7 @@ class _ActorType:
         self,
         *,
         actor_proxy_input: dict
-        | None = None,  # this is the raw proxy input from the actor run input, it is not spread or snake_cased in here
+        | None = None,  # this is the raw proxy input from the Actor run input, it is not spread or snake_cased in here
         password: str | None = None,
         groups: list[str] | None = None,
         country_code: str | None = None,
@@ -1201,7 +1234,7 @@ class _ActorType:
             country_code=country_code,
             proxy_urls=proxy_urls,
             new_url_function=new_url_function,
-            _actor_config=self._configuration,
+            _actor_config=self.configuration,
             _apify_client=self._apify_client,
         )
 

apify/_charging.py

@@ -64,6 +64,16 @@ class ChargingManager(Protocol):
         This can be used for instance when your code needs to support multiple pricing models in transition periods.
         """
 
+    def get_charged_event_count(self, event_name: str) -> int:
+        """Get the number of events with the given name that were charged so far.
+
+        Args:
+            event_name: Name of the inspected event.
+        """
+
+    def get_max_total_charge_usd(self) -> Decimal:
+        """Get the configured maximum total charge for this Actor run."""
+
 
 @docs_group('Charging')
 @dataclass(frozen=True)
@@ -309,6 +319,15 @@ class ChargingManagerImplementation(ChargingManager):
             },
         )
 
+    @ensure_context
+    def get_charged_event_count(self, event_name: str) -> int:
+        item = self._charging_state.get(event_name)
+        return item.charge_count if item is not None else 0
+
+    @ensure_context
+    def get_max_total_charge_usd(self) -> Decimal:
+        return self._max_total_charge_usd
+
 
 @dataclass
 class ChargingStateItem: