apify 2.7.1b17__py3-none-any.whl → 2.7.1b18__py3-none-any.whl

apify/_actor.py

@@ -5,6 +5,7 @@ import os
 import sys
 from contextlib import suppress
 from datetime import datetime, timedelta, timezone
+from functools import cached_property
 from typing import TYPE_CHECKING, Any, Literal, TypeVar, cast, overload
 
 from lazy_object_proxy import Proxy
@@ -14,6 +15,7 @@ from pydantic import AliasChoices
 from apify_client import ApifyClientAsync
 from apify_shared.consts import ActorEnvVars, ActorExitCodes, ApifyEnvVars
 from crawlee import service_locator
+from crawlee.errors import ServiceConflictError
 from crawlee.events import (
     Event,
     EventAbortingData,
@@ -23,6 +25,7 @@ from crawlee.events import (
     EventPersistStateData,
     EventSystemInfoData,
 )
+from crawlee.storage_clients import FileSystemStorageClient
 
 from apify._charging import ChargeResult, ChargingManager, ChargingManagerImplementation
 from apify._configuration import Configuration
@@ -34,6 +37,7 @@ from apify._utils import docs_group, docs_name, get_system_info, is_running_in_i
 from apify.events import ApifyEventManager, EventManager, LocalEventManager
 from apify.log import _configure_logging, logger
 from apify.storage_clients import ApifyStorageClient
+from apify.storage_clients._file_system import ApifyFileSystemStorageClient
 from apify.storages import Dataset, KeyValueStore, RequestQueue
 
 if TYPE_CHECKING:
@@ -119,28 +123,15 @@ class _ActorType:
         self._exit_process = self._get_default_exit_process() if exit_process is None else exit_process
         self._is_exiting = False
 
-        self._configuration = configuration or Configuration.get_global_configuration()
-        self._configure_logging = configure_logging
-        self._apify_client = self.new_client()
+        # Actor state when this method is being executed is unpredictable.
+        # Actor can be initialized by lazy object proxy or by user directly, or by both.
+        # Until `init` method is run, this state of uncertainty remains. This is the reason why any setting done here in
+        # `__init__` method should not be considered final.
 
-        # Create an instance of the cloud storage client, the local storage client is obtained
-        # from the service locator.
-        self._cloud_storage_client = ApifyStorageClient()
-
-        # Set the event manager based on whether the Actor is running on the platform or locally.
-        self._event_manager = (
-            ApifyEventManager(
-                configuration=self._configuration,
-                persist_state_interval=self._configuration.persist_state_interval,
-            )
-            if self.is_at_home()
-            else LocalEventManager(
-                system_info_interval=self._configuration.system_info_interval,
-                persist_state_interval=self._configuration.persist_state_interval,
-            )
-        )
-
-        self._charging_manager = ChargingManagerImplementation(self._configuration, self._apify_client)
+        self._configuration = configuration
+        self._configure_logging = configure_logging
+        self._apify_client: ApifyClientAsync | None = None
+        self._local_storage_client: StorageClient | None = None
 
         self._is_initialized = False
 
@@ -198,32 +189,76 @@ class _ActorType:
     @property
     def apify_client(self) -> ApifyClientAsync:
         """The ApifyClientAsync instance the Actor instance uses."""
+        if not self._apify_client:
+            self._apify_client = self.new_client()
         return self._apify_client
 
-    @property
+    @cached_property
     def configuration(self) -> Configuration:
         """The Configuration instance the Actor instance uses."""
-        return self._configuration
+        if self._configuration:
+            return self._configuration
+
+        try:
+            # Set implicit default Apify configuration, unless configuration was already set.
+            implicit_configuration = Configuration()
+            service_locator.set_configuration(implicit_configuration)
+            self._configuration = implicit_configuration
+        except ServiceConflictError:
+            self.log.debug(
+                'Configuration in service locator was set explicitly before Actor.init was called.'
+                'Using the existing configuration as implicit configuration for the Actor.'
+            )
 
-    @property
-    def config(self) -> Configuration:
-        """The Configuration instance the Actor instance uses."""
+        # Use the configuration from the service locator
+        self._configuration = Configuration.get_global_configuration()
         return self._configuration
 
-    @property
+    @cached_property
     def event_manager(self) -> EventManager:
         """The EventManager instance the Actor instance uses."""
-        return self._event_manager
+        return (
+            ApifyEventManager(
+                configuration=self.configuration,
+                persist_state_interval=self.configuration.persist_state_interval,
+            )
+            if self.is_at_home()
+            else LocalEventManager(
+                system_info_interval=self.configuration.system_info_interval,
+                persist_state_interval=self.configuration.persist_state_interval,
+            )
+        )
 
     @property
     def log(self) -> logging.Logger:
         """The logging.Logger instance the Actor uses."""
         return logger
 
-    @property
-    def _local_storage_client(self) -> StorageClient:
-        """The local storage client the Actor instance uses."""
-        return service_locator.get_storage_client()
+    def _get_local_storage_client(self) -> StorageClient:
+        """Get the local storage client the Actor instance uses."""
+        if self._local_storage_client:
+            return self._local_storage_client
+
+        try:
+            # Set implicit default local storage client, unless local storage client was already set.
+            implicit_storage_client = ApifyFileSystemStorageClient()
+            service_locator.set_storage_client(implicit_storage_client)
+            self._local_storage_client = implicit_storage_client
+        except ServiceConflictError:
+            self.log.debug(
+                'Storage client in service locator was set explicitly before Actor.init was called.'
+                'Using the existing storage client as implicit storage client for the Actor.'
+            )
+
+        self._local_storage_client = service_locator.get_storage_client()
+        if type(self._local_storage_client) is FileSystemStorageClient:
+            self.log.warning(
+                f'Using {FileSystemStorageClient.__module__}.{FileSystemStorageClient.__name__} in Actor context is not'
+                f' recommended and can lead to problems with reading the input file. Use '
+                f'`apify.storage_clients.FileSystemStorageClient` instead.'
+            )
+
+        return self._local_storage_client
 
     def _raise_if_not_initialized(self) -> None:
         if not self._is_initialized:
@@ -233,7 +268,7 @@ class _ActorType:
         if not force_cloud:
             return
 
-        if not self.is_at_home() and self.config.token is None:
+        if not self.is_at_home() and self.configuration.token is None:
             raise RuntimeError(
                 'In order to use the Apify cloud storage from your computer, '
                 'you need to provide an Apify token using the APIFY_TOKEN environment variable.'
@@ -250,12 +285,23 @@ class _ActorType:
         This method should be called immediately before performing any additional Actor actions, and it should be
         called only once.
         """
+        if self._configuration:
+            # Set explicitly the configuration in the service locator
+            service_locator.set_configuration(self.configuration)
+        else:
+            # Ensure that the configuration (cached property) is set
+            _ = self.configuration
+
         if self._is_initialized:
             raise RuntimeError('The Actor was already initialized!')
 
         if _ActorType._is_any_instance_initialized:
             self.log.warning('Repeated Actor initialization detected - this is non-standard usage, proceed with care')
 
+        # Create an instance of the cloud storage client, the local storage client is obtained
+        # from the service locator
+        self._cloud_storage_client = ApifyStorageClient()
+
         # Make sure that the currently initialized instance is also available through the global `Actor` proxy
         cast('Proxy', Actor).__wrapped__ = self
 
@@ -265,9 +311,11 @@ class _ActorType:
         # If the Actor is running on the Apify platform, we set the cloud storage client.
         if self.is_at_home():
             service_locator.set_storage_client(self._cloud_storage_client)
+            self._local_storage_client = self._cloud_storage_client
+        else:
+            self._get_local_storage_client()
 
         service_locator.set_event_manager(self.event_manager)
-        service_locator.set_configuration(self.configuration)
 
         # The logging configuration has to be called after all service_locator set methods.
         if self._configure_logging:
@@ -279,10 +327,10 @@ class _ActorType:
         # TODO: Print outdated SDK version warning (we need a new env var for this)
         # https://github.com/apify/apify-sdk-python/issues/146
 
-        await self._event_manager.__aenter__()
+        await self.event_manager.__aenter__()
         self.log.debug('Event manager initialized')
 
-        await self._charging_manager.__aenter__()
+        await self._charging_manager_implementation.__aenter__()
         self.log.debug('Charging manager initialized')
 
         self._is_initialized = True
@@ -323,10 +371,10 @@ class _ActorType:
             await asyncio.sleep(0.1)
 
             if event_listeners_timeout:
-                await self._event_manager.wait_for_all_listeners_to_complete(timeout=event_listeners_timeout)
+                await self.event_manager.wait_for_all_listeners_to_complete(timeout=event_listeners_timeout)
 
-            await self._event_manager.__aexit__(None, None, None)
-            await self._charging_manager.__aexit__(None, None, None)
+            await self.event_manager.__aexit__(None, None, None)
+            await self._charging_manager_implementation.__aexit__(None, None, None)
 
         await asyncio.wait_for(finalize(), cleanup_timeout.total_seconds())
         self._is_initialized = False
@@ -385,8 +433,8 @@ class _ActorType:
                 (increases exponentially from this value).
             timeout: The socket timeout of the HTTP requests sent to the Apify API.
         """
-        token = token or self._configuration.token
-        api_url = api_url or self._configuration.api_base_url
+        token = token or self.configuration.token
+        api_url = api_url or self.configuration.api_base_url
         return ApifyClientAsync(
             token=token,
             api_url=api_url,
@@ -427,13 +475,13 @@ class _ActorType:
         self._raise_if_not_initialized()
         self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
 
-        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
+        storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
 
         return await Dataset.open(
             id=id,
             alias=alias,
             name=name,
-            configuration=self._configuration,
+            configuration=self.configuration,
             storage_client=storage_client,
         )
 
@@ -465,13 +513,14 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
         self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
-        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
+
+        storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
 
         return await KeyValueStore.open(
             id=id,
             alias=alias,
             name=name,
-            configuration=self._configuration,
+            configuration=self.configuration,
             storage_client=storage_client,
         )
 
@@ -506,13 +555,13 @@ class _ActorType:
         self._raise_if_not_initialized()
         self._raise_if_cloud_requested_but_not_configured(force_cloud=force_cloud)
 
-        storage_client = self._cloud_storage_client if force_cloud else self._local_storage_client
+        storage_client = self._cloud_storage_client if force_cloud else self._get_local_storage_client()
 
         return await RequestQueue.open(
             id=id,
             alias=alias,
             name=name,
-            configuration=self._configuration,
+            configuration=self.configuration,
             storage_client=storage_client,
         )
 
@@ -536,7 +585,7 @@ class _ActorType:
         data = data if isinstance(data, list) else [data]
 
         max_charged_count = (
-            self._charging_manager.calculate_max_event_charge_count_within_limit(charged_event_name)
+            self.get_charging_manager().calculate_max_event_charge_count_within_limit(charged_event_name)
             if charged_event_name is not None
             else None
         )
@@ -550,7 +599,7 @@ class _ActorType:
             await dataset.push_data(data)
 
         if charged_event_name:
-            return await self._charging_manager.charge(
+            return await self.get_charging_manager().charge(
                 event_name=charged_event_name,
                 count=min(max_charged_count, len(data)) if max_charged_count is not None else len(data),
             )
@@ -561,9 +610,9 @@ class _ActorType:
         """Get the Actor input value from the default key-value store associated with the current Actor run."""
         self._raise_if_not_initialized()
 
-        input_value = await self.get_value(self._configuration.input_key)
-        input_secrets_private_key = self._configuration.input_secrets_private_key_file
-        input_secrets_key_passphrase = self._configuration.input_secrets_private_key_passphrase
+        input_value = await self.get_value(self.configuration.input_key)
+        input_secrets_private_key = self.configuration.input_secrets_private_key_file
+        input_secrets_key_passphrase = self.configuration.input_secrets_private_key_passphrase
         if input_secrets_private_key and input_secrets_key_passphrase:
             private_key = load_private_key(
                 input_secrets_private_key,
@@ -607,7 +656,11 @@ class _ActorType:
     def get_charging_manager(self) -> ChargingManager:
         """Retrieve the charging manager to access granular pricing information."""
         self._raise_if_not_initialized()
-        return self._charging_manager
+        return self._charging_manager_implementation
+
+    @cached_property
+    def _charging_manager_implementation(self) -> ChargingManagerImplementation:
+        return ChargingManagerImplementation(self.configuration, self.apify_client)
 
     async def charge(self, event_name: str, count: int = 1) -> ChargeResult:
         """Charge for a specified number of events - sub-operations of the Actor.
@@ -619,7 +672,7 @@ class _ActorType:
             count: Number of events to charge for.
         """
         self._raise_if_not_initialized()
-        return await self._charging_manager.charge(event_name, count)
+        return await self.get_charging_manager().charge(event_name, count)
 
     @overload
     def on(
@@ -670,7 +723,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        self._event_manager.on(event=event_name, listener=listener)
+        self.event_manager.on(event=event_name, listener=listener)
         return listener
 
     @overload
@@ -696,11 +749,11 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        self._event_manager.off(event=event_name, listener=listener)
+        self.event_manager.off(event=event_name, listener=listener)
 
     def is_at_home(self) -> bool:
         """Return `True` when the Actor is running on the Apify platform, and `False` otherwise (e.g. local run)."""
-        return self._configuration.is_at_home
+        return self.configuration.is_at_home
 
     def get_env(self) -> dict:
         """Return a dictionary with information parsed from all the `APIFY_XXX` environment variables.
@@ -726,7 +779,7 @@ class _ActorType:
                 aliases = [field_name]
 
             for alias in aliases:
-                config[alias] = getattr(self._configuration, field_name)
+                config[alias] = getattr(self.configuration, field_name)
 
         env_vars = {env_var.value.lower(): env_var.name.lower() for env_var in [*ActorEnvVars, *ApifyEnvVars]}
         return {option_name: config[env_var] for env_var, option_name in env_vars.items() if env_var in config}
@@ -771,7 +824,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if webhooks:
             serialized_webhooks = [
@@ -802,7 +855,7 @@ class _ActorType:
         return ActorRun.model_validate(api_result)
 
     def _get_remaining_time(self) -> timedelta | None:
-        """Get time remaining from the actor timeout. Returns `None` if not on an Apify platform."""
+        """Get time remaining from the Actor timeout. Returns `None` if not on an Apify platform."""
         if self.is_at_home() and self.configuration.timeout_at:
             return self.configuration.timeout_at - datetime.now(tz=timezone.utc)
 
@@ -838,7 +891,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if status_message:
             await client.run(run_id).update(status_message=status_message)
@@ -891,7 +944,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if webhooks:
             serialized_webhooks = [
@@ -963,7 +1016,7 @@ class _ActorType:
         """
         self._raise_if_not_initialized()
 
-        client = self.new_client(token=token) if token else self._apify_client
+        client = self.new_client(token=token) if token else self.apify_client
 
         if webhooks:
             serialized_webhooks = [
@@ -1014,13 +1067,13 @@ class _ActorType:
             return
 
         if not custom_after_sleep:
-            custom_after_sleep = self._configuration.metamorph_after_sleep
+            custom_after_sleep = self.configuration.metamorph_after_sleep
 
-        # If is_at_home() is True, config.actor_run_id is always set
-        if not self._configuration.actor_run_id:
+        # If is_at_home() is True, configuration.actor_run_id is always set
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        await self._apify_client.run(self._configuration.actor_run_id).metamorph(
+        await self.apify_client.run(self.configuration.actor_run_id).metamorph(
             target_actor_id=target_actor_id,
             run_input=run_input,
             target_actor_build=target_actor_build,
@@ -1057,7 +1110,7 @@ class _ActorType:
         _ActorType._is_rebooting = True
 
         if not custom_after_sleep:
-            custom_after_sleep = self._configuration.metamorph_after_sleep
+            custom_after_sleep = self.configuration.metamorph_after_sleep
 
         # Call all the listeners for the PERSIST_STATE and MIGRATING events, and wait for them to finish.
         # PERSIST_STATE listeners are called to allow the Actor to persist its state before the reboot.
@@ -1066,10 +1119,10 @@ class _ActorType:
         # We can't just emit the events and wait for all listeners to finish,
         # because this method might be called from an event listener itself, and we would deadlock.
         persist_state_listeners = flatten(
-            (self._event_manager._listeners_to_wrappers[Event.PERSIST_STATE] or {}).values()  # noqa: SLF001
+            (self.event_manager._listeners_to_wrappers[Event.PERSIST_STATE] or {}).values()  # noqa: SLF001
         )
         migrating_listeners = flatten(
-            (self._event_manager._listeners_to_wrappers[Event.MIGRATING] or {}).values()  # noqa: SLF001
+            (self.event_manager._listeners_to_wrappers[Event.MIGRATING] or {}).values()  # noqa: SLF001
         )
 
         await asyncio.gather(
@@ -1077,10 +1130,10 @@ class _ActorType:
             *[listener(EventMigratingData()) for listener in migrating_listeners],
         )
 
-        if not self._configuration.actor_run_id:
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        await self._apify_client.run(self._configuration.actor_run_id).reboot()
+        await self.apify_client.run(self.configuration.actor_run_id).reboot()
 
         if custom_after_sleep:
             await asyncio.sleep(custom_after_sleep.total_seconds())
@@ -1119,11 +1172,11 @@ class _ActorType:
             return
 
         # If is_at_home() is True, config.actor_run_id is always set
-        if not self._configuration.actor_run_id:
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        await self._apify_client.webhooks().create(
-            actor_run_id=self._configuration.actor_run_id,
+        await self.apify_client.webhooks().create(
+            actor_run_id=self.configuration.actor_run_id,
             event_types=webhook.event_types,
             request_url=webhook.request_url,
             payload_template=webhook.payload_template,
@@ -1155,10 +1208,10 @@ class _ActorType:
             return None
 
         # If is_at_home() is True, config.actor_run_id is always set
-        if not self._configuration.actor_run_id:
+        if not self.configuration.actor_run_id:
             raise RuntimeError('actor_run_id cannot be None when running on the Apify platform.')
 
-        api_result = await self._apify_client.run(self._configuration.actor_run_id).update(
+        api_result = await self.apify_client.run(self.configuration.actor_run_id).update(
             status_message=status_message, is_status_message_terminal=is_terminal
         )
 
@@ -1168,7 +1221,7 @@ class _ActorType:
         self,
         *,
         actor_proxy_input: dict
-        | None = None,  # this is the raw proxy input from the actor run input, it is not spread or snake_cased in here
+        | None = None,  # this is the raw proxy input from the Actor run input, it is not spread or snake_cased in here
         password: str | None = None,
         groups: list[str] | None = None,
         country_code: str | None = None,
@@ -1213,7 +1266,7 @@ class _ActorType:
             country_code=country_code,
             proxy_urls=proxy_urls,
             new_url_function=new_url_function,
-            _actor_config=self._configuration,
+            _actor_config=self.configuration,
             _apify_client=self._apify_client,
         )
 

apify/_configuration.py

@@ -8,6 +8,7 @@ from typing import Annotated, Any
 from pydantic import AliasChoices, BeforeValidator, Field, model_validator
 from typing_extensions import Self, deprecated
 
+from crawlee import service_locator
 from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 from crawlee.configuration import Configuration as CrawleeConfiguration
@@ -424,11 +425,41 @@ class Configuration(CrawleeConfiguration):
     def get_global_configuration(cls) -> Configuration:
         """Retrieve the global instance of the configuration.
 
-        Mostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`
-        instead.
+        This method ensures that ApifyConfigration is returned, even if CrawleeConfiguration was set in the
+        service locator.
         """
-        return cls()
+        global_configuration = service_locator.get_configuration()
 
+        if isinstance(global_configuration, Configuration):
+            # If Apify configuration was already stored in service locator, return it.
+            return global_configuration
 
-# Monkey-patch the base class so that it works with the extended configuration
-CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore[method-assign]
+        logger.warning(
+            'Non Apify Configration is set in the `service_locator` in the SDK context. '
+            'It is recommended to set `apify.Configuration` explicitly as early as possible by using '
+            'service_locator.set_configuration'
+        )
+
+        return cls.from_configuration(global_configuration)
+
+    @classmethod
+    def from_configuration(cls, configuration: CrawleeConfiguration) -> Configuration:
+        """Create Apify Configuration from existing Crawlee Configuration.
+
+        Args:
+            configuration: The existing Crawlee Configuration.
+
+        Returns:
+            The created Apify Configuration.
+        """
+        apify_configuration = cls()
+
+        # Ensure the returned configuration is of type Apify Configuration.
+        # Most likely crawlee configuration was already set. Create Apify configuration from it.
+        # Due to known Pydantic issue https://github.com/pydantic/pydantic/issues/9516, creating new instance of
+        # Configuration from existing one in situation where environment can have some fields set by alias is very
+        # unpredictable. Use the stable workaround.
+        for name in configuration.model_fields:
+            setattr(apify_configuration, name, getattr(configuration, name))
+
+        return apify_configuration

apify/storage_clients/_apify/_dataset_client.py

@@ -11,8 +11,9 @@ from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.file import json_dumps
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
+from crawlee.storages import Dataset
 
-from ._utils import resolve_alias_to_id, store_alias_mapping
+from ._utils import AliasResolver
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -126,19 +127,19 @@ class ApifyDatasetClient(DatasetClient):
         # Normalize 'default' alias to None
         alias = None if alias == 'default' else alias
 
-        # Handle alias resolution
         if alias:
-            # Try to resolve alias to existing storage ID
-            resolved_id = await resolve_alias_to_id(alias, 'dataset', configuration)
-            if resolved_id:
-                id = resolved_id
-            else:
-                # Create a new storage and store the alias mapping
-                new_storage_metadata = DatasetMetadata.model_validate(
-                    await apify_datasets_client.get_or_create(),
-                )
-                id = new_storage_metadata.id
-                await store_alias_mapping(alias, 'dataset', id, configuration)
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with AliasResolver(storage_type=Dataset, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    new_storage_metadata = DatasetMetadata.model_validate(
+                        await apify_datasets_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
         elif name:

apify/storage_clients/_apify/_key_value_store_client.py

@@ -10,9 +10,10 @@ from yarl import URL
 from apify_client import ApifyClientAsync
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata
+from crawlee.storages import KeyValueStore
 
 from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage
-from ._utils import resolve_alias_to_id, store_alias_mapping
+from ._utils import AliasResolver
 from apify._crypto import create_hmac_signature
 
 if TYPE_CHECKING:
@@ -117,19 +118,20 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         # Normalize 'default' alias to None
         alias = None if alias == 'default' else alias
 
-        # Handle alias resolution
         if alias:
-            # Try to resolve alias to existing storage ID
-            resolved_id = await resolve_alias_to_id(alias, 'kvs', configuration)
-            if resolved_id:
-                id = resolved_id
-            else:
-                # Create a new storage and store the alias mapping
-                new_storage_metadata = ApifyKeyValueStoreMetadata.model_validate(
-                    await apify_kvss_client.get_or_create(),
-                )
-                id = new_storage_metadata.id
-                await store_alias_mapping(alias, 'kvs', id, configuration)
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with AliasResolver(storage_type=KeyValueStore, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    # Create a new storage and store the alias mapping
+                    new_storage_metadata = ApifyKeyValueStoreMetadata.model_validate(
+                        await apify_kvss_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
         elif name:

apify/storage_clients/_apify/_request_queue_client.py

@@ -16,9 +16,10 @@ from apify_client import ApifyClientAsync
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
+from crawlee.storages import RequestQueue
 
 from ._models import CachedRequest, ProlongRequestLockResponse, RequestQueueHead
-from ._utils import resolve_alias_to_id, store_alias_mapping
+from ._utils import AliasResolver
 from apify import Request
 
 if TYPE_CHECKING:
@@ -195,19 +196,19 @@ class ApifyRequestQueueClient(RequestQueueClient):
         # Normalize 'default' alias to None
         alias = None if alias == 'default' else alias
 
-        # Handle alias resolution
         if alias:
-            # Try to resolve alias to existing storage ID
-            resolved_id = await resolve_alias_to_id(alias, 'rq', configuration)
-            if resolved_id:
-                id = resolved_id
-            else:
-                # Create a new storage and store the alias mapping
-                new_storage_metadata = RequestQueueMetadata.model_validate(
-                    await apify_rqs_client.get_or_create(),
-                )
-                id = new_storage_metadata.id
-                await store_alias_mapping(alias, 'rq', id, configuration)
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with AliasResolver(storage_type=RequestQueue, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    new_storage_metadata = RequestQueueMetadata.model_validate(
+                        await apify_rqs_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
         elif name:

apify/storage_clients/_apify/_storage_client.py

@@ -9,16 +9,35 @@ from crawlee.storage_clients._base import StorageClient
 from ._dataset_client import ApifyDatasetClient
 from ._key_value_store_client import ApifyKeyValueStoreClient
 from ._request_queue_client import ApifyRequestQueueClient
+from ._utils import hash_api_base_url_and_token
+from apify._configuration import Configuration as ApifyConfiguration
 from apify._utils import docs_group
 
 if TYPE_CHECKING:
-    from crawlee.configuration import Configuration
+    from collections.abc import Hashable
+
+    from crawlee.configuration import Configuration as CrawleeConfiguration
 
 
 @docs_group('Storage clients')
 class ApifyStorageClient(StorageClient):
     """Apify storage client."""
 
+    # This class breaches Liskov Substitution Principle. It requires specialized Configuration compared to its parent.
+    _lsp_violation_error_message_template = (
+        'Expected "configuration" to be an instance of "apify.Configuration", but got {} instead.'
+    )
+
+    @override
+    def get_additional_cache_key(self, configuration: CrawleeConfiguration) -> Hashable:
+        if isinstance(configuration, ApifyConfiguration):
+            return hash_api_base_url_and_token(configuration)
+
+        config_class = type(configuration)
+        raise TypeError(
+            self._lsp_violation_error_message_template.format(f'{config_class.__module__}.{config_class.__name__}')
+        )
+
     @override
     async def create_dataset_client(
         self,
@@ -26,19 +45,13 @@ class ApifyStorageClient(StorageClient):
         id: str | None = None,
         name: str | None = None,
         alias: str | None = None,
-        configuration: Configuration | None = None,
+        configuration: CrawleeConfiguration | None = None,
     ) -> ApifyDatasetClient:
-        # Import here to avoid circular imports.
-        from apify import Configuration as ApifyConfiguration  # noqa: PLC0415
-
         configuration = configuration or ApifyConfiguration.get_global_configuration()
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(
-            f'Expected "configuration" to be an instance of "apify.Configuration", '
-            f'but got {type(configuration).__name__} instead.'
-        )
+        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
 
     @override
     async def create_kvs_client(
@@ -47,19 +60,13 @@ class ApifyStorageClient(StorageClient):
         id: str | None = None,
         name: str | None = None,
         alias: str | None = None,
-        configuration: Configuration | None = None,
+        configuration: CrawleeConfiguration | None = None,
     ) -> ApifyKeyValueStoreClient:
-        # Import here to avoid circular imports.
-        from apify import Configuration as ApifyConfiguration  # noqa: PLC0415
-
         configuration = configuration or ApifyConfiguration.get_global_configuration()
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(
-            f'Expected "configuration" to be an instance of "apify.Configuration", '
-            f'but got {type(configuration).__name__} instead.'
-        )
+        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))
 
     @override
     async def create_rq_client(
@@ -68,16 +75,10 @@ class ApifyStorageClient(StorageClient):
         id: str | None = None,
         name: str | None = None,
         alias: str | None = None,
-        configuration: Configuration | None = None,
+        configuration: CrawleeConfiguration | None = None,
     ) -> ApifyRequestQueueClient:
-        # Import here to avoid circular imports.
-        from apify import Configuration as ApifyConfiguration  # noqa: PLC0415
-
         configuration = configuration or ApifyConfiguration.get_global_configuration()
         if isinstance(configuration, ApifyConfiguration):
             return await ApifyRequestQueueClient.open(id=id, name=name, alias=alias, configuration=configuration)
 
-        raise TypeError(
-            f'Expected "configuration" to be an instance of "apify.Configuration", '
-            f'but got {type(configuration).__name__} instead.'
-        )
+        raise TypeError(self._lsp_violation_error_message_template.format(type(configuration).__name__))

apify/storage_clients/_apify/_utils.py

@@ -1,117 +1,167 @@
 from __future__ import annotations
 
+import logging
+from asyncio import Lock
 from logging import getLogger
-from typing import TYPE_CHECKING, Literal
+from typing import TYPE_CHECKING, ClassVar
 
 from apify_client import ApifyClientAsync
+from crawlee._utils.crypto import compute_short_hash
+
+from apify._configuration import Configuration
 
 if TYPE_CHECKING:
+    from types import TracebackType
+
     from apify_client.clients import KeyValueStoreClientAsync
+    from crawlee.storages import Dataset, KeyValueStore, RequestQueue
 
-    from apify import Configuration
 
 logger = getLogger(__name__)
 
-_ALIAS_MAPPING_KEY = '__STORAGE_ALIASES_MAPPING'
-
-
-async def resolve_alias_to_id(
-    alias: str,
-    storage_type: Literal['dataset', 'kvs', 'rq'],
-    configuration: Configuration,
-) -> str | None:
-    """Resolve a storage alias to its corresponding storage ID.
-
-    Args:
-        alias: The alias to resolve.
-        storage_type: Type of storage ('dataset', 'key_value_store', or 'request_queue').
-        configuration: The configuration object containing API credentials.
-
-    Returns:
-        The storage ID if found, None if the alias doesn't exist.
-    """
-    default_kvs_client = await _get_default_kvs_client(configuration)
-
-    # Create the dictionary key for this alias.
-    alias_key = f'alias-{storage_type}-{alias}'
-
-    try:
-        record = await default_kvs_client.get_record(_ALIAS_MAPPING_KEY)
-
-        # get_record can return {key: ..., value: ..., content_type: ...}
-        if isinstance(record, dict) and 'value' in record:
-            record = record['value']
-
-        # Extract the actual data from the KVS record
-        if isinstance(record, dict) and alias_key in record:
-            storage_id = record[alias_key]
-            return str(storage_id)
 
-    except Exception as exc:
-        # If there's any error accessing the record, treat it as not found.
-        logger.warning(f'Error accessing alias mapping for {alias}: {exc}')
+class AliasResolver:
+    """Class for handling aliases.
 
-    return None
-
-
-async def store_alias_mapping(
-    alias: str,
-    storage_type: Literal['dataset', 'kvs', 'rq'],
-    storage_id: str,
-    configuration: Configuration,
-) -> None:
-    """Store a mapping from alias to storage ID in the default key-value store.
-
-    Args:
-        alias: The alias to store.
-        storage_type: Type of storage ('dataset', 'key_value_store', or 'request_queue').
-        storage_id: The storage ID to map the alias to.
-        configuration: The configuration object containing API credentials.
+    The purpose of this is class is to ensure that alias storages are created with correct id. This is achieved by using
+    default kvs as a storage for global mapping of aliases to storage ids. Same mapping is also kept in memory to avoid
+    unnecessary calls to API and also have limited support of alias storages when not running on Apify platform. When on
+     Apify platform, the storages created with alias are accessible by the same alias even after migration or reboot.
     """
-    default_kvs_client = await _get_default_kvs_client(configuration)
-
-    # Create the dictionary key for this alias.
-    alias_key = f'alias-{storage_type}-{alias}'
-
-    try:
-        record = await default_kvs_client.get_record(_ALIAS_MAPPING_KEY)
-
-        # get_record can return {key: ..., value: ..., content_type: ...}
-        if isinstance(record, dict) and 'value' in record:
-            record = record['value']
 
-        # Update or create the record with the new alias mapping
-        if isinstance(record, dict):
-            record[alias_key] = storage_id
-        else:
-            record = {alias_key: storage_id}
-
-        # Store the mapping back in the KVS.
-        await default_kvs_client.set_record(_ALIAS_MAPPING_KEY, record)
-    except Exception as exc:
-        logger.warning(f'Error accessing alias mapping for {alias}: {exc}')
-
-
-async def _get_default_kvs_client(configuration: Configuration) -> KeyValueStoreClientAsync:
+    _alias_map: ClassVar[dict[str, str]] = {}
+    """Map containing pre-existing alias storages and their ids. Global for all instances."""
+    _alias_init_lock: Lock | None = None
+    """Lock for creating alias storages. Only one alias storage can be created at the time. Global for all instances."""
+
+    _ALIAS_STORAGE_KEY_SEPARATOR = ','
+    _ALIAS_MAPPING_KEY = '__STORAGE_ALIASES_MAPPING'
+
+    def __init__(
+        self, storage_type: type[Dataset | KeyValueStore | RequestQueue], alias: str, configuration: Configuration
+    ) -> None:
+        self._storage_type = storage_type
+        self._alias = alias
+        self._additional_cache_key = hash_api_base_url_and_token(configuration)
+
+    async def __aenter__(self) -> AliasResolver:
+        """Context manager to prevent race condition in alias creation."""
+        lock = await self._get_alias_init_lock()
+        await lock.acquire()
+        return self
+
+    async def __aexit__(
+        self, exc_type: type[BaseException] | None, exc_value: BaseException | None, exc_traceback: TracebackType | None
+    ) -> None:
+        lock = await self._get_alias_init_lock()
+        lock.release()
+
+    @classmethod
+    async def _get_alias_init_lock(cls) -> Lock:
+        """Get lock for controlling the creation of the alias storages.
+
+        The lock is shared for all instances of the AliasResolver class.
+        It is created in async method to ensure that some event loop is already running.
+        """
+        if cls._alias_init_lock is None:
+            cls._alias_init_lock = Lock()
+        return cls._alias_init_lock
+
+    @classmethod
+    async def _get_alias_map(cls) -> dict[str, str]:
+        """Get the aliases and storage ids mapping from the default kvs.
+
+        Mapping is loaded from kvs only once and is shared for all instances of the AliasResolver class.
+
+        Returns:
+            Map of aliases and storage ids.
+        """
+        if not cls._alias_map:
+            default_kvs_client = await _get_default_kvs_client()
+
+            record = await default_kvs_client.get_record(cls._ALIAS_MAPPING_KEY)
+
+            # get_record can return {key: ..., value: ..., content_type: ...}
+            if isinstance(record, dict):
+                if 'value' in record and isinstance(record['value'], dict):
+                    cls._alias_map = record['value']
+                else:
+                    cls._alias_map = record
+            else:
+                cls._alias_map = dict[str, str]()
+
+        return cls._alias_map
+
+    async def resolve_id(self) -> str | None:
+        """Get id of the aliased storage.
+
+        Either locate the id in the in-memory mapping or create the new storage.
+
+        Returns:
+            Storage id if it exists, None otherwise.
+        """
+        return (await self._get_alias_map()).get(self._storage_key, None)
+
+    async def store_mapping(self, storage_id: str) -> None:
+        """Add alias and related storage id to the mapping in default kvs and local in-memory mapping."""
+        # Update in-memory mapping
+        (await self._get_alias_map())[self._storage_key] = storage_id
+        if not Configuration.get_global_configuration().is_at_home:
+            logging.getLogger(__name__).warning(
+                'AliasResolver storage limited retention is only supported on Apify platform. Storage is not exported.'
+            )
+            return
+
+        default_kvs_client = await _get_default_kvs_client()
+        await default_kvs_client.get()
+
+        try:
+            record = await default_kvs_client.get_record(self._ALIAS_MAPPING_KEY)
+
+            # get_record can return {key: ..., value: ..., content_type: ...}
+            if isinstance(record, dict) and 'value' in record:
+                record = record['value']
+
+            # Update or create the record with the new alias mapping
+            if isinstance(record, dict):
+                record[self._storage_key] = storage_id
+            else:
+                record = {self._storage_key: storage_id}
+
+            # Store the mapping back in the KVS.
+            await default_kvs_client.set_record(self._ALIAS_MAPPING_KEY, record)
+        except Exception as exc:
+            logger.warning(f'Error storing alias mapping for {self._alias}: {exc}')
+
+    @property
+    def _storage_key(self) -> str:
+        """Get a unique storage key used for storing the alias in the mapping."""
+        return self._ALIAS_STORAGE_KEY_SEPARATOR.join(
+            [
+                self._storage_type.__name__,
+                self._alias,
+                self._additional_cache_key,
+            ]
+        )
+
+
+async def _get_default_kvs_client() -> KeyValueStoreClientAsync:
     """Get a client for the default key-value store."""
-    token = configuration.token
-    if not token:
-        raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
+    configuration = Configuration.get_global_configuration()
 
-    api_url = configuration.api_base_url
-    if not api_url:
-        raise ValueError(f'Apify storage client requires a valid API URL in Configuration (api_url={api_url}).')
-
-    # Create Apify client with the provided token and API URL
     apify_client_async = ApifyClientAsync(
-        token=token,
-        api_url=api_url,
+        token=configuration.token,
+        api_url=configuration.api_base_url,
         max_retries=8,
         min_delay_between_retries_millis=500,
         timeout_secs=360,
     )
 
-    # Get the default key-value store ID from configuration
-    default_kvs_id = configuration.default_key_value_store_id
+    return apify_client_async.key_value_store(key_value_store_id=configuration.default_key_value_store_id)
+
 
-    return apify_client_async.key_value_store(key_value_store_id=default_kvs_id)
+def hash_api_base_url_and_token(configuration: Configuration) -> str:
+    """Hash configuration.api_public_base_url and configuration.token in deterministic way."""
+    if configuration.api_public_base_url is None or configuration.token is None:
+        raise ValueError("'Configuration.api_public_base_url' and 'Configuration.token' must be set.")
+    return compute_short_hash(f'{configuration.api_public_base_url}{configuration.token}'.encode())

{apify-2.7.1b17.dist-info → apify-2.7.1b18.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: apify
-Version: 2.7.1b17
+Version: 2.7.1b18
 Summary: Apify SDK for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://docs.apify.com/sdk/python/docs/changelog
@@ -228,7 +228,7 @@ Requires-Python: >=3.10
 Requires-Dist: apify-client<3.0.0,>=2.0.0
 Requires-Dist: apify-shared<3.0.0,>=2.0.0
 Requires-Dist: cachetools>=5.5.0
-Requires-Dist: crawlee==0.6.13b37
+Requires-Dist: crawlee==0.6.13b42
 Requires-Dist: cryptography>=42.0.0
 Requires-Dist: impit>=0.6.1
 Requires-Dist: lazy-object-proxy>=1.11.0

{apify-2.7.1b17.dist-info → apify-2.7.1b18.dist-info}/RECORD

@@ -1,7 +1,7 @@
 apify/__init__.py,sha256=HpgKg2FZWJuSPfDygzJ62psylhw4NN4tKFnoYUIhcd4,838
-apify/_actor.py,sha256=i0pWWcysaLCZ64fJH9dznbpWF59BxG67QQjm3w0OO3s,53322
+apify/_actor.py,sha256=2Y_e7t2zlkZbuW7rQME7WdC4kjsoZBoCs4KCuowjbQE,56108
 apify/_charging.py,sha256=KjZ2DnEMS0Tt8ibizmmt0RwBq8FOAsD1z-hKFgdazcY,13143
-apify/_configuration.py,sha256=Ta-qPlKenLaI0IOlODg_A4ZwmPtnuS7OWLSJyexcqmA,13283
+apify/_configuration.py,sha256=DOKRjDGE2qU7LVPa5VZJzLShKGlkr7_207UOzfCAk_U,14676
 apify/_consts.py,sha256=CjhyEJ4Mi0lcIrzfqz8dN7nPJWGjCeBrrXQy1PZ6zRI,440
 apify/_crypto.py,sha256=tqUs13QkemDtGzvU41pIA2HUEawpDlgzqbwKjm4I8kM,6852
 apify/_models.py,sha256=EzU-inWeJ7T5HNVYEwnYb79W-q4OAPhtrYctfRYzpTE,7848
@@ -35,19 +35,19 @@ apify/scrapy/pipelines/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSu
 apify/storage_clients/__init__.py,sha256=9WLAKs2GnnP0yyKR0mc3AfJ1IqXF48V3KPMp6KaB8kU,277
 apify/storage_clients/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/storage_clients/_apify/__init__.py,sha256=mtbVDsxqWL3kx30elnh0kAn2kZ4s3BBsWa15Y5e7RMU,347
-apify/storage_clients/_apify/_dataset_client.py,sha256=tyx5QKbhyF9s2Y2gByVS2Pm3oE-MTvli6b8bkwQzMtk,12004
-apify/storage_clients/_apify/_key_value_store_client.py,sha256=gFt7AFIE6oAJ-g2rx7CG3r7ZWD0URUXqaw4Tk5zES0M,9916
+apify/storage_clients/_apify/_dataset_client.py,sha256=aKQsoIDLbuP6R8wo2DX6pzp-EHV5EWZlHWB8s8OLuG4,12096
+apify/storage_clients/_apify/_key_value_store_client.py,sha256=EWcWE6HiXBGklCAilj4My3T95py7Nu6Zt3e_XzpySUM,10099
 apify/storage_clients/_apify/_models.py,sha256=C6FpXswtO6kXE5RUumazm_conzJJS6PrXAGF9XBuDb8,3651
-apify/storage_clients/_apify/_request_queue_client.py,sha256=Ti48ncuVQkrf6RyoROgYy6MzZ2Tsmi_lurm-2UpWWSs,32528
-apify/storage_clients/_apify/_storage_client.py,sha256=iU67040i7Tmb-qoVZGwcgTF_qZUxsGQpAeV_oIBrpNU,3031
-apify/storage_clients/_apify/_utils.py,sha256=5fy0WErl9HFy78qUPw-fS4jgxMFg-NJvJkuP58WGHuY,3992
+apify/storage_clients/_apify/_request_queue_client.py,sha256=4m7Evp0d_f-1rjatmyYIwnYYG3PGtNCBjw5TFiU0H0w,32640
+apify/storage_clients/_apify/_storage_client.py,sha256=7oqn8-7zG7_cruw6jzmRl2htX2rOt-KPTzCRVNCcyTA,3304
+apify/storage_clients/_apify/_utils.py,sha256=1g0oGqHmB-AeNfeMiB33om2bRZ_7wwhpijPXdWRPNsM,6609
 apify/storage_clients/_apify/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 apify/storage_clients/_file_system/__init__.py,sha256=rDbXatXV9wHKPhKTrXDzWnexhTm7sIJQWucMi-P-SD4,130
 apify/storage_clients/_file_system/_key_value_store_client.py,sha256=fnSJ1EIOPCGfcE6e5S3Tux9VbnMVLCJjugkaQoH_9yo,2267
 apify/storage_clients/_file_system/_storage_client.py,sha256=rcwpKYlrWzvlSA2xoxftg-EZAi_iGZ3vOCbu0C5lKDE,1396
 apify/storages/__init__.py,sha256=-9tEYJVabVs_eRVhUehxN58GH0UG8OfuGjGwuDieP2M,122
 apify/storages/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-apify-2.7.1b17.dist-info/METADATA,sha256=7yn182sNjFOwzP5f98YL5Ap18RYwBVDESLflDJdU8OE,22580
-apify-2.7.1b17.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
-apify-2.7.1b17.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
-apify-2.7.1b17.dist-info/RECORD,,
+apify-2.7.1b18.dist-info/METADATA,sha256=JJjECttKjzSG_t9zHcmc5CBlfwV2u3RuknNEOhXOEVY,22580
+apify-2.7.1b18.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+apify-2.7.1b18.dist-info/licenses/LICENSE,sha256=AsFjHssKjj4LGd2ZCqXn6FBzMqcWdjQre1byPPSypVw,11355
+apify-2.7.1b18.dist-info/RECORD,,