apify 3.0.0rc1__py3-none-any.whl → 3.0.1__py3-none-any.whl

apify/_configuration.py

@@ -3,11 +3,13 @@ from __future__ import annotations
 from datetime import datetime, timedelta
 from decimal import Decimal
 from logging import getLogger
+from pathlib import Path
 from typing import Annotated, Any
 
 from pydantic import AliasChoices, BeforeValidator, Field, model_validator
 from typing_extensions import Self, deprecated
 
+from crawlee import service_locator
 from crawlee._utils.models import timedelta_ms
 from crawlee._utils.urls import validate_http_url
 from crawlee.configuration import Configuration as CrawleeConfiguration
@@ -141,7 +143,7 @@ class Configuration(CrawleeConfiguration):
     ] = None
 
     default_dataset_id: Annotated[
-        str,
+        str | None,
         Field(
             validation_alias=AliasChoices(
                 'actor_default_dataset_id',
@@ -149,10 +151,10 @@ class Configuration(CrawleeConfiguration):
             ),
             description='Default dataset ID used by the Apify storage client when no ID or name is provided.',
         ),
-    ] = 'default'
+    ] = None
 
     default_key_value_store_id: Annotated[
-        str,
+        str | None,
         Field(
             validation_alias=AliasChoices(
                 'actor_default_key_value_store_id',
@@ -160,10 +162,10 @@ class Configuration(CrawleeConfiguration):
             ),
             description='Default key-value store ID for the Apify storage client when no ID or name is provided.',
         ),
-    ] = 'default'
+    ] = None
 
     default_request_queue_id: Annotated[
-        str,
+        str | None,
         Field(
             validation_alias=AliasChoices(
                 'actor_default_request_queue_id',
@@ -171,7 +173,7 @@ class Configuration(CrawleeConfiguration):
             ),
             description='Default request queue ID for the Apify storage client when no ID or name is provided.',
         ),
-    ] = 'default'
+    ] = None
 
     disable_outdated_warning: Annotated[
         bool,
@@ -420,15 +422,53 @@ class Configuration(CrawleeConfiguration):
             logger.warning('Actor is running on the Apify platform, `disable_browser_sandbox` was changed to True.')
         return self
 
+    @property
+    def canonical_input_key(self) -> str:
+        return str(Path(self.input_key).with_suffix('.json'))
+
+    @property
+    def input_key_candidates(self) -> set[str]:
+        return {self.input_key, self.canonical_input_key, Path(self.canonical_input_key).stem}
+
     @classmethod
     def get_global_configuration(cls) -> Configuration:
         """Retrieve the global instance of the configuration.
 
-        Mostly for the backwards compatibility. It is recommended to use the `service_locator.get_configuration()`
-        instead.
+        This method ensures that ApifyConfigration is returned, even if CrawleeConfiguration was set in the
+        service locator.
+        """
+        global_configuration = service_locator.get_configuration()
+
+        if isinstance(global_configuration, Configuration):
+            # If Apify configuration was already stored in service locator, return it.
+            return global_configuration
+
+        logger.warning(
+            'Non Apify Configration is set in the `service_locator` in the SDK context. '
+            'It is recommended to set `apify.Configuration` explicitly as early as possible by using '
+            'service_locator.set_configuration'
+        )
+
+        return cls.from_configuration(global_configuration)
+
+    @classmethod
+    def from_configuration(cls, configuration: CrawleeConfiguration) -> Configuration:
+        """Create Apify Configuration from existing Crawlee Configuration.
+
+        Args:
+            configuration: The existing Crawlee Configuration.
+
+        Returns:
+            The created Apify Configuration.
         """
-        return cls()
+        apify_configuration = cls()
 
+        # Ensure the returned configuration is of type Apify Configuration.
+        # Most likely crawlee configuration was already set. Create Apify configuration from it.
+        # Due to known Pydantic issue https://github.com/pydantic/pydantic/issues/9516, creating new instance of
+        # Configuration from existing one in situation where environment can have some fields set by alias is very
+        # unpredictable. Use the stable workaround.
+        for name in configuration.model_fields:
+            setattr(apify_configuration, name, getattr(configuration, name))
 
-# Monkey-patch the base class so that it works with the extended configuration
-CrawleeConfiguration.get_global_configuration = Configuration.get_global_configuration  # type: ignore[method-assign]
+        return apify_configuration

apify/events/__init__.py

@@ -1,5 +1,5 @@
-from crawlee.events import EventManager, LocalEventManager
+from crawlee.events import Event, EventManager, LocalEventManager
 
 from ._apify_event_manager import ApifyEventManager
 
-__all__ = ['ApifyEventManager', 'EventManager', 'LocalEventManager']
+__all__ = ['ApifyEventManager', 'Event', 'EventManager', 'LocalEventManager']

apify/storage_clients/__init__.py

@@ -2,9 +2,11 @@ from crawlee.storage_clients import MemoryStorageClient
 
 from ._apify import ApifyStorageClient
 from ._file_system import ApifyFileSystemStorageClient as FileSystemStorageClient
+from ._smart_apify import SmartApifyStorageClient
 
 __all__ = [
     'ApifyStorageClient',
     'FileSystemStorageClient',
     'MemoryStorageClient',
+    'SmartApifyStorageClient',
 ]

apify/storage_clients/_apify/_dataset_client.py

@@ -11,6 +11,9 @@ from crawlee._utils.byte_size import ByteSize
 from crawlee._utils.file import json_dumps
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
+from crawlee.storages import Dataset
+
+from ._utils import AliasResolver
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -66,6 +69,7 @@ class ApifyDatasetClient(DatasetClient):
         *,
         id: str | None,
         name: str | None,
+        alias: str | None,
         configuration: Configuration,
     ) -> ApifyDatasetClient:
         """Open an Apify dataset client.
@@ -74,22 +78,27 @@ class ApifyDatasetClient(DatasetClient):
         It handles authentication, storage lookup/creation, and metadata retrieval.
 
         Args:
-            id: The ID of an existing dataset to open. If provided, the client will connect to this specific storage.
-                Cannot be used together with `name`.
-            name: The name of a dataset to get or create. If a storage with this name exists, it will be opened;
-                otherwise, a new one will be created. Cannot be used together with `id`.
+            id: The ID of the dataset to open. If provided, searches for existing dataset by ID.
+                Mutually exclusive with name and alias.
+            name: The name of the dataset to open (global scope, persists across runs).
+                Mutually exclusive with id and alias.
+            alias: The alias of the dataset to open (run scope, creates unnamed storage).
+                Mutually exclusive with id and name.
             configuration: The configuration object containing API credentials and settings. Must include a valid
                 `token` and `api_base_url`. May also contain a `default_dataset_id` for fallback when neither
-                `id` nor `name` is provided.
+                `id`, `name`, nor `alias` is provided.
 
         Returns:
             An instance for the opened or created storage client.
 
         Raises:
-            ValueError: If the configuration is missing required fields (token, api_base_url), if both `id` and `name`
-                are provided, or if neither `id` nor `name` is provided and no default storage ID is available in
-                the configuration.
+            ValueError: If the configuration is missing required fields (token, api_base_url), if more than one of
+                `id`, `name`, or `alias` is provided, or if none are provided and no default storage ID is available
+                in the configuration.
         """
+        if sum(1 for param in [id, name, alias] if param is not None) > 1:
+            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+
         token = configuration.token
         if not token:
             raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
@@ -115,27 +124,42 @@ class ApifyDatasetClient(DatasetClient):
         )
         apify_datasets_client = apify_client_async.datasets()
 
-        # If both id and name are provided, raise an error.
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
-        # If id is provided, get the storage by ID.
-        if id and name is None:
-            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
+        # Normalize unnamed default storage in cases where not defined in `configuration.default_dataset_id` to unnamed
+        # storage aliased as `__default__`
+        if not any([alias, name, id, configuration.default_dataset_id]):
+            alias = '__default__'
+
+        if alias:
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with AliasResolver(storage_type=Dataset, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    new_storage_metadata = DatasetMetadata.model_validate(
+                        await apify_datasets_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
-        if name and id is None:
+        elif name:
             id = DatasetMetadata.model_validate(
                 await apify_datasets_client.get_or_create(name=name),
             ).id
-            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
-        # If both id and name are None, try to get the default storage ID from environment variables.
-        # The default storage ID environment variable is set by the Apify platform. It also contains
-        # a new storage ID after Actor's reboot or migration.
-        if id is None and name is None:
+        # If none are provided, try to get the default storage ID from environment variables.
+        elif id is None:
             id = configuration.default_dataset_id
-            apify_dataset_client = apify_client_async.dataset(dataset_id=id)
+            if not id:
+                raise ValueError(
+                    'Dataset "id", "name", or "alias" must be specified, '
+                    'or a default dataset ID must be set in the configuration.'
+                )
+
+        # Now create the client for the determined ID
+        apify_dataset_client = apify_client_async.dataset(dataset_id=id)
 
         # Fetch its metadata.
         metadata = await apify_dataset_client.get()
@@ -150,7 +174,7 @@ class ApifyDatasetClient(DatasetClient):
         # Verify that the storage exists by fetching its metadata again.
         metadata = await apify_dataset_client.get()
         if metadata is None:
-            raise ValueError(f'Opening dataset with id={id} and name={name} failed.')
+            raise ValueError(f'Opening dataset with id={id}, name={name}, and alias={alias} failed.')
 
         return cls(
             api_client=apify_dataset_client,

apify/storage_clients/_apify/_key_value_store_client.py

@@ -10,8 +10,10 @@ from yarl import URL
 from apify_client import ApifyClientAsync
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreRecord, KeyValueStoreRecordMetadata
+from crawlee.storages import KeyValueStore
 
 from ._models import ApifyKeyValueStoreMetadata, KeyValueStoreListKeysPage
+from ._utils import AliasResolver
 from apify._crypto import create_hmac_signature
 
 if TYPE_CHECKING:
@@ -58,6 +60,7 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         *,
         id: str | None,
         name: str | None,
+        alias: str | None,
         configuration: Configuration,
     ) -> ApifyKeyValueStoreClient:
         """Open an Apify key-value store client.
@@ -66,22 +69,27 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         It handles authentication, storage lookup/creation, and metadata retrieval.
 
         Args:
-            id: The ID of an existing key-value store to open. If provided, the client will connect to this specific
-                storage. Cannot be used together with `name`.
-            name: The name of a key-value store to get or create. If a storage with this name exists, it will be
-                opened; otherwise, a new one will be created. Cannot be used together with `id`.
+            id: The ID of the KVS to open. If provided, searches for existing KVS by ID.
+                Mutually exclusive with name and alias.
+            name: The name of the KVS to open (global scope, persists across runs).
+                Mutually exclusive with id and alias.
+            alias: The alias of the KVS to open (run scope, creates unnamed storage).
+                Mutually exclusive with id and name.
             configuration: The configuration object containing API credentials and settings. Must include a valid
                 `token` and `api_base_url`. May also contain a `default_key_value_store_id` for fallback when
-                neither `id` nor `name` is provided.
+                neither `id`, `name`, nor `alias` is provided.
 
         Returns:
             An instance for the opened or created storage client.
 
         Raises:
-            ValueError: If the configuration is missing required fields (token, api_base_url), if both `id` and `name`
-                are provided, or if neither `id` nor `name` is provided and no default storage ID is available
+            ValueError: If the configuration is missing required fields (token, api_base_url), if more than one of
+                `id`, `name`, or `alias` is provided, or if none are provided and no default storage ID is available
                 in the configuration.
         """
+        if sum(1 for param in [id, name, alias] if param is not None) > 1:
+            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+
         token = configuration.token
         if not token:
             raise ValueError(f'Apify storage client requires a valid token in Configuration (token={token}).')
@@ -107,27 +115,43 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         )
         apify_kvss_client = apify_client_async.key_value_stores()
 
-        # If both id and name are provided, raise an error.
-        if id and name:
-            raise ValueError('Only one of "id" or "name" can be specified, not both.')
-
-        # If id is provided, get the storage by ID.
-        if id and name is None:
-            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
+        # Normalize unnamed default storage in cases where not defined in `configuration.default_key_value_store_id` to
+        # unnamed storage aliased as `__default__`
+        if not any([alias, name, id, configuration.default_key_value_store_id]):
+            alias = '__default__'
+
+        if alias:
+            # Check if there is pre-existing alias mapping in the default KVS.
+            async with AliasResolver(storage_type=KeyValueStore, alias=alias, configuration=configuration) as _alias:
+                id = await _alias.resolve_id()
+
+                # There was no pre-existing alias in the mapping.
+                # Create a new unnamed storage and store the mapping.
+                if id is None:
+                    # Create a new storage and store the alias mapping
+                    new_storage_metadata = ApifyKeyValueStoreMetadata.model_validate(
+                        await apify_kvss_client.get_or_create(),
+                    )
+                    id = new_storage_metadata.id
+                    await _alias.store_mapping(storage_id=id)
 
         # If name is provided, get or create the storage by name.
-        if name and id is None:
+        elif name:
             id = ApifyKeyValueStoreMetadata.model_validate(
                 await apify_kvss_client.get_or_create(name=name),
             ).id
-            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
-        # If both id and name are None, try to get the default storage ID from environment variables.
-        # The default storage ID environment variable is set by the Apify platform. It also contains
-        # a new storage ID after Actor's reboot or migration.
-        if id is None and name is None:
+        # If none are provided, try to get the default storage ID from environment variables.
+        elif id is None:
             id = configuration.default_key_value_store_id
-            apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
+            if not id:
+                raise ValueError(
+                    'KeyValueStore "id", "name", or "alias" must be specified, '
+                    'or a default KeyValueStore ID must be set in the configuration.'
+                )
+
+        # Now create the client for the determined ID
+        apify_kvs_client = apify_client_async.key_value_store(key_value_store_id=id)
 
         # Fetch its metadata.
         metadata = await apify_kvs_client.get()
@@ -142,7 +166,7 @@ class ApifyKeyValueStoreClient(KeyValueStoreClient):
         # Verify that the storage exists by fetching its metadata again.
         metadata = await apify_kvs_client.get()
         if metadata is None:
-            raise ValueError(f'Opening key-value store with id={id} and name={name} failed.')
+            raise ValueError(f'Opening key-value store with id={id}, name={name}, and alias={alias} failed.')
 
         return cls(
             api_client=apify_kvs_client,

apify/storage_clients/_apify/_models.py

@@ -5,7 +5,7 @@ from typing import Annotated
 
 from pydantic import BaseModel, ConfigDict, Field
 
-from crawlee.storage_clients.models import KeyValueStoreMetadata
+from crawlee.storage_clients.models import KeyValueStoreMetadata, RequestQueueMetadata
 
 from apify import Request
 from apify._utils import docs_group
@@ -105,3 +105,27 @@ class CachedRequest(BaseModel):
 
     lock_expires_at: datetime | None = None
     """The expiration time of the lock on the request."""
+
+
+class RequestQueueStats(BaseModel):
+    model_config = ConfigDict(populate_by_name=True)
+
+    delete_count: Annotated[int, Field(alias='deleteCount', default=0)]
+    """"The number of request queue deletes."""
+
+    head_item_read_count: Annotated[int, Field(alias='headItemReadCount', default=0)]
+    """The number of request queue head reads."""
+
+    read_count: Annotated[int, Field(alias='readCount', default=0)]
+    """The number of request queue reads."""
+
+    storage_bytes: Annotated[int, Field(alias='storageBytes', default=0)]
+    """Storage size in Bytes."""
+
+    write_count: Annotated[int, Field(alias='writeCount', default=0)]
+    """The number of request queue writes."""
+
+
+class ApifyRequestQueueMetadata(RequestQueueMetadata):
+    stats: Annotated[RequestQueueStats, Field(alias='stats', default_factory=RequestQueueStats)]
+    """Additional statistics about the request queue."""