crawlee 0.6.13b31__py3-none-any.whl → 1.1.1b1__py3-none-any.whl

crawlee/storage_clients/_file_system/_request_queue_client.py

@@ -11,12 +11,13 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 from pydantic import BaseModel, ValidationError
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee import Request
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import atomic_write, json_dumps
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee._utils.recoverable_state import RecoverableState
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import (
@@ -30,6 +31,7 @@ if TYPE_CHECKING:
     from collections.abc import Sequence
 
     from crawlee.configuration import Configuration
+    from crawlee.storages import KeyValueStore
 
 logger = getLogger(__name__)
 
@@ -89,8 +91,9 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         self,
         *,
         metadata: RequestQueueMetadata,
-        storage_dir: Path,
+        path_to_rq: Path,
         lock: asyncio.Lock,
+        recoverable_state: RecoverableState[RequestQueueState],
     ) -> None:
         """Initialize a new instance.
 
@@ -98,8 +101,8 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         """
         self._metadata = metadata
 
-        self._storage_dir = storage_dir
-        """The base directory where the storage data are being persisted."""
+        self._path_to_rq = path_to_rq
+        """The full path to the request queue directory."""
 
         self._lock = lock
         """A lock to ensure that only one operation is performed at a time."""
@@ -113,13 +116,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         self._is_empty_cache: bool | None = None
         """Cache for is_empty result: None means unknown, True/False is cached state."""
 
-        self._state = RecoverableState[RequestQueueState](
-            default_state=RequestQueueState(),
-            persist_state_key='request_queue_state',
-            persistence_enabled=True,
-            persist_state_kvs_name=f'__RQ_STATE_{self._metadata.id}',
-            logger=logger,
-        )
+        self._state = recoverable_state
         """Recoverable state to maintain request ordering, in-progress status, and handled status."""
 
     @override
@@ -129,24 +126,38 @@ class FileSystemRequestQueueClient(RequestQueueClient):
     @property
     def path_to_rq(self) -> Path:
         """The full path to the request queue directory."""
-        if self._metadata.name is None:
-            return self._storage_dir / self._STORAGE_SUBDIR / self._STORAGE_SUBSUBDIR_DEFAULT
-
-        return self._storage_dir / self._STORAGE_SUBDIR / self._metadata.name
+        return self._path_to_rq
 
     @property
     def path_to_metadata(self) -> Path:
         """The full path to the request queue metadata file."""
         return self.path_to_rq / METADATA_FILENAME
 
+    @classmethod
+    async def _create_recoverable_state(cls, id: str, configuration: Configuration) -> RecoverableState:
+        async def kvs_factory() -> KeyValueStore:
+            from crawlee.storage_clients import FileSystemStorageClient  # noqa: PLC0415 avoid circular import
+            from crawlee.storages import KeyValueStore  # noqa: PLC0415 avoid circular import
+
+            return await KeyValueStore.open(storage_client=FileSystemStorageClient(), configuration=configuration)
+
+        return RecoverableState[RequestQueueState](
+            default_state=RequestQueueState(),
+            persist_state_key=f'__RQ_STATE_{id}',
+            persist_state_kvs_factory=kvs_factory,
+            persistence_enabled=True,
+            logger=logger,
+        )
+
     @classmethod
     async def open(
         cls,
         *,
         id: str | None,
         name: str | None,
+        alias: str | None,
         configuration: Configuration,
-    ) -> FileSystemRequestQueueClient:
+    ) -> Self:
         """Open or create a file system request queue client.
 
         This method attempts to open an existing request queue from the file system. If a queue with the specified
@@ -155,17 +166,21 @@ class FileSystemRequestQueueClient(RequestQueueClient):
 
         Args:
             id: The ID of the request queue to open. If provided, searches for existing queue by ID.
-            name: The name of the request queue to open. If not provided, uses the default queue.
+            name: The name of the request queue for named (global scope) storages.
+            alias: The alias of the request queue for unnamed (run scope) storages.
             configuration: The configuration object containing storage directory settings.
 
         Returns:
             An instance for the opened or created storage client.
 
         Raises:
-            ValueError: If a queue with the specified ID is not found, or if metadata is invalid.
+            ValueError: If a queue with the specified ID is not found, if metadata is invalid,
+                or if both name and alias are provided.
         """
-        storage_dir = Path(configuration.storage_dir)
-        rq_base_path = storage_dir / cls._STORAGE_SUBDIR
+        # Validate input parameters.
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+        rq_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 
         if not rq_base_path.exists():
             await asyncio.to_thread(rq_base_path.mkdir, parents=True, exist_ok=True)
@@ -177,12 +192,12 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                 if not rq_dir.is_dir():
                     continue
 
-                metadata_path = rq_dir / METADATA_FILENAME
-                if not metadata_path.exists():
+                path_to_metadata = rq_dir / METADATA_FILENAME
+                if not path_to_metadata.exists():
                     continue
 
                 try:
-                    file = await asyncio.to_thread(metadata_path.open)
+                    file = await asyncio.to_thread(path_to_metadata.open)
                     try:
                         file_content = json.load(file)
                         metadata = RequestQueueMetadata(**file_content)
@@ -190,8 +205,11 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                         if metadata.id == id:
                             client = cls(
                                 metadata=metadata,
-                                storage_dir=storage_dir,
+                                path_to_rq=rq_base_path / rq_dir,
                                 lock=asyncio.Lock(),
+                                recoverable_state=await cls._create_recoverable_state(
+                                    id=id, configuration=configuration
+                                ),
                             )
                             await client._state.initialize()
                             await client._discover_existing_requests()
@@ -206,14 +224,15 @@ class FileSystemRequestQueueClient(RequestQueueClient):
             if not found:
                 raise ValueError(f'Request queue with ID "{id}" not found')
 
-        # Open an existing RQ by its name, or create a new one if not found.
+        # Open an existing RQ by its name or alias, or create a new one if not found.
         else:
-            rq_path = rq_base_path / cls._STORAGE_SUBSUBDIR_DEFAULT if name is None else rq_base_path / name
-            metadata_path = rq_path / METADATA_FILENAME
+            rq_dir = Path(name) if name else Path(alias) if alias else Path('default')
+            path_to_rq = rq_base_path / rq_dir
+            path_to_metadata = path_to_rq / METADATA_FILENAME
 
             # If the RQ directory exists, reconstruct the client from the metadata file.
-            if rq_path.exists() and metadata_path.exists():
-                file = await asyncio.to_thread(open, metadata_path)
+            if path_to_rq.exists() and path_to_metadata.exists():
+                file = await asyncio.to_thread(open, path_to_metadata)
                 try:
                     file_content = json.load(file)
                 finally:
@@ -221,14 +240,13 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                 try:
                     metadata = RequestQueueMetadata(**file_content)
                 except ValidationError as exc:
-                    raise ValueError(f'Invalid metadata file for request queue "{name}"') from exc
-
-                metadata.name = name
+                    raise ValueError(f'Invalid metadata file for request queue "{name or alias}"') from exc
 
                 client = cls(
                     metadata=metadata,
-                    storage_dir=storage_dir,
+                    path_to_rq=path_to_rq,
                     lock=asyncio.Lock(),
+                    recoverable_state=await cls._create_recoverable_state(id=metadata.id, configuration=configuration),
                 )
 
                 await client._state.initialize()
@@ -251,8 +269,9 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                 )
                 client = cls(
                     metadata=metadata,
-                    storage_dir=storage_dir,
+                    path_to_rq=path_to_rq,
                     lock=asyncio.Lock(),
+                    recoverable_state=await cls._create_recoverable_state(id=metadata.id, configuration=configuration),
                 )
                 await client._state.initialize()
                 await client._update_metadata()

crawlee/storage_clients/_file_system/_storage_client.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from typing_extensions import override
 
 from crawlee._utils.docs import docs_group
@@ -10,6 +12,9 @@ from ._dataset_client import FileSystemDatasetClient
 from ._key_value_store_client import FileSystemKeyValueStoreClient
 from ._request_queue_client import FileSystemRequestQueueClient
 
+if TYPE_CHECKING:
+    from collections.abc import Hashable
+
 
 @docs_group('Storage clients')
 class FileSystemStorageClient(StorageClient):
@@ -29,16 +34,22 @@ class FileSystemStorageClient(StorageClient):
     Use it only when running a single crawler process at a time.
     """
 
+    @override
+    def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable:
+        # Even different client instances should return same storage if the storage_dir is the same.
+        return super().get_storage_client_cache_key(configuration), configuration.storage_dir
+
     @override
     async def create_dataset_client(
         self,
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> FileSystemDatasetClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await FileSystemDatasetClient.open(id=id, name=name, configuration=configuration)
+        client = await FileSystemDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -48,10 +59,11 @@ class FileSystemStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> FileSystemKeyValueStoreClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await FileSystemKeyValueStoreClient.open(id=id, name=name, configuration=configuration)
+        client = await FileSystemKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -61,9 +73,10 @@ class FileSystemStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> FileSystemRequestQueueClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await FileSystemRequestQueueClient.open(id=id, name=name, configuration=configuration)
+        client = await FileSystemRequestQueueClient.open(id=id, name=name, alias=alias, configuration=configuration)
         await self._purge_if_needed(client, configuration)
         return client

crawlee/storage_clients/_memory/_dataset_client.py

@@ -4,9 +4,10 @@ from datetime import datetime, timezone
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
@@ -53,21 +54,32 @@ class MemoryDatasetClient(DatasetClient):
         *,
         id: str | None,
         name: str | None,
-    ) -> MemoryDatasetClient:
+        alias: str | None,
+    ) -> Self:
         """Open or create a new memory dataset client.
 
         This method creates a new in-memory dataset instance. Unlike persistent storage implementations, memory
         datasets don't check for existing datasets with the same name or ID since all data exists only in memory
         and is lost when the process terminates.
 
+        Alias does not have any effect on the memory storage client implementation, because unnamed storages
+        are supported by default, since data are not persisted.
+
         Args:
             id: The ID of the dataset. If not provided, a random ID will be generated.
-            name: The name of the dataset. If not provided, the dataset will be unnamed.
+            name: The name of the dataset for named (global scope) storages.
+            alias: The alias of the dataset for unnamed (run scope) storages.
 
         Returns:
             An instance for the opened or created storage client.
+
+        Raises:
+            ValueError: If both name and alias are provided, or if neither id, name, nor alias is provided.
         """
-        # Otherwise create a new dataset
+        # Validate input parameters.
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+        # Create a new dataset
         dataset_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)
 

crawlee/storage_clients/_memory/_key_value_store_client.py

@@ -4,10 +4,11 @@ import sys
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import infer_mime_type
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
@@ -51,21 +52,32 @@ class MemoryKeyValueStoreClient(KeyValueStoreClient):
         *,
         id: str | None,
         name: str | None,
-    ) -> MemoryKeyValueStoreClient:
+        alias: str | None,
+    ) -> Self:
         """Open or create a new memory key-value store client.
 
         This method creates a new in-memory key-value store instance. Unlike persistent storage implementations,
         memory KVS don't check for existing stores with the same name or ID since all data exists only in memory
         and is lost when the process terminates.
 
+        Alias does not have any effect on the memory storage client implementation, because unnamed storages
+        are supported by default, since data are not persisted.
+
         Args:
             id: The ID of the key-value store. If not provided, a random ID will be generated.
-            name: The name of the key-value store. If not provided, the store will be unnamed.
+            name: The name of the key-value store for named (global scope) storages.
+            alias: The alias of the key-value store for unnamed (run scope) storages.
 
         Returns:
             An instance for the opened or created storage client.
+
+        Raises:
+            ValueError: If both name and alias are provided.
         """
-        # Otherwise create a new key-value store
+        # Validate input parameters.
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+        # Create a new key-value store
         store_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)
 

crawlee/storage_clients/_memory/_request_queue_client.py

@@ -6,10 +6,11 @@ from datetime import datetime, timezone
 from logging import getLogger
 from typing import TYPE_CHECKING
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee import Request
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
 
@@ -63,21 +64,32 @@ class MemoryRequestQueueClient(RequestQueueClient):
         *,
         id: str | None,
         name: str | None,
-    ) -> MemoryRequestQueueClient:
+        alias: str | None,
+    ) -> Self:
         """Open or create a new memory request queue client.
 
         This method creates a new in-memory request queue instance. Unlike persistent storage implementations,
         memory queues don't check for existing queues with the same name or ID since all data exists only
         in memory and is lost when the process terminates.
 
+        Alias does not have any effect on the memory storage client implementation, because unnamed storages
+        are supported by default, since data are not persisted.
+
         Args:
             id: The ID of the request queue. If not provided, a random ID will be generated.
-            name: The name of the request queue. If not provided, the queue will be unnamed.
+            name: The name of the request queue for named (global scope) storages.
+            alias: The alias of the request queue for unnamed (run scope) storages.
 
         Returns:
             An instance for the opened or created storage client.
+
+        Raises:
+            ValueError: If both name and alias are provided.
         """
-        # Otherwise create a new queue
+        # Validate input parameters.
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+        # Create a new queue
         queue_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)
 

crawlee/storage_clients/_memory/_storage_client.py

@@ -33,10 +33,11 @@ class MemoryStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> MemoryDatasetClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await MemoryDatasetClient.open(id=id, name=name)
+        client = await MemoryDatasetClient.open(id=id, name=name, alias=alias)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -46,10 +47,11 @@ class MemoryStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> MemoryKeyValueStoreClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await MemoryKeyValueStoreClient.open(id=id, name=name)
+        client = await MemoryKeyValueStoreClient.open(id=id, name=name, alias=alias)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -59,9 +61,10 @@ class MemoryStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> MemoryRequestQueueClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await MemoryRequestQueueClient.open(id=id, name=name)
+        client = await MemoryRequestQueueClient.open(id=id, name=name, alias=alias)
         await self._purge_if_needed(client, configuration)
         return client

crawlee/storage_clients/_redis/__init__.py

@@ -0,0 +1,6 @@
+from ._dataset_client import RedisDatasetClient
+from ._key_value_store_client import RedisKeyValueStoreClient
+from ._request_queue_client import RedisRequestQueueClient
+from ._storage_client import RedisStorageClient
+
+__all__ = ['RedisDatasetClient', 'RedisKeyValueStoreClient', 'RedisRequestQueueClient', 'RedisStorageClient']