crawlee 0.6.13b43__py3-none-any.whl → 1.1.2b4__py3-none-any.whl

crawlee/storage_clients/_base/_storage_client.py

@@ -30,12 +30,13 @@ class StorageClient(ABC):
     (where applicable), and consistent access patterns across all storage types it supports.
     """
 
-    def get_additional_cache_key(self, configuration: Configuration) -> Hashable:  # noqa: ARG002
-        """Return a cache key that can differentiate between different storages of this client.
+    def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable:  # noqa: ARG002
+        """Return a cache key that can differentiate between different storages of this and other clients.
 
-        Can be based on configuration or on the client itself. By default, returns an empty string.
+        Can be based on configuration or on the client itself. By default, returns a module and name of the client
+        class.
         """
-        return ''
+        return f'{self.__class__.__module__}.{self.__class__.__name__}'
 
     @abstractmethod
     async def create_dataset_client(

crawlee/storage_clients/_file_system/_dataset_client.py

@@ -9,11 +9,12 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from pydantic import ValidationError
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import atomic_write, json_dumps
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
@@ -93,7 +94,7 @@ class FileSystemDatasetClient(DatasetClient):
         name: str | None,
         alias: str | None,
         configuration: Configuration,
-    ) -> FileSystemDatasetClient:
+    ) -> Self:
         """Open or create a file system dataset client.
 
         This method attempts to open an existing dataset from the file system. If a dataset with the specified ID
@@ -114,9 +115,7 @@ class FileSystemDatasetClient(DatasetClient):
                 or if both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         dataset_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 
@@ -135,7 +134,7 @@ class FileSystemDatasetClient(DatasetClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = DatasetMetadata(**file_content)
@@ -164,7 +163,7 @@ class FileSystemDatasetClient(DatasetClient):
 
             # If the dataset directory exists, reconstruct the client from the metadata file.
             if path_to_dataset.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:

crawlee/storage_clients/_file_system/_key_value_store_client.py

@@ -10,11 +10,12 @@ from pathlib import Path
 from typing import TYPE_CHECKING, Any
 
 from pydantic import ValidationError
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import atomic_write, infer_mime_type, json_dumps
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
@@ -92,7 +93,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
         name: str | None,
         alias: str | None,
         configuration: Configuration,
-    ) -> FileSystemKeyValueStoreClient:
+    ) -> Self:
         """Open or create a file system key-value store client.
 
         This method attempts to open an existing key-value store from the file system. If a KVS with the specified
@@ -113,9 +114,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
                 or if both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         kvs_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 
@@ -134,7 +133,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = KeyValueStoreMetadata(**file_content)
@@ -163,7 +162,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
 
             # If the key-value store directory exists, reconstruct the client from the metadata file.
             if path_to_kvs.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:
@@ -240,7 +239,7 @@ class FileSystemKeyValueStoreClient(KeyValueStoreClient):
         # Read the metadata file
         async with self._lock:
             try:
-                file = await asyncio.to_thread(open, record_metadata_filepath)
+                file = await asyncio.to_thread(open, record_metadata_filepath, 'r', encoding='utf-8')
             except FileNotFoundError:
                 logger.warning(f'Metadata file disappeared for key "{key}", aborting get_value')
                 return None

crawlee/storage_clients/_file_system/_request_queue_client.py

@@ -11,12 +11,13 @@ from pathlib import Path
 from typing import TYPE_CHECKING
 
 from pydantic import BaseModel, ValidationError
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee import Request
 from crawlee._consts import METADATA_FILENAME
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import atomic_write, json_dumps
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee._utils.recoverable_state import RecoverableState
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import (
@@ -30,6 +31,7 @@ if TYPE_CHECKING:
     from collections.abc import Sequence
 
     from crawlee.configuration import Configuration
+    from crawlee.storages import KeyValueStore
 
 logger = getLogger(__name__)
 
@@ -91,6 +93,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         metadata: RequestQueueMetadata,
         path_to_rq: Path,
         lock: asyncio.Lock,
+        recoverable_state: RecoverableState[RequestQueueState],
     ) -> None:
         """Initialize a new instance.
 
@@ -113,13 +116,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         self._is_empty_cache: bool | None = None
         """Cache for is_empty result: None means unknown, True/False is cached state."""
 
-        self._state = RecoverableState[RequestQueueState](
-            default_state=RequestQueueState(),
-            persist_state_key='request_queue_state',
-            persistence_enabled=True,
-            persist_state_kvs_name=f'__RQ_STATE_{self._metadata.id}',
-            logger=logger,
-        )
+        self._state = recoverable_state
         """Recoverable state to maintain request ordering, in-progress status, and handled status."""
 
     @override
@@ -136,6 +133,22 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         """The full path to the request queue metadata file."""
         return self.path_to_rq / METADATA_FILENAME
 
+    @classmethod
+    async def _create_recoverable_state(cls, id: str, configuration: Configuration) -> RecoverableState:
+        async def kvs_factory() -> KeyValueStore:
+            from crawlee.storage_clients import FileSystemStorageClient  # noqa: PLC0415 avoid circular import
+            from crawlee.storages import KeyValueStore  # noqa: PLC0415 avoid circular import
+
+            return await KeyValueStore.open(storage_client=FileSystemStorageClient(), configuration=configuration)
+
+        return RecoverableState[RequestQueueState](
+            default_state=RequestQueueState(),
+            persist_state_key=f'__RQ_STATE_{id}',
+            persist_state_kvs_factory=kvs_factory,
+            persistence_enabled=True,
+            logger=logger,
+        )
+
     @classmethod
     async def open(
         cls,
@@ -144,7 +157,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         name: str | None,
         alias: str | None,
         configuration: Configuration,
-    ) -> FileSystemRequestQueueClient:
+    ) -> Self:
         """Open or create a file system request queue client.
 
         This method attempts to open an existing request queue from the file system. If a queue with the specified
@@ -165,9 +178,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                 or if both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         rq_base_path = Path(configuration.storage_dir) / cls._STORAGE_SUBDIR
 
@@ -186,7 +197,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                     continue
 
                 try:
-                    file = await asyncio.to_thread(path_to_metadata.open)
+                    file = await asyncio.to_thread(path_to_metadata.open, 'r', encoding='utf-8')
                     try:
                         file_content = json.load(file)
                         metadata = RequestQueueMetadata(**file_content)
@@ -196,6 +207,9 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                                 metadata=metadata,
                                 path_to_rq=rq_base_path / rq_dir,
                                 lock=asyncio.Lock(),
+                                recoverable_state=await cls._create_recoverable_state(
+                                    id=id, configuration=configuration
+                                ),
                             )
                             await client._state.initialize()
                             await client._discover_existing_requests()
@@ -218,7 +232,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
 
             # If the RQ directory exists, reconstruct the client from the metadata file.
             if path_to_rq.exists() and path_to_metadata.exists():
-                file = await asyncio.to_thread(open, path_to_metadata)
+                file = await asyncio.to_thread(open, path_to_metadata, 'r', encoding='utf-8')
                 try:
                     file_content = json.load(file)
                 finally:
@@ -232,6 +246,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                     metadata=metadata,
                     path_to_rq=path_to_rq,
                     lock=asyncio.Lock(),
+                    recoverable_state=await cls._create_recoverable_state(id=metadata.id, configuration=configuration),
                 )
 
                 await client._state.initialize()
@@ -256,6 +271,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
                     metadata=metadata,
                     path_to_rq=path_to_rq,
                     lock=asyncio.Lock(),
+                    recoverable_state=await cls._create_recoverable_state(id=metadata.id, configuration=configuration),
                 )
                 await client._state.initialize()
                 await client._update_metadata()
@@ -759,7 +775,7 @@ class FileSystemRequestQueueClient(RequestQueueClient):
         """
         # Open the request file.
         try:
-            file = await asyncio.to_thread(open, file_path)
+            file = await asyncio.to_thread(open, file_path, 'r', encoding='utf-8')
         except FileNotFoundError:
             logger.warning(f'Request file "{file_path}" not found.')
             return None

crawlee/storage_clients/_file_system/_storage_client.py

@@ -35,9 +35,9 @@ class FileSystemStorageClient(StorageClient):
     """
 
     @override
-    def get_additional_cache_key(self, configuration: Configuration) -> Hashable:
+    def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable:
         # Even different client instances should return same storage if the storage_dir is the same.
-        return configuration.storage_dir
+        return super().get_storage_client_cache_key(configuration), configuration.storage_dir
 
     @override
     async def create_dataset_client(

crawlee/storage_clients/_memory/_dataset_client.py

@@ -4,9 +4,10 @@ from datetime import datetime, timezone
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
@@ -54,7 +55,7 @@ class MemoryDatasetClient(DatasetClient):
         id: str | None,
         name: str | None,
         alias: str | None,
-    ) -> MemoryDatasetClient:
+    ) -> Self:
         """Open or create a new memory dataset client.
 
         This method creates a new in-memory dataset instance. Unlike persistent storage implementations, memory
@@ -76,9 +77,7 @@ class MemoryDatasetClient(DatasetClient):
             ValueError: If both name and alias are provided, or if neither id, name, nor alias is provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         # Create a new dataset
         dataset_id = id or crypto_random_object_id()

crawlee/storage_clients/_memory/_key_value_store_client.py

@@ -4,10 +4,11 @@ import sys
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import infer_mime_type
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
@@ -52,7 +53,7 @@ class MemoryKeyValueStoreClient(KeyValueStoreClient):
         id: str | None,
         name: str | None,
         alias: str | None,
-    ) -> MemoryKeyValueStoreClient:
+    ) -> Self:
         """Open or create a new memory key-value store client.
 
         This method creates a new in-memory key-value store instance. Unlike persistent storage implementations,
@@ -74,9 +75,7 @@ class MemoryKeyValueStoreClient(KeyValueStoreClient):
             ValueError: If both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         # Create a new key-value store
         store_id = id or crypto_random_object_id()

crawlee/storage_clients/_memory/_request_queue_client.py

@@ -6,10 +6,11 @@ from datetime import datetime, timezone
 from logging import getLogger
 from typing import TYPE_CHECKING
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee import Request
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
 
@@ -64,7 +65,7 @@ class MemoryRequestQueueClient(RequestQueueClient):
         id: str | None,
         name: str | None,
         alias: str | None,
-    ) -> MemoryRequestQueueClient:
+    ) -> Self:
         """Open or create a new memory request queue client.
 
         This method creates a new in-memory request queue instance. Unlike persistent storage implementations,
@@ -86,9 +87,7 @@ class MemoryRequestQueueClient(RequestQueueClient):
             ValueError: If both name and alias are provided.
         """
         # Validate input parameters.
-        specified_params = sum(1 for param in [id, name, alias] if param is not None)
-        if specified_params > 1:
-            raise ValueError('Only one of "id", "name", or "alias" can be specified, not multiple.')
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
 
         # Create a new queue
         queue_id = id or crypto_random_object_id()

crawlee/storage_clients/_redis/__init__.py

@@ -0,0 +1,6 @@
+from ._dataset_client import RedisDatasetClient
+from ._key_value_store_client import RedisKeyValueStoreClient
+from ._request_queue_client import RedisRequestQueueClient
+from ._storage_client import RedisStorageClient
+
+__all__ = ['RedisDatasetClient', 'RedisKeyValueStoreClient', 'RedisRequestQueueClient', 'RedisStorageClient']