crawlee 0.6.13b15__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

crawlee/storage_clients/_file_system/_storage_client.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+from typing import TYPE_CHECKING
+
 from typing_extensions import override
 
 from crawlee._utils.docs import docs_group
@@ -10,6 +12,9 @@ from ._dataset_client import FileSystemDatasetClient
 from ._key_value_store_client import FileSystemKeyValueStoreClient
 from ._request_queue_client import FileSystemRequestQueueClient
 
+if TYPE_CHECKING:
+    from collections.abc import Hashable
+
 
 @docs_group('Storage clients')
 class FileSystemStorageClient(StorageClient):
@@ -29,16 +34,22 @@ class FileSystemStorageClient(StorageClient):
     Use it only when running a single crawler process at a time.
     """
 
+    @override
+    def get_storage_client_cache_key(self, configuration: Configuration) -> Hashable:
+        # Even different client instances should return same storage if the storage_dir is the same.
+        return super().get_storage_client_cache_key(configuration), configuration.storage_dir
+
     @override
     async def create_dataset_client(
         self,
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> FileSystemDatasetClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await FileSystemDatasetClient.open(id=id, name=name, configuration=configuration)
+        client = await FileSystemDatasetClient.open(id=id, name=name, alias=alias, configuration=configuration)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -48,10 +59,11 @@ class FileSystemStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> FileSystemKeyValueStoreClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await FileSystemKeyValueStoreClient.open(id=id, name=name, configuration=configuration)
+        client = await FileSystemKeyValueStoreClient.open(id=id, name=name, alias=alias, configuration=configuration)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -61,9 +73,10 @@ class FileSystemStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> FileSystemRequestQueueClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await FileSystemRequestQueueClient.open(id=id, name=name, configuration=configuration)
+        client = await FileSystemRequestQueueClient.open(id=id, name=name, alias=alias, configuration=configuration)
         await self._purge_if_needed(client, configuration)
         return client

crawlee/storage_clients/_memory/_dataset_client.py

@@ -4,9 +4,10 @@ from datetime import datetime, timezone
 from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
 
@@ -53,21 +54,32 @@ class MemoryDatasetClient(DatasetClient):
         *,
         id: str | None,
         name: str | None,
-    ) -> MemoryDatasetClient:
+        alias: str | None,
+    ) -> Self:
         """Open or create a new memory dataset client.
 
         This method creates a new in-memory dataset instance. Unlike persistent storage implementations, memory
         datasets don't check for existing datasets with the same name or ID since all data exists only in memory
         and is lost when the process terminates.
 
+        Alias does not have any effect on the memory storage client implementation, because unnamed storages
+        are supported by default, since data are not persisted.
+
         Args:
             id: The ID of the dataset. If not provided, a random ID will be generated.
-            name: The name of the dataset. If not provided, the dataset will be unnamed.
+            name: The name of the dataset for named (global scope) storages.
+            alias: The alias of the dataset for unnamed (run scope) storages.
 
         Returns:
             An instance for the opened or created storage client.
+
+        Raises:
+            ValueError: If both name and alias are provided, or if neither id, name, nor alias is provided.
         """
-        # Otherwise create a new dataset
+        # Validate input parameters.
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+        # Create a new dataset
         dataset_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)
 

crawlee/storage_clients/_memory/_key_value_store_client.py

@@ -4,10 +4,11 @@ import sys
 from datetime import datetime, timezone
 from typing import TYPE_CHECKING, Any
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee._utils.crypto import crypto_random_object_id
 from crawlee._utils.file import infer_mime_type
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import KeyValueStoreClient
 from crawlee.storage_clients.models import KeyValueStoreMetadata, KeyValueStoreRecord, KeyValueStoreRecordMetadata
 
@@ -51,21 +52,32 @@ class MemoryKeyValueStoreClient(KeyValueStoreClient):
         *,
         id: str | None,
         name: str | None,
-    ) -> MemoryKeyValueStoreClient:
+        alias: str | None,
+    ) -> Self:
         """Open or create a new memory key-value store client.
 
         This method creates a new in-memory key-value store instance. Unlike persistent storage implementations,
         memory KVS don't check for existing stores with the same name or ID since all data exists only in memory
         and is lost when the process terminates.
 
+        Alias does not have any effect on the memory storage client implementation, because unnamed storages
+        are supported by default, since data are not persisted.
+
         Args:
             id: The ID of the key-value store. If not provided, a random ID will be generated.
-            name: The name of the key-value store. If not provided, the store will be unnamed.
+            name: The name of the key-value store for named (global scope) storages.
+            alias: The alias of the key-value store for unnamed (run scope) storages.
 
         Returns:
             An instance for the opened or created storage client.
+
+        Raises:
+            ValueError: If both name and alias are provided.
         """
-        # Otherwise create a new key-value store
+        # Validate input parameters.
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+        # Create a new key-value store
         store_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)
 

crawlee/storage_clients/_memory/_request_queue_client.py

@@ -6,10 +6,11 @@ from datetime import datetime, timezone
 from logging import getLogger
 from typing import TYPE_CHECKING
 
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee import Request
 from crawlee._utils.crypto import crypto_random_object_id
+from crawlee._utils.raise_if_too_many_kwargs import raise_if_too_many_kwargs
 from crawlee.storage_clients._base import RequestQueueClient
 from crawlee.storage_clients.models import AddRequestsResponse, ProcessedRequest, RequestQueueMetadata
 
@@ -50,9 +51,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
         self._in_progress_requests = dict[str, Request]()
         """In-progress requests are those that have been fetched but not yet marked as handled or reclaimed."""
 
-        self._requests_by_id = dict[str, Request]()
-        """ID -> Request mapping for fast lookup by request ID."""
-
         self._requests_by_unique_key = dict[str, Request]()
         """Unique key -> Request mapping for fast lookup by unique key."""
 
@@ -66,21 +64,32 @@ class MemoryRequestQueueClient(RequestQueueClient):
         *,
         id: str | None,
         name: str | None,
-    ) -> MemoryRequestQueueClient:
+        alias: str | None,
+    ) -> Self:
         """Open or create a new memory request queue client.
 
         This method creates a new in-memory request queue instance. Unlike persistent storage implementations,
         memory queues don't check for existing queues with the same name or ID since all data exists only
         in memory and is lost when the process terminates.
 
+        Alias does not have any effect on the memory storage client implementation, because unnamed storages
+        are supported by default, since data are not persisted.
+
         Args:
             id: The ID of the request queue. If not provided, a random ID will be generated.
-            name: The name of the request queue. If not provided, the queue will be unnamed.
+            name: The name of the request queue for named (global scope) storages.
+            alias: The alias of the request queue for unnamed (run scope) storages.
 
         Returns:
             An instance for the opened or created storage client.
+
+        Raises:
+            ValueError: If both name and alias are provided.
         """
-        # Otherwise create a new queue
+        # Validate input parameters.
+        raise_if_too_many_kwargs(id=id, name=name, alias=alias)
+
+        # Create a new queue
         queue_id = id or crypto_random_object_id()
         now = datetime.now(timezone.utc)
 
@@ -102,7 +111,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
     async def drop(self) -> None:
         self._pending_requests.clear()
         self._handled_requests.clear()
-        self._requests_by_id.clear()
         self._requests_by_unique_key.clear()
         self._in_progress_requests.clear()
 
@@ -118,7 +126,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
     async def purge(self) -> None:
         self._pending_requests.clear()
         self._handled_requests.clear()
-        self._requests_by_id.clear()
         self._requests_by_unique_key.clear()
         self._in_progress_requests.clear()
 
@@ -142,12 +149,12 @@ class MemoryRequestQueueClient(RequestQueueClient):
 
             was_already_present = existing_request is not None
             was_already_handled = was_already_present and existing_request and existing_request.handled_at is not None
+            is_in_progress = request.unique_key in self._in_progress_requests
 
             # If the request is already in the queue and handled, don't add it again.
             if was_already_handled:
                 processed_requests.append(
                     ProcessedRequest(
-                        id=request.id,
                         unique_key=request.unique_key,
                         was_already_present=True,
                         was_already_handled=True,
@@ -155,22 +162,40 @@ class MemoryRequestQueueClient(RequestQueueClient):
                 )
                 continue
 
+            # If the request is already in progress, don't add it again.
+            if is_in_progress:
+                processed_requests.append(
+                    ProcessedRequest(
+                        unique_key=request.unique_key,
+                        was_already_present=True,
+                        was_already_handled=False,
+                    )
+                )
+                continue
+
             # If the request is already in the queue but not handled, update it.
             if was_already_present and existing_request:
-                # Update the existing request with any new data and
-                # remove old request from pending queue if it's there.
-                with suppress(ValueError):
-                    self._pending_requests.remove(existing_request)
-
                 # Update indexes.
-                self._requests_by_id[request.id] = request
                 self._requests_by_unique_key[request.unique_key] = request
 
-                # Add updated request back to queue.
+                # We only update `forefront` by updating its position by shifting it to the left.
                 if forefront:
+                    # Update the existing request with any new data and
+                    # remove old request from pending queue if it's there.
+                    with suppress(ValueError):
+                        self._pending_requests.remove(existing_request)
+
+                    # Add updated request back to queue.
                     self._pending_requests.appendleft(request)
-                else:
-                    self._pending_requests.append(request)
+
+                processed_requests.append(
+                    ProcessedRequest(
+                        unique_key=request.unique_key,
+                        was_already_present=True,
+                        was_already_handled=False,
+                    )
+                )
+
             # Add the new request to the queue.
             else:
                 if forefront:
@@ -179,7 +204,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
                     self._pending_requests.append(request)
 
                 # Update indexes.
-                self._requests_by_id[request.id] = request
                 self._requests_by_unique_key[request.unique_key] = request
 
                 await self._update_metadata(
@@ -189,7 +213,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
 
             processed_requests.append(
                 ProcessedRequest(
-                    id=request.id,
                     unique_key=request.unique_key,
                     was_already_present=was_already_present,
                     was_already_handled=False,
@@ -213,25 +236,24 @@ class MemoryRequestQueueClient(RequestQueueClient):
                 continue
 
             # Skip if already in progress (shouldn't happen, but safety check).
-            if request.id in self._in_progress_requests:
-                self._pending_requests.appendleft(request)
-                break
+            if request.unique_key in self._in_progress_requests:
+                continue
 
             # Mark as in progress.
-            self._in_progress_requests[request.id] = request
+            self._in_progress_requests[request.unique_key] = request
             return request
 
         return None
 
     @override
-    async def get_request(self, request_id: str) -> Request | None:
+    async def get_request(self, unique_key: str) -> Request | None:
         await self._update_metadata(update_accessed_at=True)
-        return self._requests_by_id.get(request_id)
+        return self._requests_by_unique_key.get(unique_key)
 
     @override
     async def mark_request_as_handled(self, request: Request) -> ProcessedRequest | None:
         # Check if the request is in progress.
-        if request.id not in self._in_progress_requests:
+        if request.unique_key not in self._in_progress_requests:
             return None
 
         # Set handled_at timestamp if not already set.
@@ -239,14 +261,13 @@ class MemoryRequestQueueClient(RequestQueueClient):
             request.handled_at = datetime.now(timezone.utc)
 
         # Move request to handled storage.
-        self._handled_requests[request.id] = request
+        self._handled_requests[request.unique_key] = request
 
-        # Update indexes (keep the request in indexes for get_request to work).
-        self._requests_by_id[request.id] = request
+        # Update index (keep the request in indexes for get_request to work).
         self._requests_by_unique_key[request.unique_key] = request
 
         # Remove from in-progress.
-        del self._in_progress_requests[request.id]
+        del self._in_progress_requests[request.unique_key]
 
         # Update metadata.
         await self._update_metadata(
@@ -256,7 +277,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
         )
 
         return ProcessedRequest(
-            id=request.id,
             unique_key=request.unique_key,
             was_already_present=True,
             was_already_handled=True,
@@ -270,11 +290,11 @@ class MemoryRequestQueueClient(RequestQueueClient):
         forefront: bool = False,
     ) -> ProcessedRequest | None:
         # Check if the request is in progress.
-        if request.id not in self._in_progress_requests:
+        if request.unique_key not in self._in_progress_requests:
             return None
 
         # Remove from in-progress.
-        del self._in_progress_requests[request.id]
+        del self._in_progress_requests[request.unique_key]
 
         # Add request back to pending queue.
         if forefront:
@@ -286,7 +306,6 @@ class MemoryRequestQueueClient(RequestQueueClient):
         await self._update_metadata(update_modified_at=True)
 
         return ProcessedRequest(
-            id=request.id,
             unique_key=request.unique_key,
             was_already_present=True,
             was_already_handled=False,

crawlee/storage_clients/_memory/_storage_client.py

@@ -33,10 +33,11 @@ class MemoryStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> MemoryDatasetClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await MemoryDatasetClient.open(id=id, name=name)
+        client = await MemoryDatasetClient.open(id=id, name=name, alias=alias)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -46,10 +47,11 @@ class MemoryStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> MemoryKeyValueStoreClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await MemoryKeyValueStoreClient.open(id=id, name=name)
+        client = await MemoryKeyValueStoreClient.open(id=id, name=name, alias=alias)
         await self._purge_if_needed(client, configuration)
         return client
 
@@ -59,9 +61,10 @@ class MemoryStorageClient(StorageClient):
         *,
         id: str | None = None,
         name: str | None = None,
+        alias: str | None = None,
         configuration: Configuration | None = None,
     ) -> MemoryRequestQueueClient:
         configuration = configuration or Configuration.get_global_configuration()
-        client = await MemoryRequestQueueClient.open(id=id, name=name)
+        client = await MemoryRequestQueueClient.open(id=id, name=name, alias=alias)
         await self._purge_if_needed(client, configuration)
         return client

crawlee/storage_clients/_redis/__init__.py

@@ -0,0 +1,6 @@
+from ._dataset_client import RedisDatasetClient
+from ._key_value_store_client import RedisKeyValueStoreClient
+from ._request_queue_client import RedisRequestQueueClient
+from ._storage_client import RedisStorageClient
+
+__all__ = ['RedisDatasetClient', 'RedisKeyValueStoreClient', 'RedisRequestQueueClient', 'RedisStorageClient']