crawlee 1.0.1b9__py3-none-any.whl → 1.3.1b3__py3-none-any.whl

crawlee/storage_clients/_redis/_storage_client.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+import warnings
+from typing import Literal
+
+from redis.asyncio import Redis
+from typing_extensions import override
+
+from crawlee._utils.docs import docs_group
+from crawlee.configuration import Configuration
+from crawlee.storage_clients._base import StorageClient
+
+from ._dataset_client import RedisDatasetClient
+from ._key_value_store_client import RedisKeyValueStoreClient
+from ._request_queue_client import RedisRequestQueueClient
+
+
+@docs_group('Storage clients')
+class RedisStorageClient(StorageClient):
+    """Redis implementation of the storage client.
+
+    This storage client provides access to datasets, key-value stores, and request queues that persist data
+    to a Redis database v8.0+. Each storage type uses Redis-specific data structures and key patterns for
+    efficient storage and retrieval.
+
+    The client accepts either a Redis connection string or a pre-configured Redis client instance.
+    Exactly one of these parameters must be provided during initialization.
+
+    Storage types use the following Redis data structures:
+    - **Datasets**: Redis JSON arrays for item storage with metadata in JSON objects
+    - **Key-value stores**: Redis hashes for key-value pairs with separate metadata storage
+    - **Request queues**: Redis lists for FIFO queuing, hashes for request data and in-progress tracking,
+      and Bloom filters for request deduplication
+
+    Warning:
+        This is an experimental feature. The behavior and interface may change in future versions.
+    """
+
+    def __init__(
+        self,
+        *,
+        connection_string: str | None = None,
+        redis: Redis | None = None,
+        queue_dedup_strategy: Literal['default', 'bloom'] = 'default',
+        queue_bloom_error_rate: float = 1e-7,
+    ) -> None:
+        """Initialize the Redis storage client.
+
+        Args:
+            connection_string: Redis connection string (e.g., "redis://localhost:6379").
+                Supports standard Redis URL format with optional database selection.
+            redis: Pre-configured Redis client instance.
+            queue_dedup_strategy: Strategy for request queue deduplication. Options are:
+                - 'default': Uses Redis sets for exact deduplication.
+                - 'bloom': Uses Redis Bloom filters for probabilistic deduplication with lower memory usage. When using
+                    this approach, approximately 1 in 1e-7 requests will be falsely considered duplicate.
+            queue_bloom_error_rate: Desired false positive rate for Bloom filter deduplication. Only relevant if
+                `queue_dedup_strategy` is set to 'bloom'.
+        """
+        if redis is None and connection_string is None:
+            raise ValueError('Either redis or connection_string must be provided.')
+
+        if redis is not None and connection_string is not None:
+            raise ValueError('Either redis or connection_string must be provided, not both.')
+
+        if isinstance(redis, Redis) and connection_string is None:
+            self._redis = redis
+
+        if isinstance(connection_string, str) and redis is None:
+            self._redis = Redis.from_url(connection_string)
+
+        self._redis: Redis  # to help type checker
+        self._queue_dedup_strategy = queue_dedup_strategy
+        self._queue_bloom_error_rate = queue_bloom_error_rate
+
+        # Call the notification only once
+        warnings.warn(
+            (
+                'RedisStorageClient is experimental and its API, behavior, and key structure may change in future '
+                'releases.'
+            ),
+            category=UserWarning,
+            stacklevel=2,
+        )
+
+    @override
+    async def create_dataset_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> RedisDatasetClient:
+        configuration = configuration or Configuration.get_global_configuration()
+
+        client = await RedisDatasetClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            redis=self._redis,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client
+
+    @override
+    async def create_kvs_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> RedisKeyValueStoreClient:
+        configuration = configuration or Configuration.get_global_configuration()
+
+        client = await RedisKeyValueStoreClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            redis=self._redis,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client
+
+    @override
+    async def create_rq_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> RedisRequestQueueClient:
+        configuration = configuration or Configuration.get_global_configuration()
+
+        client = await RedisRequestQueueClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            redis=self._redis,
+            dedup_strategy=self._queue_dedup_strategy,
+            bloom_error_rate=self._queue_bloom_error_rate,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client

crawlee/storage_clients/_redis/_utils.py

@@ -0,0 +1,23 @@
+from collections.abc import Awaitable
+from pathlib import Path
+from typing import TypeVar, overload
+
+T = TypeVar('T')
+
+
+@overload
+async def await_redis_response(response: Awaitable[T]) -> T: ...
+@overload
+async def await_redis_response(response: T) -> T: ...
+
+
+async def await_redis_response(response: Awaitable[T] | T) -> T:
+    """Solve the problem of ambiguous typing for redis."""
+    return await response if isinstance(response, Awaitable) else response
+
+
+def read_lua_script(script_name: str) -> str:
+    """Read a Lua script from a file."""
+    file_path = Path(__file__).parent / 'lua_scripts' / script_name
+    with file_path.open(mode='r', encoding='utf-8') as file:
+        return file.read()

crawlee/storage_clients/_redis/lua_scripts/atomic_bloom_add_requests.lua

@@ -0,0 +1,36 @@
+local added_filter_key = KEYS[1]
+local queue_key = KEYS[2]
+local data_key = KEYS[3]
+
+local forefront = ARGV[1] == '1'
+local unique_keys = cjson.decode(ARGV[2])
+local requests_data = cjson.decode(ARGV[3])
+
+-- Add and check which unique keys are actually new using Bloom filter
+local bf_results = redis.call('bf.madd', added_filter_key, unpack(unique_keys))
+
+local actually_added = {}
+local hset_args = {}
+
+-- Process the results
+for i, unique_key in ipairs(unique_keys) do
+    if bf_results[i] == 1 then
+        -- This key was added by us (did not exist before)
+        table.insert(hset_args, unique_key)
+        table.insert(hset_args, requests_data[unique_key])
+        table.insert(actually_added, unique_key)
+    end
+end
+
+-- Add only those that are actually new
+if #actually_added > 0 then
+    redis.call('hset', data_key, unpack(hset_args))
+
+    if forefront then
+        redis.call('lpush', queue_key, unpack(actually_added))
+    else
+        redis.call('rpush', queue_key, unpack(actually_added))
+    end
+end
+
+return cjson.encode(actually_added)

crawlee/storage_clients/_redis/lua_scripts/atomic_fetch_request.lua

@@ -0,0 +1,49 @@
+local queue_key = KEYS[1]
+local in_progress_key = KEYS[2]
+local data_key = KEYS[3]
+local client_id = ARGV[1]
+local blocked_until_timestamp = ARGV[2]
+local batch_size = tonumber(ARGV[3])
+
+-- Pop batch unique_key from queue
+local batch_result = redis.call('LMPOP', 1, queue_key, 'LEFT', 'COUNT', batch_size)
+if not batch_result then
+    return nil
+end
+local unique_keys = batch_result[2]
+
+-- Get requests data
+local requests_data = redis.call('HMGET', data_key, unpack(unique_keys))
+if not requests_data then
+    -- Data missing, skip this request
+    return nil
+end
+
+-- Prepare results and update in_progress
+local final_result = {}
+local in_progress_hmset = {}
+local pending_decrement = 0
+local in_progress_data = cjson.encode({
+    client_id = client_id,
+    blocked_until_timestamp = tonumber(blocked_until_timestamp)
+})
+for i = 1, #unique_keys do
+    local unique_key = unique_keys[i]
+    local request_data = requests_data[i]
+
+    if request_data then
+        -- Add to in_progress hash
+        table.insert(in_progress_hmset, unique_key)
+        table.insert(in_progress_hmset, in_progress_data)
+
+        table.insert(final_result, request_data)
+    end
+end
+
+-- Update in_progress hash
+if #in_progress_hmset > 0 then
+    redis.call('HMSET', in_progress_key, unpack(in_progress_hmset))
+end
+
+-- Return result with requests data
+return final_result

crawlee/storage_clients/_redis/lua_scripts/atomic_set_add_requests.lua

@@ -0,0 +1,37 @@
+local added_filter_key = KEYS[1]
+local queue_key = KEYS[2]
+local data_key = KEYS[3]
+
+local forefront = ARGV[1] == '1'
+local unique_keys = cjson.decode(ARGV[2])
+local requests_data = cjson.decode(ARGV[3])
+
+-- Add and check which unique keys are actually new using Redis set
+local actually_added = {}
+local hset_args = {}
+
+-- Process each unique key
+for _, unique_key in ipairs(unique_keys) do
+    -- Try to add the key to the set, returns 1 if added, 0 if already existed
+    local set_result = redis.call('sadd', added_filter_key, unique_key)
+
+    if set_result == 1 then
+        -- This key was added by us (did not exist before)
+        table.insert(hset_args, unique_key)
+        table.insert(hset_args, requests_data[unique_key])
+        table.insert(actually_added, unique_key)
+    end
+end
+
+-- Add only those that are actually new
+if #actually_added > 0 then
+    redis.call('hset', data_key, unpack(hset_args))
+
+    if forefront then
+        redis.call('lpush', queue_key, unpack(actually_added))
+    else
+        redis.call('rpush', queue_key, unpack(actually_added))
+    end
+end
+
+return cjson.encode(actually_added)

crawlee/storage_clients/_redis/lua_scripts/reclaim_stale_requests.lua

@@ -0,0 +1,34 @@
+local in_progress_key = KEYS[1]
+local queue_key = KEYS[2]
+local data_key = KEYS[3]
+local current_time = tonumber(ARGV[1])
+
+local max_reclaim = 1000
+
+local cursor = "0"
+local count = 0
+
+repeat
+    local result = redis.call('hscan', in_progress_key, cursor, 'COUNT', 100)
+    cursor = result[1]
+    local entries = result[2]
+
+    for i = 1, #entries, 2 do
+        if count >= max_reclaim then
+            break
+        end
+
+        local unique_key = entries[i]
+        local data = cjson.decode(entries[i + 1])
+
+        -- Check if timed out
+        if current_time > data.blocked_until_timestamp then
+            -- Atomically remove from in_progress and add back to queue
+            redis.call('hdel', in_progress_key, unique_key)
+            redis.call('rpush', queue_key, unique_key)
+            count = count + 1
+        end
+    end
+until cursor == "0" or count >= max_reclaim
+
+return count

crawlee/storage_clients/_sql/_client_mixin.py

@@ -105,7 +105,7 @@ class SqlClientMixin(ABC):
         else:
             stmt = select(cls._METADATA_TABLE).where(cls._METADATA_TABLE.internal_name == internal_name)
             result = await session.execute(stmt)
-            orm_metadata = result.scalar_one_or_none()  # type: ignore[assignment]
+            orm_metadata = result.scalar_one_or_none()
 
         if orm_metadata:
             client = cls(id=orm_metadata.id, storage_client=storage_client)

crawlee/storage_clients/_sql/_dataset_client.py

@@ -4,7 +4,7 @@ from logging import getLogger
 from typing import TYPE_CHECKING, Any
 
 from sqlalchemy import Select, insert, select
-from typing_extensions import override
+from typing_extensions import Self, override
 
 from crawlee.storage_clients._base import DatasetClient
 from crawlee.storage_clients.models import DatasetItemsListPage, DatasetMetadata
@@ -78,7 +78,7 @@ class SqlDatasetClient(DatasetClient, SqlClientMixin):
         name: str | None,
         alias: str | None,
         storage_client: SqlStorageClient,
-    ) -> SqlDatasetClient:
+    ) -> Self:
         """Open an existing dataset or create a new one.
 
         Args:

crawlee/storage_clients/_sql/_db_models.py

@@ -205,9 +205,8 @@ class RequestDb(Base):
             'idx_fetch_available',
             'request_queue_id',
             'is_handled',
-            'time_blocked_until',
             'sequence_number',
-            postgresql_where=text('is_handled = false'),
+            postgresql_where=text('is_handled is false'),
         ),
     )
 

crawlee/storage_clients/_sql/_key_value_store_client.py

@@ -2,10 +2,10 @@ from __future__ import annotations
 
 import json
 from logging import getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
-from sqlalchemy import delete, select
-from typing_extensions import override
+from sqlalchemy import CursorResult, delete, select
+from typing_extensions import Self, override
 
 from crawlee._utils.file import infer_mime_type
 from crawlee.storage_clients._base import KeyValueStoreClient
@@ -77,7 +77,7 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin):
         name: str | None,
         alias: str | None,
         storage_client: SqlStorageClient,
-    ) -> SqlKeyValueStoreClient:
+    ) -> Self:
         """Open or create a SQL key-value store client.
 
         This method attempts to open an existing key-value store from the SQL database. If a KVS with the specified
@@ -227,6 +227,7 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin):
         async with self.get_session(with_simple_commit=True) as session:
             # Delete the record if it exists
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
 
             # Update metadata if we actually deleted something
             if result.rowcount > 0:

crawlee/storage_clients/_sql/_request_queue_client.py

@@ -5,12 +5,12 @@ from datetime import datetime, timedelta, timezone
 from functools import lru_cache
 from hashlib import sha256
 from logging import getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
-from sqlalchemy import func, or_, select, update
+from sqlalchemy import CursorResult, func, or_, select, update
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import load_only
-from typing_extensions import NotRequired, override
+from typing_extensions import NotRequired, Self, override
 
 from crawlee import Request
 from crawlee._utils.crypto import crypto_random_object_id
@@ -119,7 +119,7 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
         name: str | None,
         alias: str | None,
         storage_client: SqlStorageClient,
-    ) -> SqlRequestQueueClient:
+    ) -> Self:
         """Open an existing request queue or create a new one.
 
         This method first tries to find an existing queue by ID or name.
@@ -231,6 +231,7 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
 
         async with self.get_session() as session:
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
             existing_requests = {req.request_id: req for req in result.scalars()}
             state = await self._get_state(session)
             insert_values: list[dict] = []
@@ -498,9 +499,12 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
         )
         async with self.get_session() as session:
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
+
             if result.rowcount == 0:
                 logger.warning(f'Request {request.unique_key} not found in database.')
                 return None
+
             await self._update_metadata(
                 session,
                 **_QueueMetadataUpdateParams(
@@ -542,14 +546,24 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
                 block_until = now + timedelta(seconds=self._BLOCK_REQUEST_TIME)
                 # Extend blocking for forefront request, it is considered blocked by the current client.
                 stmt = stmt.values(
-                    sequence_number=new_sequence, time_blocked_until=block_until, client_key=self.client_key
+                    sequence_number=new_sequence,
+                    time_blocked_until=block_until,
+                    client_key=self.client_key,
+                    data=request.model_dump_json(),
                 )
             else:
                 new_sequence = state.sequence_counter
                 state.sequence_counter += 1
-                stmt = stmt.values(sequence_number=new_sequence, time_blocked_until=None, client_key=None)
+                stmt = stmt.values(
+                    sequence_number=new_sequence,
+                    time_blocked_until=None,
+                    client_key=None,
+                    data=request.model_dump_json(),
+                )
 
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
+
             if result.rowcount == 0:
                 logger.warning(f'Request {request.unique_key} not found in database.')
                 return None

crawlee/storage_clients/_sql/_storage_client.py

@@ -149,7 +149,7 @@ class SqlStorageClient(StorageClient):
                     # Raise an error if the new version creates breaking changes in the database schema.
                     if db_version and db_version != __version__:
                         warnings.warn(
-                            f'Database version {db_version.version} does not match library version {__version__}. '
+                            f'Database version {db_version} does not match library version {__version__}. '
                             'This may lead to unexpected behavior. Drop the db if you want to make sure that '
                             'everything will work fine.',
                             category=UserWarning,

crawlee/storage_clients/models.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Annotated, Any, Generic
+from typing import TYPE_CHECKING, Annotated, Any, Generic
 
 from pydantic import BaseModel, BeforeValidator, ConfigDict, Field
 from typing_extensions import TypeVar
@@ -127,8 +127,13 @@ class DatasetItemsListPage(BaseModel):
     desc: Annotated[bool, Field(default=False)]
     """Indicates if the returned list is in descending order."""
 
-    items: Annotated[list[dict], Field(default_factory=list)]
-    """The list of dataset items returned on this page."""
+    # Workaround for Pydantic and type checkers when using Annotated with default_factory
+    if TYPE_CHECKING:
+        items: list[dict] = []
+        """The list of dataset items returned on this page."""
+    else:
+        items: Annotated[list[dict], Field(default_factory=list)]
+        """The list of dataset items returned on this page."""
 
 
 @docs_group('Storage data')

crawlee/storages/_base.py

@@ -44,7 +44,9 @@ class Storage(ABC):
 
         Args:
             id: The storage ID.
-            name: The storage name (global scope, persists across runs).
+            name: The storage name (global scope, persists across runs). Name can only contain letters "a" through "z",
+                the digits "0" through "9", and the hyphen ("-") but only in the middle of the string
+                (e.g. "my-value-1").
             alias: The storage alias (run scope, creates unnamed storage).
             configuration: Configuration object used during the storage creation or restoration process.
             storage_client: Underlying storage client to use. If not provided, the default global storage client

crawlee/storages/_dataset.py

@@ -12,6 +12,7 @@ from crawlee._utils.file import export_csv_to_stream, export_json_to_stream
 
 from ._base import Storage
 from ._key_value_store import KeyValueStore
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -75,6 +76,8 @@ class Dataset(Storage):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name

crawlee/storages/_key_value_store.py

@@ -15,6 +15,7 @@ from crawlee._utils.recoverable_state import RecoverableState
 from crawlee.storage_clients.models import KeyValueStoreMetadata
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import AsyncIterator
@@ -84,6 +85,8 @@ class KeyValueStore(Storage):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name
@@ -278,11 +281,14 @@ class KeyValueStore(Storage):
             if key in cache:
                 return cache[key].current_value.root
 
+            async def kvs_factory() -> KeyValueStore:
+                return self
+
             cache[key] = recoverable_state = RecoverableState(
                 default_state=AutosavedValue(default_value),
-                persistence_enabled=True,
-                persist_state_kvs_id=self.id,
                 persist_state_key=key,
+                persistence_enabled=True,
+                persist_state_kvs_factory=kvs_factory,
                 logger=logger,
             )
 

crawlee/storages/_request_queue.py

@@ -13,6 +13,7 @@ from crawlee._utils.wait import wait_for_all_tasks_for_finish
 from crawlee.request_loaders import RequestManager
 
 from ._base import Storage
+from ._utils import validate_storage_name
 
 if TYPE_CHECKING:
     from collections.abc import Sequence
@@ -80,6 +81,8 @@ class RequestQueue(Storage, RequestManager):
             id: The unique identifier of the storage.
             name: The name of the storage, if available.
         """
+        validate_storage_name(name)
+
         self._client = client
         self._id = id
         self._name = name