crawlee 1.0.2b3__py3-none-any.whl → 1.1.2b7__py3-none-any.whl

crawlee/storage_clients/_redis/_storage_client.py

@@ -0,0 +1,146 @@
+from __future__ import annotations
+
+import warnings
+from typing import Literal
+
+from redis.asyncio import Redis
+from typing_extensions import override
+
+from crawlee._utils.docs import docs_group
+from crawlee.configuration import Configuration
+from crawlee.storage_clients._base import StorageClient
+
+from ._dataset_client import RedisDatasetClient
+from ._key_value_store_client import RedisKeyValueStoreClient
+from ._request_queue_client import RedisRequestQueueClient
+
+
+@docs_group('Storage clients')
+class RedisStorageClient(StorageClient):
+    """Redis implementation of the storage client.
+
+    This storage client provides access to datasets, key-value stores, and request queues that persist data
+    to a Redis database v8.0+. Each storage type uses Redis-specific data structures and key patterns for
+    efficient storage and retrieval.
+
+    The client accepts either a Redis connection string or a pre-configured Redis client instance.
+    Exactly one of these parameters must be provided during initialization.
+
+    Storage types use the following Redis data structures:
+    - **Datasets**: Redis JSON arrays for item storage with metadata in JSON objects
+    - **Key-value stores**: Redis hashes for key-value pairs with separate metadata storage
+    - **Request queues**: Redis lists for FIFO queuing, hashes for request data and in-progress tracking,
+      and Bloom filters for request deduplication
+
+    Warning:
+        This is an experimental feature. The behavior and interface may change in future versions.
+    """
+
+    def __init__(
+        self,
+        *,
+        connection_string: str | None = None,
+        redis: Redis | None = None,
+        queue_dedup_strategy: Literal['default', 'bloom'] = 'default',
+        queue_bloom_error_rate: float = 1e-7,
+    ) -> None:
+        """Initialize the Redis storage client.
+
+        Args:
+            connection_string: Redis connection string (e.g., "redis://localhost:6379").
+                Supports standard Redis URL format with optional database selection.
+            redis: Pre-configured Redis client instance.
+            queue_dedup_strategy: Strategy for request queue deduplication. Options are:
+                - 'default': Uses Redis sets for exact deduplication.
+                - 'bloom': Uses Redis Bloom filters for probabilistic deduplication with lower memory usage. When using
+                    this approach, approximately 1 in 1e-7 requests will be falsely considered duplicate.
+            queue_bloom_error_rate: Desired false positive rate for Bloom filter deduplication. Only relevant if
+                `queue_dedup_strategy` is set to 'bloom'.
+        """
+        match (redis, connection_string):
+            case (None, None):
+                raise ValueError('Either redis or connection_string must be provided.')
+            case (Redis(), None):
+                self._redis = redis
+            case (None, str()):
+                self._redis = Redis.from_url(connection_string)
+            case (Redis(), str()):
+                raise ValueError('Either redis or connection_string must be provided, not both.')
+
+        self._queue_dedup_strategy = queue_dedup_strategy
+        self._queue_bloom_error_rate = queue_bloom_error_rate
+
+        # Call the notification only once
+        warnings.warn(
+            (
+                'RedisStorageClient is experimental and its API, behavior, and key structure may change in future '
+                'releases.'
+            ),
+            category=UserWarning,
+            stacklevel=2,
+        )
+
+    @override
+    async def create_dataset_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> RedisDatasetClient:
+        configuration = configuration or Configuration.get_global_configuration()
+
+        client = await RedisDatasetClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            redis=self._redis,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client
+
+    @override
+    async def create_kvs_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> RedisKeyValueStoreClient:
+        configuration = configuration or Configuration.get_global_configuration()
+
+        client = await RedisKeyValueStoreClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            redis=self._redis,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client
+
+    @override
+    async def create_rq_client(
+        self,
+        *,
+        id: str | None = None,
+        name: str | None = None,
+        alias: str | None = None,
+        configuration: Configuration | None = None,
+    ) -> RedisRequestQueueClient:
+        configuration = configuration or Configuration.get_global_configuration()
+
+        client = await RedisRequestQueueClient.open(
+            id=id,
+            name=name,
+            alias=alias,
+            redis=self._redis,
+            dedup_strategy=self._queue_dedup_strategy,
+            bloom_error_rate=self._queue_bloom_error_rate,
+        )
+
+        await self._purge_if_needed(client, configuration)
+        return client

crawlee/storage_clients/_redis/_utils.py

@@ -0,0 +1,23 @@
+from collections.abc import Awaitable
+from pathlib import Path
+from typing import TypeVar, overload
+
+T = TypeVar('T')
+
+
+@overload
+async def await_redis_response(response: Awaitable[T]) -> T: ...
+@overload
+async def await_redis_response(response: T) -> T: ...
+
+
+async def await_redis_response(response: Awaitable[T] | T) -> T:
+    """Solve the problem of ambiguous typing for redis."""
+    return await response if isinstance(response, Awaitable) else response
+
+
+def read_lua_script(script_name: str) -> str:
+    """Read a Lua script from a file."""
+    file_path = Path(__file__).parent / 'lua_scripts' / script_name
+    with file_path.open('r', encoding='utf-8') as file:
+        return file.read()

crawlee/storage_clients/_redis/lua_scripts/atomic_bloom_add_requests.lua

@@ -0,0 +1,36 @@
+local added_filter_key = KEYS[1]
+local queue_key = KEYS[2]
+local data_key = KEYS[3]
+
+local forefront = ARGV[1] == '1'
+local unique_keys = cjson.decode(ARGV[2])
+local requests_data = cjson.decode(ARGV[3])
+
+-- Add and check which unique keys are actually new using Bloom filter
+local bf_results = redis.call('bf.madd', added_filter_key, unpack(unique_keys))
+
+local actually_added = {}
+local hset_args = {}
+
+-- Process the results
+for i, unique_key in ipairs(unique_keys) do
+    if bf_results[i] == 1 then
+        -- This key was added by us (did not exist before)
+        table.insert(hset_args, unique_key)
+        table.insert(hset_args, requests_data[unique_key])
+        table.insert(actually_added, unique_key)
+    end
+end
+
+-- Add only those that are actually new
+if #actually_added > 0 then
+    redis.call('hset', data_key, unpack(hset_args))
+
+    if forefront then
+        redis.call('lpush', queue_key, unpack(actually_added))
+    else
+        redis.call('rpush', queue_key, unpack(actually_added))
+    end
+end
+
+return cjson.encode(actually_added)

crawlee/storage_clients/_redis/lua_scripts/atomic_fetch_request.lua

@@ -0,0 +1,49 @@
+local queue_key = KEYS[1]
+local in_progress_key = KEYS[2]
+local data_key = KEYS[3]
+local client_id = ARGV[1]
+local blocked_until_timestamp = ARGV[2]
+local batch_size = tonumber(ARGV[3])
+
+-- Pop batch unique_key from queue
+local batch_result = redis.call('LMPOP', 1, queue_key, 'LEFT', 'COUNT', batch_size)
+if not batch_result then
+    return nil
+end
+local unique_keys = batch_result[2]
+
+-- Get requests data
+local requests_data = redis.call('HMGET', data_key, unpack(unique_keys))
+if not requests_data then
+    -- Data missing, skip this request
+    return nil
+end
+
+-- Prepare results and update in_progress
+local final_result = {}
+local in_progress_hmset = {}
+local pending_decrement = 0
+local in_progress_data = cjson.encode({
+    client_id = client_id,
+    blocked_until_timestamp = tonumber(blocked_until_timestamp)
+})
+for i = 1, #unique_keys do
+    local unique_key = unique_keys[i]
+    local request_data = requests_data[i]
+
+    if request_data then
+        -- Add to in_progress hash
+        table.insert(in_progress_hmset, unique_key)
+        table.insert(in_progress_hmset, in_progress_data)
+
+        table.insert(final_result, request_data)
+    end
+end
+
+-- Update in_progress hash
+if #in_progress_hmset > 0 then
+    redis.call('HMSET', in_progress_key, unpack(in_progress_hmset))
+end
+
+-- Return result with requests data
+return final_result

crawlee/storage_clients/_redis/lua_scripts/atomic_set_add_requests.lua

@@ -0,0 +1,37 @@
+local added_filter_key = KEYS[1]
+local queue_key = KEYS[2]
+local data_key = KEYS[3]
+
+local forefront = ARGV[1] == '1'
+local unique_keys = cjson.decode(ARGV[2])
+local requests_data = cjson.decode(ARGV[3])
+
+-- Add and check which unique keys are actually new using Redis set
+local actually_added = {}
+local hset_args = {}
+
+-- Process each unique key
+for _, unique_key in ipairs(unique_keys) do
+    -- Try to add the key to the set, returns 1 if added, 0 if already existed
+    local set_result = redis.call('sadd', added_filter_key, unique_key)
+
+    if set_result == 1 then
+        -- This key was added by us (did not exist before)
+        table.insert(hset_args, unique_key)
+        table.insert(hset_args, requests_data[unique_key])
+        table.insert(actually_added, unique_key)
+    end
+end
+
+-- Add only those that are actually new
+if #actually_added > 0 then
+    redis.call('hset', data_key, unpack(hset_args))
+
+    if forefront then
+        redis.call('lpush', queue_key, unpack(actually_added))
+    else
+        redis.call('rpush', queue_key, unpack(actually_added))
+    end
+end
+
+return cjson.encode(actually_added)

crawlee/storage_clients/_redis/lua_scripts/reclaim_stale_requests.lua

@@ -0,0 +1,34 @@
+local in_progress_key = KEYS[1]
+local queue_key = KEYS[2]
+local data_key = KEYS[3]
+local current_time = tonumber(ARGV[1])
+
+local max_reclaim = 1000
+
+local cursor = "0"
+local count = 0
+
+repeat
+    local result = redis.call('hscan', in_progress_key, cursor, 'COUNT', 100)
+    cursor = result[1]
+    local entries = result[2]
+
+    for i = 1, #entries, 2 do
+        if count >= max_reclaim then
+            break
+        end
+
+        local unique_key = entries[i]
+        local data = cjson.decode(entries[i + 1])
+
+        -- Check if timed out
+        if current_time > data.blocked_until_timestamp then
+            -- Atomically remove from in_progress and add back to queue
+            redis.call('hdel', in_progress_key, unique_key)
+            redis.call('rpush', queue_key, unique_key)
+            count = count + 1
+        end
+    end
+until cursor == "0" or count >= max_reclaim
+
+return count

crawlee/storage_clients/_sql/_db_models.py

@@ -205,9 +205,8 @@ class RequestDb(Base):
             'idx_fetch_available',
             'request_queue_id',
             'is_handled',
-            'time_blocked_until',
             'sequence_number',
-            postgresql_where=text('is_handled = false'),
+            postgresql_where=text('is_handled is false'),
         ),
     )
 

crawlee/storage_clients/_sql/_key_value_store_client.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import json
 from logging import getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
-from sqlalchemy import delete, select
+from sqlalchemy import CursorResult, delete, select
 from typing_extensions import Self, override
 
 from crawlee._utils.file import infer_mime_type
@@ -227,6 +227,7 @@ class SqlKeyValueStoreClient(KeyValueStoreClient, SqlClientMixin):
         async with self.get_session(with_simple_commit=True) as session:
             # Delete the record if it exists
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
 
             # Update metadata if we actually deleted something
             if result.rowcount > 0:

crawlee/storage_clients/_sql/_request_queue_client.py

@@ -5,9 +5,9 @@ from datetime import datetime, timedelta, timezone
 from functools import lru_cache
 from hashlib import sha256
 from logging import getLogger
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, cast
 
-from sqlalchemy import func, or_, select, update
+from sqlalchemy import CursorResult, func, or_, select, update
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.orm import load_only
 from typing_extensions import NotRequired, Self, override
@@ -231,6 +231,7 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
 
         async with self.get_session() as session:
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
             existing_requests = {req.request_id: req for req in result.scalars()}
             state = await self._get_state(session)
             insert_values: list[dict] = []
@@ -498,9 +499,12 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
         )
         async with self.get_session() as session:
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
+
             if result.rowcount == 0:
                 logger.warning(f'Request {request.unique_key} not found in database.')
                 return None
+
             await self._update_metadata(
                 session,
                 **_QueueMetadataUpdateParams(
@@ -542,14 +546,24 @@ class SqlRequestQueueClient(RequestQueueClient, SqlClientMixin):
                 block_until = now + timedelta(seconds=self._BLOCK_REQUEST_TIME)
                 # Extend blocking for forefront request, it is considered blocked by the current client.
                 stmt = stmt.values(
-                    sequence_number=new_sequence, time_blocked_until=block_until, client_key=self.client_key
+                    sequence_number=new_sequence,
+                    time_blocked_until=block_until,
+                    client_key=self.client_key,
+                    data=request.model_dump_json(),
                 )
             else:
                 new_sequence = state.sequence_counter
                 state.sequence_counter += 1
-                stmt = stmt.values(sequence_number=new_sequence, time_blocked_until=None, client_key=None)
+                stmt = stmt.values(
+                    sequence_number=new_sequence,
+                    time_blocked_until=None,
+                    client_key=None,
+                    data=request.model_dump_json(),
+                )
 
             result = await session.execute(stmt)
+            result = cast('CursorResult', result) if not isinstance(result, CursorResult) else result
+
             if result.rowcount == 0:
                 logger.warning(f'Request {request.unique_key} not found in database.')
                 return None

crawlee/storage_clients/_sql/_storage_client.py

@@ -149,7 +149,7 @@ class SqlStorageClient(StorageClient):
                     # Raise an error if the new version creates breaking changes in the database schema.
                     if db_version and db_version != __version__:
                         warnings.warn(
-                            f'Database version {db_version.version} does not match library version {__version__}. '
+                            f'Database version {db_version} does not match library version {__version__}. '
                             'This may lead to unexpected behavior. Drop the db if you want to make sure that '
                             'everything will work fine.',
                             category=UserWarning,

crawlee/storages/_key_value_store.py

@@ -281,11 +281,14 @@ class KeyValueStore(Storage):
             if key in cache:
                 return cache[key].current_value.root
 
+            async def kvs_factory() -> KeyValueStore:
+                return self
+
             cache[key] = recoverable_state = RecoverableState(
                 default_state=AutosavedValue(default_value),
-                persistence_enabled=True,
-                persist_state_kvs_id=self.id,
                 persist_state_key=key,
+                persistence_enabled=True,
+                persist_state_kvs_factory=kvs_factory,
                 logger=logger,
             )
 

{crawlee-1.0.2b3.dist-info → crawlee-1.1.2b7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.2b3
+Version: 1.1.2b7
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -223,15 +223,17 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
 Classifier: Topic :: Software Development :: Libraries
 Requires-Python: >=3.10
+Requires-Dist: async-timeout>=5.0.1
 Requires-Dist: cachetools>=5.5.0
 Requires-Dist: colorama>=0.4.0
-Requires-Dist: impit>=0.6.1
+Requires-Dist: impit>=0.8.0
 Requires-Dist: more-itertools>=10.2.0
 Requires-Dist: protego>=0.5.0
 Requires-Dist: psutil>=6.0.0
-Requires-Dist: pydantic-settings!=2.7.0,!=2.7.1,!=2.8.0,>=2.2.0
+Requires-Dist: pydantic-settings>=2.12.0
 Requires-Dist: pydantic>=2.11.0
 Requires-Dist: pyee>=9.0.0
 Requires-Dist: tldextract>=5.1.0
@@ -263,6 +265,7 @@ Requires-Dist: opentelemetry-sdk>=1.34.1; extra == 'all'
 Requires-Dist: opentelemetry-semantic-conventions>=0.54; extra == 'all'
 Requires-Dist: parsel>=1.10.0; extra == 'all'
 Requires-Dist: playwright>=1.27.0; extra == 'all'
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'all'
 Requires-Dist: rich>=13.9.0; extra == 'all'
 Requires-Dist: scikit-learn>=1.6.0; extra == 'all'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'all'
@@ -296,6 +299,8 @@ Provides-Extra: playwright
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'playwright'
 Requires-Dist: browserforge>=1.2.3; extra == 'playwright'
 Requires-Dist: playwright>=1.27.0; extra == 'playwright'
+Provides-Extra: redis
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
 Provides-Extra: sql-postgres
 Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'