crawlee 1.0.5b12__tar.gz → 1.0.5b13__tar.gz

{crawlee-1.0.5b12 → crawlee-1.0.5b13}/CHANGELOG.md

@@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file.
 ### 🚀 Features
 
 - Add `chrome` `BrowserType` for `PlaywrightCrawler` to use the Chrome browser ([#1487](https://github.com/apify/crawlee-python/pull/1487)) ([b06937b](https://github.com/apify/crawlee-python/commit/b06937bbc3afe3c936b554bfc503365c1b2c526b)) by [@Mantisus](https://github.com/Mantisus), closes [#1071](https://github.com/apify/crawlee-python/issues/1071)
+- Add `RedisStorageClient` based on Redis v8.0+ ([#1406](https://github.com/apify/crawlee-python/pull/1406)) ([d08d13d](https://github.com/apify/crawlee-python/commit/d08d13d39203c24ab61fe254b0956d6744db3b5f)) by [@Mantisus](https://github.com/Mantisus)
 
 ### 🐛 Bug Fixes
 

{crawlee-1.0.5b12 → crawlee-1.0.5b13}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: crawlee
-Version: 1.0.5b12
+Version: 1.0.5b13
 Summary: Crawlee for Python
 Project-URL: Apify Homepage, https://apify.com
 Project-URL: Changelog, https://crawlee.dev/python/docs/changelog
@@ -263,6 +263,7 @@ Requires-Dist: opentelemetry-sdk>=1.34.1; extra == 'all'
 Requires-Dist: opentelemetry-semantic-conventions>=0.54; extra == 'all'
 Requires-Dist: parsel>=1.10.0; extra == 'all'
 Requires-Dist: playwright>=1.27.0; extra == 'all'
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'all'
 Requires-Dist: rich>=13.9.0; extra == 'all'
 Requires-Dist: scikit-learn>=1.6.0; extra == 'all'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'all'
@@ -296,6 +297,8 @@ Provides-Extra: playwright
 Requires-Dist: apify-fingerprint-datapoints>=0.0.2; extra == 'playwright'
 Requires-Dist: browserforge>=1.2.3; extra == 'playwright'
 Requires-Dist: playwright>=1.27.0; extra == 'playwright'
+Provides-Extra: redis
+Requires-Dist: redis[hiredis]>=7.0.0; extra == 'redis'
 Provides-Extra: sql-postgres
 Requires-Dist: asyncpg>=0.24.0; extra == 'sql-postgres'
 Requires-Dist: sqlalchemy[asyncio]<3.0.0,>=2.0.0; extra == 'sql-postgres'

crawlee-1.0.5b13/docs/guides/code_examples/storage_clients/redis_storage_client_basic_example.py

@@ -0,0 +1,10 @@
+from crawlee.crawlers import ParselCrawler
+from crawlee.storage_clients import RedisStorageClient
+
+# Create a new instance of storage client using connection string.
+# 'redis://localhost:6379' is the just placeholder, replace it with your actual
+# connection string.
+storage_client = RedisStorageClient(connection_string='redis://localhost:6379')
+
+# And pass it to the crawler.
+crawler = ParselCrawler(storage_client=storage_client)

crawlee-1.0.5b13/docs/guides/code_examples/storage_clients/redis_storage_client_configuration_example.py

@@ -0,0 +1,27 @@
+from redis.asyncio import Redis
+
+from crawlee.configuration import Configuration
+from crawlee.crawlers import ParselCrawler
+from crawlee.storage_clients import RedisStorageClient
+
+# Create a new instance of storage client using a Redis client with custom settings.
+# Replace host and port with your actual Redis server configuration.
+# Other Redis client settings can be adjusted as needed.
+storage_client = RedisStorageClient(
+    redis=Redis(
+        host='localhost',
+        port=6379,
+        retry_on_timeout=True,
+        socket_keepalive=True,
+        socket_connect_timeout=10,
+    )
+)
+
+# Create a configuration with custom settings.
+configuration = Configuration(purge_on_start=False)
+
+# And pass them to the crawler.
+crawler = ParselCrawler(
+    storage_client=storage_client,
+    configuration=configuration,
+)

{crawlee-1.0.5b12 → crawlee-1.0.5b13}/docs/guides/storage_clients.mdx

@@ -17,6 +17,8 @@ import CustomStorageClientExample from '!!raw-loader!roa-loader!./code_examples/
 import RegisteringStorageClientsExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/registering_storage_clients_example.py';
 import SQLStorageClientBasicExample from '!!raw-loader!roa-loader!./code_examples/storage_clients/sql_storage_client_basic_example.py';
 import SQLStorageClientConfigurationExample from '!!raw-loader!./code_examples/storage_clients/sql_storage_client_configuration_example.py';
+import RedisStorageClientBasicExample from '!!raw-loader!./code_examples/storage_clients/redis_storage_client_basic_example.py';
+import RedisStorageClientConfigurationExample from '!!raw-loader!./code_examples/storage_clients/redis_storage_client_configuration_example.py';
 
 Storage clients provide a unified interface for interacting with <ApiLink to="class/Dataset">`Dataset`</ApiLink>, <ApiLink to="class/KeyValueStore">`KeyValueStore`</ApiLink>, and <ApiLink to="class/RequestQueue">`RequestQueue`</ApiLink>, regardless of the underlying implementation. They handle operations like creating, reading, updating, and deleting storage instances, as well as managing data persistence and cleanup. This abstraction makes it easy to switch between different environments, such as local development and cloud production setups.
 
@@ -26,7 +28,8 @@ Crawlee provides three main storage client implementations:
 
 - <ApiLink to="class/FileSystemStorageClient">`FileSystemStorageClient`</ApiLink> - Provides persistent file system storage with in-memory caching.
 - <ApiLink to="class/MemoryStorageClient">`MemoryStorageClient`</ApiLink> - Stores data in memory with no persistence.
-- <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> – Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/) or [PostgreSQL](https://www.postgresql.org/)). Requires installing the extra dependency: 'crawlee[sql_sqlite]' for SQLite or 'crawlee[sql_postgres]' for PostgreSQL.
+- <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> - Provides persistent storage using a SQL database ([SQLite](https://sqlite.org/) or [PostgreSQL](https://www.postgresql.org/)). Requires installing the extra dependency: `crawlee[sql_sqlite]` for SQLite or `crawlee[sql_postgres]` for PostgreSQL.
+- <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> - Provides persistent storage using a [Redis](https://redis.io/) database v8.0+. Requires installing the extra dependency `crawlee[redis]`.
 - [`ApifyStorageClient`](https://docs.apify.com/sdk/python/reference/class/ApifyStorageClient) - Manages storage on the [Apify platform](https://apify.com), implemented in the [Apify SDK](https://github.com/apify/apify-sdk-python).
 
 ```mermaid
@@ -56,6 +59,8 @@ class MemoryStorageClient
 
 class SqlStorageClient
 
+class RedisStorageClient
+
 class ApifyStorageClient
 
 %% ========================
@@ -65,6 +70,7 @@ class ApifyStorageClient
 StorageClient --|> FileSystemStorageClient
 StorageClient --|> MemoryStorageClient
 StorageClient --|> SqlStorageClient
+StorageClient --|> RedisStorageClient
 StorageClient --|> ApifyStorageClient
 ```
 
@@ -304,8 +310,8 @@ Configuration options for the <ApiLink to="class/SqlStorageClient">`SqlStorageCl
 
 Configuration options for the <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> can be set via constructor arguments:
 
-- **`connection_string`** (default: SQLite in <ApiLink to="class/Configuration">`Configuration`</ApiLink> storage dir) – SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db` or `postgresql+asyncpg://user:pass@host/db`.
-- **`engine`** – Pre-configured SQLAlchemy AsyncEngine (optional).
+- **`connection_string`** (default: SQLite in <ApiLink to="class/Configuration">`Configuration`</ApiLink> storage dir) - SQLAlchemy connection string, e.g. `sqlite+aiosqlite:///my.db` or `postgresql+asyncpg://user:pass@host/db`.
+- **`engine`** - Pre-configured SQLAlchemy AsyncEngine (optional).
 
 For advanced scenarios, you can configure <ApiLink to="class/SqlStorageClient">`SqlStorageClient`</ApiLink> with a custom SQLAlchemy engine and additional options via the <ApiLink to="class/Configuration">`Configuration`</ApiLink> class. This is useful, for example, when connecting to an external PostgreSQL database or customizing connection pooling.
 
@@ -313,6 +319,172 @@ For advanced scenarios, you can configure <ApiLink to="class/SqlStorageClient">`
     {SQLStorageClientConfigurationExample}
 </CodeBlock>
 
+### Redis storage client
+
+:::warning Experimental feature
+The <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> is experimental. Its API and behavior may change in future releases.
+:::
+
+The <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> provides persistent storage using [Redis](https://redis.io/) database. It supports concurrent access from multiple independent clients or processes and uses Redis native data structures for efficient operations.
+
+:::note dependencies
+The <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> is not included in the core Crawlee package.
+To use it, you need to install Crawlee with the Redis extra dependency:
+
+<code>pip install 'crawlee[redis]'</code>
+
+Additionally, Redis version 8.0 or higher is required.
+:::
+
+:::note Redis persistence
+Data persistence in Redis depends on your [database configuration](https://redis.io/docs/latest/operate/oss_and_stack/management/persistence/).
+:::
+
+The client requires either a Redis connection string or a pre-configured Redis client instance. Use a pre-configured client when you need custom Redis settings such as connection pooling, timeouts, or SSL/TLS encryption.
+
+<CodeBlock className="language-python" language="python">
+    {RedisStorageClientBasicExample}
+</CodeBlock>
+
+Data is organized using Redis key patterns. Below are the main data structures used for each storage type:
+
+```mermaid
+---
+config:
+    class:
+        hideEmptyMembersBox: true
+---
+
+classDiagram
+
+%% ========================
+%% Storage Client
+%% ========================
+
+class RedisDatasetClient {
+    <<Dataset>>
+}
+
+%% ========================
+%% Dataset Keys
+%% ========================
+
+class DatasetKeys {
+    datasets:[name]:items - JSON Array
+    datasets:[name]:metadata - JSON Object
+}
+
+class DatasetsIndexes {
+    datasets:id_to_name - Hash
+    datasets:name_to_id - Hash
+}
+
+%% ========================
+%% Client to Keys arrows
+%% ========================
+
+RedisDatasetClient --> DatasetKeys
+RedisDatasetClient --> DatasetsIndexes
+```
+
+```mermaid
+---
+config:
+    class:
+        hideEmptyMembersBox: true
+---
+
+classDiagram
+
+%% ========================
+%% Storage Clients
+%% ========================
+
+class RedisKeyValueStoreClient {
+    <<Key-value store>>
+}
+
+%% ========================
+%% Key-Value Store Keys
+%% ========================
+
+class KeyValueStoreKeys {
+    key_value_stores:[name]:items - Hash
+    key_value_stores:[name]:metadata_items - Hash
+    key_value_stores:[name]:metadata - JSON Object
+}
+
+class KeyValueStoresIndexes {
+    key_value_stores:id_to_name - Hash
+    key_value_stores:name_to_id - Hash
+}
+
+%% ========================
+%% Client to Keys arrows
+%% ========================
+
+RedisKeyValueStoreClient --> KeyValueStoreKeys
+RedisKeyValueStoreClient --> KeyValueStoresIndexes
+```
+
+```mermaid
+---
+config:
+    class:
+        hideEmptyMembersBox: true
+---
+
+classDiagram
+
+%% ========================
+%% Storage Clients
+%% ========================
+
+class RedisRequestQueueClient {
+    <<Request queue>>
+}
+
+%% ========================
+%% Request Queue Keys
+%% ========================
+
+class RequestQueueKeys{
+    request_queues:[name]:queue - List
+    request_queues:[name]:data - Hash
+    request_queues:[name]:in_progress - Hash
+    request_queues:[name]:added_bloom_filter - Bloom Filter | bloom queue_dedup_strategy
+    request_queues:[name]:handled_bloom_filter - Bloom Filter | bloom queue_dedup_strategy
+    request_queues:[name]:pending_set - Set | default queue_dedup_strategy
+    request_queues:[name]:handled_set - Set | default queue_dedup_strategy
+    request_queues:[name]:metadata - JSON Object
+}
+
+class RequestQueuesIndexes {
+    request_queues:id_to_name - Hash
+    request_queues:name_to_id - Hash
+}
+
+%% ========================
+%% Client to Keys arrows
+%% ========================
+
+RedisRequestQueueClient --> RequestQueueKeys
+RedisRequestQueueClient --> RequestQueuesIndexes
+```
+
+Configuration options for the <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> can be set through environment variables or the <ApiLink to="class/Configuration">`Configuration`</ApiLink> class:
+
+- **`purge_on_start`** (env: `CRAWLEE_PURGE_ON_START`, default: `True`) - Whether to purge default storages on start.
+
+Configuration options for the <ApiLink to="class/RedisStorageClient">`RedisStorageClient`</ApiLink> can be set via constructor arguments:
+
+- **`connection_string`** - Redis connection string, e.g. `redis://localhost:6379/0`.
+- **`redis`** - Pre-configured Redis client instance (optional).
+
+<CodeBlock className="language-python" language="python">
+    {RedisStorageClientConfigurationExample}
+</CodeBlock>
+
 ## Creating a custom storage client
 
 A storage client consists of two parts: the storage client factory and individual storage type clients. The <ApiLink to="class/StorageClient">`StorageClient`</ApiLink> acts as a factory that creates specific clients (<ApiLink to="class/DatasetClient">`DatasetClient`</ApiLink>, <ApiLink to="class/KeyValueStoreClient">`KeyValueStoreClient`</ApiLink>, <ApiLink to="class/RequestQueueClient">`RequestQueueClient`</ApiLink>) where the actual storage logic is implemented.

{crawlee-1.0.5b12 → crawlee-1.0.5b13}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "crawlee"
-version = "1.0.5b12"
+version = "1.0.5b13"
 description = "Crawlee for Python"
 authors = [{ name = "Apify Technologies s.r.o.", email = "support@apify.com" }]
 license = { file = "LICENSE" }
@@ -48,7 +48,7 @@ dependencies = [
 ]
 
 [project.optional-dependencies]
-all = ["crawlee[adaptive-crawler,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql_sqlite,sql_postgres]"]
+all = ["crawlee[adaptive-crawler,beautifulsoup,cli,curl-impersonate,httpx,parsel,playwright,otel,sql_sqlite,sql_postgres,redis]"]
 adaptive-crawler = [
     "jaro-winkler>=2.0.3",
     "playwright>=1.27.0",
@@ -79,6 +79,7 @@ sql_sqlite = [
     "sqlalchemy[asyncio]>=2.0.0,<3.0.0",
     "aiosqlite>=0.21.0",
 ]
+redis = ["redis[hiredis] >= 7.0.0"]
 
 [project.scripts]
 crawlee = "crawlee._cli:cli"
@@ -98,6 +99,7 @@ dev = [
     "apify_client", # For e2e tests.
     "build<2.0.0", # For e2e tests.
     "dycw-pytest-only<3.0.0",
+    "fakeredis[probabilistic,json,lua]<3.0.0",
     "mypy~=1.18.0",
     "pre-commit<5.0.0",
     "proxy-py<3.0.0",

{crawlee-1.0.5b12 → crawlee-1.0.5b13}/src/crawlee/storage_clients/__init__.py

@@ -13,9 +13,13 @@ _install_import_hook(__name__)
 with _try_import(__name__, 'SqlStorageClient'):
     from ._sql import SqlStorageClient
 
+with _try_import(__name__, 'RedisStorageClient'):
+    from ._redis import RedisStorageClient
+
 __all__ = [
     'FileSystemStorageClient',
     'MemoryStorageClient',
+    'RedisStorageClient',
     'SqlStorageClient',
     'StorageClient',
 ]

crawlee-1.0.5b13/src/crawlee/storage_clients/_redis/__init__.py

@@ -0,0 +1,6 @@
+from ._dataset_client import RedisDatasetClient
+from ._key_value_store_client import RedisKeyValueStoreClient
+from ._request_queue_client import RedisRequestQueueClient
+from ._storage_client import RedisStorageClient
+
+__all__ = ['RedisDatasetClient', 'RedisKeyValueStoreClient', 'RedisRequestQueueClient', 'RedisStorageClient']

crawlee-1.0.5b13/src/crawlee/storage_clients/_redis/_client_mixin.py

@@ -0,0 +1,295 @@
+from __future__ import annotations
+
+import asyncio
+from contextlib import asynccontextmanager
+from datetime import datetime, timezone
+from logging import getLogger
+from typing import TYPE_CHECKING, Any, ClassVar, TypedDict, overload
+
+from crawlee._utils.crypto import crypto_random_object_id
+
+from ._utils import await_redis_response, read_lua_script
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncIterator
+
+    from redis.asyncio import Redis
+    from redis.asyncio.client import Pipeline
+    from redis.commands.core import AsyncScript
+    from typing_extensions import NotRequired, Self
+
+    from crawlee.storage_clients.models import DatasetMetadata, KeyValueStoreMetadata, RequestQueueMetadata
+
+
+logger = getLogger(__name__)
+
+
+class MetadataUpdateParams(TypedDict, total=False):
+    """Parameters for updating metadata."""
+
+    update_accessed_at: NotRequired[bool]
+    update_modified_at: NotRequired[bool]
+
+
+class RedisClientMixin:
+    """Mixin class for Redis clients.
+
+    This mixin provides common Redis operations and basic methods for Redis storage clients.
+    """
+
+    _DEFAULT_NAME = 'default'
+    """Default storage name in key prefix when none provided."""
+
+    _MAIN_KEY: ClassVar[str]
+    """Main Redis key prefix for this storage type."""
+
+    _CLIENT_TYPE: ClassVar[str]
+    """Human-readable client type for error messages."""
+
+    def __init__(self, storage_name: str, storage_id: str, redis: Redis) -> None:
+        self._storage_name = storage_name
+        self._storage_id = storage_id
+        self._redis = redis
+
+        self._scripts_loaded = False
+
+    @property
+    def redis(self) -> Redis:
+        """Return the Redis client instance."""
+        return self._redis
+
+    @property
+    def metadata_key(self) -> str:
+        """Return the Redis key for the metadata of this storage."""
+        return f'{self._MAIN_KEY}:{self._storage_name}:metadata'
+
+    @classmethod
+    async def _get_metadata_by_name(cls, name: str, redis: Redis, *, with_wait: bool = False) -> dict | None:
+        """Retrieve metadata by storage name.
+
+        Args:
+            name: The name of the storage.
+            redis: The Redis client instance.
+            with_wait: Whether to wait for the storage to be created if it doesn't exist.
+        """
+        if with_wait:
+            # Wait for the creation signal (max 30 seconds)
+            await await_redis_response(redis.blpop([f'{cls._MAIN_KEY}:{name}:created_signal'], timeout=30))
+            # Signal consumed, push it back for other waiters
+            await await_redis_response(redis.lpush(f'{cls._MAIN_KEY}:{name}:created_signal', 1))
+
+        response = await await_redis_response(redis.json().get(f'{cls._MAIN_KEY}:{name}:metadata'))
+        data = response[0] if response is not None and isinstance(response, list) else response
+        if data is not None and not isinstance(data, dict):
+            raise TypeError('The metadata data was received in an incorrect format.')
+        return data
+
+    @classmethod
+    async def _get_metadata_name_by_id(cls, id: str, redis: Redis) -> str | None:
+        """Retrieve storage name by ID from id_to_name index.
+
+        Args:
+            id: The ID of the storage.
+            redis: The Redis client instance.
+        """
+        name = await await_redis_response(redis.hget(f'{cls._MAIN_KEY}:id_to_name', id))
+        if isinstance(name, str) or name is None:
+            return name
+        if isinstance(name, bytes):
+            return name.decode('utf-8')
+        return None
+
+    @classmethod
+    async def _open(
+        cls,
+        *,
+        id: str | None,
+        name: str | None,
+        alias: str | None,
+        metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata],
+        redis: Redis,
+        extra_metadata_fields: dict[str, Any],
+        instance_kwargs: dict[str, Any],
+    ) -> Self:
+        """Open or create a new Redis storage client.
+
+        Args:
+            id: The ID of the storage. If not provided, a random ID will be generated.
+            name: The name of the storage for named (global scope) storages.
+            alias: The alias of the storage for unnamed (run scope) storages.
+            redis: Redis client instance.
+            metadata_model: Pydantic model for metadata validation.
+            extra_metadata_fields: Storage-specific metadata fields.
+            instance_kwargs: Additional arguments for the client constructor.
+
+        Returns:
+            An instance for the opened or created storage client.
+        """
+        internal_name = name or alias or cls._DEFAULT_NAME
+        storage_id: str | None = None
+        # Determine if storage exists by ID or name
+        if id:
+            storage_name = await cls._get_metadata_name_by_id(id=id, redis=redis)
+            storage_id = id
+            if storage_name is None:
+                raise ValueError(f'{cls._CLIENT_TYPE} with ID "{id}" does not exist.')
+        else:
+            metadata_data = await cls._get_metadata_by_name(name=internal_name, redis=redis)
+            storage_name = internal_name if metadata_data is not None else None
+            storage_id = metadata_data['id'] if metadata_data is not None else None
+        # If both storage_name and storage_id are found, open existing storage
+        if storage_name and storage_id:
+            client = cls(storage_name=storage_name, storage_id=storage_id, redis=redis, **instance_kwargs)
+            async with client._get_pipeline() as pipe:
+                await client._update_metadata(pipe, update_accessed_at=True)
+        # Otherwise, create a new storage
+        else:
+            now = datetime.now(timezone.utc)
+            metadata = metadata_model(
+                id=crypto_random_object_id(),
+                name=name,
+                created_at=now,
+                accessed_at=now,
+                modified_at=now,
+                **extra_metadata_fields,
+            )
+            client = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs)
+            created = await client._create_metadata_and_storage(internal_name, metadata.model_dump())
+            # The client was probably not created due to a race condition. Let's try to open it using the name.
+            if not created:
+                metadata_data = await cls._get_metadata_by_name(name=internal_name, redis=redis, with_wait=True)
+                client = cls(storage_name=internal_name, storage_id=metadata.id, redis=redis, **instance_kwargs)
+
+        # Ensure Lua scripts are loaded
+        await client._ensure_scripts_loaded()
+        return client
+
+    async def _load_scripts(self) -> None:
+        """Load Lua scripts in Redis."""
+        return
+
+    async def _ensure_scripts_loaded(self) -> None:
+        """Ensure Lua scripts are loaded in Redis."""
+        if not self._scripts_loaded:
+            await self._load_scripts()
+            self._scripts_loaded = True
+
+    @asynccontextmanager
+    async def _get_pipeline(self, *, with_execute: bool = True) -> AsyncIterator[Pipeline]:
+        """Create a new Redis pipeline."""
+        async with self._redis.pipeline() as pipe:
+            try:
+                pipe.multi()  # type: ignore[no-untyped-call]
+                yield pipe
+            finally:
+                if with_execute:
+                    await pipe.execute()
+
+    async def _create_storage(self, pipeline: Pipeline) -> None:
+        """Create the actual storage structure in Redis."""
+        _ = pipeline  # To avoid unused variable mypy error
+
+    async def _create_script(self, script_name: str) -> AsyncScript:
+        """Load a Lua script from a file and return a Script object."""
+        script_content = await asyncio.to_thread(read_lua_script, script_name)
+
+        return self._redis.register_script(script_content)
+
+    async def _create_metadata_and_storage(self, storage_name: str, metadata: dict) -> bool:
+        index_id_to_name = f'{self._MAIN_KEY}:id_to_name'
+        index_name_to_id = f'{self._MAIN_KEY}:name_to_id'
+        metadata['created_at'] = metadata['created_at'].isoformat()
+        metadata['accessed_at'] = metadata['accessed_at'].isoformat()
+        metadata['modified_at'] = metadata['modified_at'].isoformat()
+
+        # Try to create name_to_id index entry, if it already exists, return False.
+        name_to_id = await await_redis_response(self._redis.hsetnx(index_name_to_id, storage_name, metadata['id']))
+        # If name already exists, return False. Probably an attempt at parallel creation.
+        if not name_to_id:
+            return False
+
+        # Create id_to_name index entry, metadata, and storage structure in a transaction.
+        async with self._get_pipeline() as pipe:
+            await await_redis_response(pipe.hsetnx(index_id_to_name, metadata['id'], storage_name))
+            await await_redis_response(pipe.json().set(self.metadata_key, '$', metadata))
+            await await_redis_response(pipe.lpush(f'{self._MAIN_KEY}:{storage_name}:created_signal', 1))
+
+            await self._create_storage(pipe)
+
+        return True
+
+    async def _drop(self, extra_keys: list[str]) -> None:
+        async with self._get_pipeline() as pipe:
+            await pipe.delete(self.metadata_key)
+            await pipe.delete(f'{self._MAIN_KEY}:id_to_name', self._storage_id)
+            await pipe.delete(f'{self._MAIN_KEY}:name_to_id', self._storage_name)
+            await pipe.delete(f'{self._MAIN_KEY}:{self._storage_name}:created_signal')
+            for key in extra_keys:
+                await pipe.delete(key)
+
+    async def _purge(self, extra_keys: list[str], metadata_kwargs: MetadataUpdateParams) -> None:
+        async with self._get_pipeline() as pipe:
+            for key in extra_keys:
+                await pipe.delete(key)
+            await self._update_metadata(pipe, **metadata_kwargs)
+            await self._create_storage(pipe)
+
+    @overload
+    async def _get_metadata(self, metadata_model: type[DatasetMetadata]) -> DatasetMetadata: ...
+    @overload
+    async def _get_metadata(self, metadata_model: type[KeyValueStoreMetadata]) -> KeyValueStoreMetadata: ...
+    @overload
+    async def _get_metadata(self, metadata_model: type[RequestQueueMetadata]) -> RequestQueueMetadata: ...
+
+    async def _get_metadata(
+        self, metadata_model: type[DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata]
+    ) -> DatasetMetadata | KeyValueStoreMetadata | RequestQueueMetadata:
+        """Retrieve client metadata."""
+        metadata_dict = await self._get_metadata_by_name(name=self._storage_name, redis=self._redis)
+        if metadata_dict is None:
+            raise ValueError(f'{self._CLIENT_TYPE} with name "{self._storage_name}" does not exist.')
+        async with self._get_pipeline() as pipe:
+            await self._update_metadata(pipe, update_accessed_at=True)
+
+        return metadata_model.model_validate(metadata_dict)
+
+    async def _specific_update_metadata(self, pipeline: Pipeline, **kwargs: Any) -> None:
+        """Pipeline operations storage-specific metadata updates.
+
+        Must be implemented by concrete classes.
+
+        Args:
+            pipeline: The Redis pipeline to use for the update.
+            **kwargs: Storage-specific update parameters.
+        """
+        _ = pipeline  # To avoid unused variable mypy error
+        _ = kwargs
+
+    async def _update_metadata(
+        self,
+        pipeline: Pipeline,
+        *,
+        update_accessed_at: bool = False,
+        update_modified_at: bool = False,
+        **kwargs: Any,
+    ) -> None:
+        """Update storage metadata combining common and specific fields.
+
+        Args:
+            pipeline: The Redis pipeline to use for the update.
+            update_accessed_at: Whether to update accessed_at timestamp.
+            update_modified_at: Whether to update modified_at timestamp.
+            **kwargs: Additional arguments for _specific_update_metadata.
+        """
+        now = datetime.now(timezone.utc)
+
+        if update_accessed_at:
+            await await_redis_response(
+                pipeline.json().set(self.metadata_key, '$.accessed_at', now.isoformat(), nx=False, xx=True)
+            )
+        if update_modified_at:
+            await await_redis_response(
+                pipeline.json().set(self.metadata_key, '$.modified_at', now.isoformat(), nx=False, xx=True)
+            )
+
+        await self._specific_update_metadata(pipeline, **kwargs)