cledar-sdk 2.0.3__py3-none-any.whl → 2.1.0__py3-none-any.whl

cledar/redis/redis.py

@@ -1,3 +1,5 @@
+"""Redis service implementations for the Cledar SDK."""
+
 import json
 import logging
 from dataclasses import dataclass
@@ -20,11 +22,18 @@ logger = logging.getLogger("redis_service")
 
 
 class CustomEncoder(json.JSONEncoder):
-    """
-    Custom JSON encoder that can handle Enum objects and datetime objects.
-    """
+    """Custom JSON encoder that can handle Enum objects and datetime objects."""
 
     def default(self, o: Any) -> Any:
+        """Process objects for JSON encoding.
+
+        Args:
+            o: The object to encode.
+
+        Returns:
+            Any: The encoded object.
+
+        """
         if isinstance(o, Enum):
             return o.name.lower()
         if isinstance(o, datetime):
@@ -37,12 +46,16 @@ T = TypeVar("T", bound=BaseModel)
 
 @dataclass
 class FailedValue:
+    """Represents a failed Redis operation for a specific key."""
+
     key: str
     error: Exception
 
 
 @dataclass
 class RedisServiceConfig:
+    """Configuration for Redis services."""
+
     redis_host: str
     redis_port: int
     redis_db: int = 0
@@ -50,12 +63,26 @@ class RedisServiceConfig:
 
 
 class RedisService:
+    """Synchronous Redis service with Pydantic model support."""
+
     def __init__(self, config: RedisServiceConfig):
+        """Initialize the synchronous Redis service.
+
+        Args:
+            config: The service configuration.
+
+        """
         self.config = config
         self._client: redis.Redis
         self.connect()
 
     def connect(self) -> None:
+        """Establish a connection to Redis.
+
+        Raises:
+            RedisConnectionError: If the connection fails.
+
+        """
         try:
             self._client = redis.Redis(
                 host=self.config.redis_host,
@@ -82,6 +109,12 @@ class RedisService:
             raise RedisConnectionError("Could not initialize Redis client") from exc
 
     def is_alive(self) -> bool:
+        """Check if the Redis connection is alive.
+
+        Returns:
+            bool: True if connection is alive, False otherwise.
+
+        """
         try:
             return bool(self._client.ping())
         except redis.ConnectionError:
@@ -92,9 +125,17 @@ class RedisService:
             return False
 
     def _prepare_for_serialization(self, value: Any) -> Any:
-        """
+        """Process data structures for serialization.
+
         Recursively process data structures, converting BaseModel instances to
         serializable dicts.
+
+        Args:
+            value: The value to process.
+
+        Returns:
+            Any: The processed value.
+
         """
         if isinstance(value, BaseModel):
             return value.model_dump()
@@ -105,6 +146,22 @@ class RedisService:
         return value
 
     def set(self, key: str, value: Any) -> bool:
+        """Set a value in Redis.
+
+        Args:
+            key: The key to set.
+            value: The value to set.
+
+        Returns:
+            bool: True if successful, False otherwise.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisSerializationError: If serialization fails.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
         if value is None:
@@ -139,6 +196,22 @@ class RedisService:
             raise RedisOperationError(f"Failed to set key '{key}'") from exc
 
     def get(self, key: str, model: type[T]) -> T | None:
+        """Get a value from Redis and validate it against a model.
+
+        Args:
+            key: The key to fetch.
+            model: The Pydantic model to validate against.
+
+        Returns:
+            T | None: The validated model or None if key does not exist.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisDeserializationError: If decoding or validation fails.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
 
@@ -175,6 +248,20 @@ class RedisService:
             raise RedisOperationError(f"Failed to get key '{key}'") from exc
 
     def get_raw(self, key: str) -> Any | None:
+        """Get a raw value from Redis.
+
+        Args:
+            key: The key to fetch.
+
+        Returns:
+            Any | None: The raw value or None if key does not exist.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
 
@@ -195,6 +282,20 @@ class RedisService:
             raise RedisOperationError(f"Failed to get key '{key}'") from exc
 
     def list_keys(self, pattern: str) -> list[str]:
+        """List all keys matching a pattern.
+
+        Args:
+            pattern: The pattern to match.
+
+        Returns:
+            list[str]: A list of matching keys.
+
+        Raises:
+            ValueError: If the pattern is not a string.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(pattern, str):
             raise ValueError(f"Pattern must be a string, got {type(pattern)}")
 
@@ -215,6 +316,22 @@ class RedisService:
             ) from exc
 
     def mget(self, keys: list[str], model: type[T]) -> list[T | None | FailedValue]:
+        """Get multiple values from Redis and validate them against a model.
+
+        Args:
+            keys: A list of keys to fetch.
+            model: The Pydantic model to validate against.
+
+        Returns:
+            list[T | None | FailedValue]: A list of validated models, None for
+                missing keys, or FailedValue for items that failed validation.
+
+        Raises:
+            ValueError: If the keys are not a list.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(keys, list):
             raise ValueError(f"Keys must be a list, got {type(keys)}")
 
@@ -260,6 +377,20 @@ class RedisService:
             raise RedisOperationError("Failed to mget keys") from exc
 
     def delete(self, key: str) -> bool:
+        """Delete a key from Redis.
+
+        Args:
+            key: The key to delete.
+
+        Returns:
+            bool: True if the key was deleted, False otherwise.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
 
@@ -283,11 +414,22 @@ class AsyncRedisService:
     """Asynchronous Redis service with async/await support."""
 
     def __init__(self, config: RedisServiceConfig):
+        """Initialize the asynchronous Redis service.
+
+        Args:
+            config: The service configuration.
+
+        """
         self.config = config
         self._client: aioredis.Redis
 
     async def connect(self) -> None:
-        """Establish connection to Redis asynchronously."""
+        """Establish connection to Redis asynchronously.
+
+        Raises:
+            RedisConnectionError: If the connection fails.
+
+        """
         try:
             self._client = aioredis.Redis(
                 host=self.config.redis_host,
@@ -319,7 +461,12 @@ class AsyncRedisService:
         logger.info("Async Redis client closed.")
 
     async def is_alive(self) -> bool:
-        """Check if Redis connection is alive."""
+        """Check if Redis connection is alive.
+
+        Returns:
+            bool: True if the connection is alive, False otherwise.
+
+        """
         try:
             return bool(await self._client.ping())
         except aioredis.ConnectionError:
@@ -330,9 +477,17 @@ class AsyncRedisService:
             return False
 
     def _prepare_for_serialization(self, value: Any) -> Any:
-        """
+        """Process data structures for serialization.
+
         Recursively process data structures, converting BaseModel instances to
         serializable dicts.
+
+        Args:
+            value: The value to process.
+
+        Returns:
+            Any: The processed value.
+
         """
         if isinstance(value, BaseModel):
             return value.model_dump()
@@ -343,7 +498,22 @@ class AsyncRedisService:
         return value
 
     async def set(self, key: str, value: Any) -> bool:
-        """Set a key-value pair in Redis."""
+        """Set a key-value pair in Redis.
+
+        Args:
+            key: The key to set.
+            value: The value to set.
+
+        Returns:
+            bool: True if successful, False otherwise.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisSerializationError: If serialization fails.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
         if value is None:
@@ -378,7 +548,22 @@ class AsyncRedisService:
             raise RedisOperationError(f"Failed to set key '{key}'") from exc
 
     async def get(self, key: str, model: type[T]) -> T | None:
-        """Get a value from Redis and validate it against a Pydantic model."""
+        """Get a value from Redis and validate it against a Pydantic model.
+
+        Args:
+            key: The key to fetch.
+            model: The Pydantic model to validate against.
+
+        Returns:
+            T | None: The validated model or None if key does not exist.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisDeserializationError: If decoding or validation fails.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
 
@@ -415,7 +600,20 @@ class AsyncRedisService:
             raise RedisOperationError(f"Failed to get key '{key}'") from exc
 
     async def get_raw(self, key: str) -> Any | None:
-        """Get a raw value from Redis without deserialization."""
+        """Get a raw value from Redis without deserialization.
+
+        Args:
+            key: The key to fetch.
+
+        Returns:
+            Any | None: The raw value or None if key does not exist.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
 
@@ -436,7 +634,20 @@ class AsyncRedisService:
             raise RedisOperationError(f"Failed to get key '{key}'") from exc
 
     async def list_keys(self, pattern: str) -> list[str]:
-        """List keys matching a pattern."""
+        """List keys matching a pattern.
+
+        Args:
+            pattern: The pattern to match.
+
+        Returns:
+            list[str]: A list of matching keys.
+
+        Raises:
+            ValueError: If the pattern is not a string.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(pattern, str):
             raise ValueError(f"Pattern must be a string, got {type(pattern)}")
 
@@ -459,7 +670,22 @@ class AsyncRedisService:
     async def mget(
         self, keys: list[str], model: type[T]
     ) -> list[T | None | FailedValue]:
-        """Get multiple values from Redis."""
+        """Get multiple values from Redis.
+
+        Args:
+            keys: A list of keys to fetch.
+            model: The Pydantic model to validate against.
+
+        Returns:
+            list[T | None | FailedValue]: A list of validated models, None for
+                missing keys, or FailedValue for items that failed validation.
+
+        Raises:
+            ValueError: If the keys are not a list.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(keys, list):
             raise ValueError(f"Keys must be a list, got {type(keys)}")
 
@@ -505,7 +731,20 @@ class AsyncRedisService:
             raise RedisOperationError("Failed to mget keys") from exc
 
     async def delete(self, key: str) -> bool:
-        """Delete a key from Redis."""
+        """Delete a key from Redis.
+
+        Args:
+            key: The key to delete.
+
+        Returns:
+            bool: True if the key was deleted, False otherwise.
+
+        Raises:
+            ValueError: If the key is not a string.
+            RedisConnectionError: If the connection fails.
+            RedisOperationError: If the operation fails.
+
+        """
         if not isinstance(key, str):
             raise ValueError(f"Key must be a string, got {type(key)}")
 

cledar/redis/redis_config_store.py

@@ -1,3 +1,5 @@
+"""Redis-based configuration store with caching and watching capabilities."""
+
 import json
 import re
 import time
@@ -19,6 +21,11 @@ OP_EVENT_FORMAT = "__keyevent@{DB}__:{OPERATION}"
 
 
 class RedisConfigStore:
+    """Store for configuration objects in Redis with local caching and pub/sub.
+
+    Provides updates on configuration changes.
+    """
+
     TYPE_NONE = "none"
     TYPE_LIST = "list"
     TYPE_STRING = "string"
@@ -29,6 +36,13 @@ class RedisConfigStore:
     EVENT_LSET = "lset"
 
     def __init__(self, redis: Redis, prefix: str | None = None) -> None:
+        """Initialize the RedisConfigStore.
+
+        Args:
+            redis: An initialized Redis client.
+            prefix: Optional prefix for keys stored in Redis.
+
+        """
         self._redis: Redis = redis
         self._pubsub = redis.pubsub()  # type: ignore
         self._db: int = redis.connection_pool.connection_kwargs.get("db")
@@ -42,18 +56,52 @@ class RedisConfigStore:
         self._watcher_thread.start()
 
     def is_ready(self) -> bool:
+        """Check if the Redis connection is ready.
+
+        Returns:
+            bool: True if Redis responds to ping, False otherwise.
+
+        """
         try:
             return self._redis.ping()  # type: ignore
         except RedisConnectionError:
             return False
 
     def versions(self, key: str) -> int | None:
+        """Get the version (e.g., list length or existence) of a key.
+
+        Args:
+            key: The key to check.
+
+        Returns:
+            int | None: The version of the key.
+
+        """
         return self._key_versions(key)
 
     def cached_version(self, key: str) -> int | None:
+        """Get the version of a key stored in the local cache.
+
+        Args:
+            key: The key to check.
+
+        Returns:
+            int | None: The cached version of the key.
+
+        """
         return self._cache_verisons.get(key)
 
     def fetch(self, cls: type[T], key: str) -> T | None:
+        """Fetch a configuration object from Redis or local cache.
+
+        Args:
+            cls: The class to instantiate with the fetched data.
+            key: The key associated with the configuration.
+
+        Returns:
+            T | None: The configuration object, or None if not found.
+
+        """
         if key not in self._cache:
             new_value = self._key_fetch(key)
             if new_value is None:
@@ -64,11 +112,24 @@ class RedisConfigStore:
         return cls(**json.loads(self._cache[key]))
 
     def update(self, key: str, value: T) -> None:
+        """Update a configuration object in Redis and local cache.
+
+        Args:
+            key: The key to update.
+            value: The configuration object to store.
+
+        """
         self._cache[key] = self._key_update(key, value)
         self._cache_verisons[key] = self._key_versions(key) or -1
         self._key_watch(key)
 
     def delete(self, key: str) -> None:
+        """Delete a configuration object from Redis and local cache.
+
+        Args:
+            key: The key to delete.
+
+        """
         if key in self._cache:
             del self._cache[key]
             del self._cache_verisons[key]
@@ -78,12 +139,32 @@ class RedisConfigStore:
     def watch(
         self, key: str, callback: Callable[[int, str, str, str], None] | None = None
     ) -> None:
+        """Watch a key for changes and execute a callback.
+
+        Args:
+            key: The key to watch.
+            callback: The callback function to execute on change.
+
+        """
         self._key_watch(key, callback)
 
     def __setitem__(self, key: str, value: T) -> None:
+        """Alias for update method.
+
+        Args:
+            key: The key to update.
+            value: The value to set.
+
+        """
         self.update(key, value)
 
     def __delitem__(self, key: str) -> None:
+        """Alias for delete method.
+
+        Args:
+            key: The key to delete.
+
+        """
         self.delete(key)
 
     def _key_watch(

cledar/storage/__init__.py

@@ -1,3 +1,5 @@
+"""Object storage module for handling S3, ABFS, and local filesystem operations."""
+
 from .models import ObjectStorageServiceConfig
 from .object_storage import ObjectStorageService
 

cledar/storage/constants.py

@@ -1,3 +1,5 @@
+"""Constants for object storage path prefixes."""
+
 S3_PATH_PREFIX = "s3://"
 ABFS_PATH_PREFIX = "abfs://"
 ABFSS_PATH_PREFIX = "abfss://"

cledar/storage/exceptions.py

@@ -1,50 +1,79 @@
+"""Exceptions for object storage operations."""
+
+
 class ObjectStorageError(Exception):
+    """Base exception for object storage errors."""
+
     pass
 
 
 class RequiredBucketNotFoundError(ObjectStorageError):
+    """Exception raised when a required bucket is not found."""
+
     pass
 
 
 class UploadBufferError(ObjectStorageError):
+    """Exception raised when uploading a buffer fails."""
+
     pass
 
 
 class UploadFileError(ObjectStorageError):
+    """Exception raised when uploading a file fails."""
+
     pass
 
 
 class ReadFileError(ObjectStorageError):
+    """Exception raised when reading a file fails."""
+
     pass
 
 
 class DownloadFileError(ObjectStorageError):
+    """Exception raised when downloading a file fails."""
+
     pass
 
 
 class ListObjectsError(ObjectStorageError):
+    """Exception raised when listing objects fails."""
+
     pass
 
 
 class DeleteFileError(ObjectStorageError):
+    """Exception raised when deleting a file fails."""
+
     pass
 
 
 class GetFileSizeError(ObjectStorageError):
+    """Exception raised when getting file size fails."""
+
     pass
 
 
 class GetFileInfoError(ObjectStorageError):
+    """Exception raised when getting file info fails."""
+
     pass
 
 
 class CopyFileError(ObjectStorageError):
+    """Exception raised when copying a file fails."""
+
     pass
 
 
 class MoveFileError(ObjectStorageError):
+    """Exception raised when moving a file fails."""
+
     pass
 
 
 class CheckFileExistenceError(ObjectStorageError):
+    """Exception raised when checking file existence fails."""
+
     pass

cledar/storage/models.py

@@ -1,9 +1,23 @@
+"""Models for object storage configuration and transfer paths."""
+
 from typing import Literal
 
 from pydantic import BaseModel
 
 
 class ObjectStorageServiceConfig(BaseModel):
+    """Configuration for object storage service.
+
+    Attributes:
+        s3_endpoint_url: S3 endpoint URL for custom S3-compatible services.
+        s3_access_key: S3 access key for authentication.
+        s3_secret_key: S3 secret key for authentication.
+        s3_max_concurrency: Maximum number of concurrent S3 operations.
+        azure_account_name: Azure storage account name.
+        azure_account_key: Azure storage account key.
+
+    """
+
     # s3 configuration
     s3_endpoint_url: str | None = None
     s3_access_key: str | None = None
@@ -15,5 +29,13 @@ class ObjectStorageServiceConfig(BaseModel):
 
 
 class TransferPath(BaseModel):
+    """Transfer path information with backend type.
+
+    Attributes:
+        backend: Storage backend type (s3, abfs, or local).
+        path: Full path to the file or object.
+
+    """
+
     backend: Literal["s3", "abfs", "local"]
     path: str