cledar-sdk 2.0.2__py3-none-any.whl → 2.1.0__py3-none-any.whl

cledar/redis/redis_config_store.py

@@ -0,0 +1,333 @@
+"""Redis-based configuration store with caching and watching capabilities."""
+
+import json
+import re
+import time
+from collections.abc import Callable
+from dataclasses import asdict
+from threading import Thread
+from typing import Any
+
+from redis import ConnectionError as RedisConnectionError
+from redis import Redis
+
+from .logger import logger
+from .model import ConfigAbstract as T
+
+KEY_EVENT_FORMAT = "__keyspace@{DB}__:{KEY}"
+KEY_EVENT_REGEX = r"__keyspace@(?P<db>\d+)__:(?P<key>.+)"
+
+OP_EVENT_FORMAT = "__keyevent@{DB}__:{OPERATION}"
+
+
+class RedisConfigStore:
+    """Store for configuration objects in Redis with local caching and pub/sub.
+
+    Provides updates on configuration changes.
+    """
+
+    TYPE_NONE = "none"
+    TYPE_LIST = "list"
+    TYPE_STRING = "string"
+
+    EVENT_DELETE = "del"
+    EVENT_SET = "set"
+    EVENT_RPUSH = "rpush"
+    EVENT_LSET = "lset"
+
+    def __init__(self, redis: Redis, prefix: str | None = None) -> None:
+        """Initialize the RedisConfigStore.
+
+        Args:
+            redis: An initialized Redis client.
+            prefix: Optional prefix for keys stored in Redis.
+
+        """
+        self._redis: Redis = redis
+        self._pubsub = redis.pubsub()  # type: ignore
+        self._db: int = redis.connection_pool.connection_kwargs.get("db")
+        self._prefix: str = prefix or ""
+        self._cache: dict[str, str] = {}
+        self._cache_verisons: dict[str, int] = {}
+        self._monitoring: dict[
+            str, list[Callable[[int, str, str, str], None] | None]
+        ] = {}
+        self._watcher_thread: Thread = Thread(target=self._watcher)
+        self._watcher_thread.start()
+
+    def is_ready(self) -> bool:
+        """Check if the Redis connection is ready.
+
+        Returns:
+            bool: True if Redis responds to ping, False otherwise.
+
+        """
+        try:
+            return self._redis.ping()  # type: ignore
+        except RedisConnectionError:
+            return False
+
+    def versions(self, key: str) -> int | None:
+        """Get the version (e.g., list length or existence) of a key.
+
+        Args:
+            key: The key to check.
+
+        Returns:
+            int | None: The version of the key.
+
+        """
+        return self._key_versions(key)
+
+    def cached_version(self, key: str) -> int | None:
+        """Get the version of a key stored in the local cache.
+
+        Args:
+            key: The key to check.
+
+        Returns:
+            int | None: The cached version of the key.
+
+        """
+        return self._cache_verisons.get(key)
+
+    def fetch(self, cls: type[T], key: str) -> T | None:
+        """Fetch a configuration object from Redis or local cache.
+
+        Args:
+            cls: The class to instantiate with the fetched data.
+            key: The key associated with the configuration.
+
+        Returns:
+            T | None: The configuration object, or None if not found.
+
+        """
+        if key not in self._cache:
+            new_value = self._key_fetch(key)
+            if new_value is None:
+                return None
+            self._cache[key] = new_value
+            self._cache_verisons[key] = self._key_versions(key) or -1
+            self._key_watch(key)
+        return cls(**json.loads(self._cache[key]))
+
+    def update(self, key: str, value: T) -> None:
+        """Update a configuration object in Redis and local cache.
+
+        Args:
+            key: The key to update.
+            value: The configuration object to store.
+
+        """
+        self._cache[key] = self._key_update(key, value)
+        self._cache_verisons[key] = self._key_versions(key) or -1
+        self._key_watch(key)
+
+    def delete(self, key: str) -> None:
+        """Delete a configuration object from Redis and local cache.
+
+        Args:
+            key: The key to delete.
+
+        """
+        if key in self._cache:
+            del self._cache[key]
+            del self._cache_verisons[key]
+        self._key_delete(key)
+        self._key_watch(key)
+
+    def watch(
+        self, key: str, callback: Callable[[int, str, str, str], None] | None = None
+    ) -> None:
+        """Watch a key for changes and execute a callback.
+
+        Args:
+            key: The key to watch.
+            callback: The callback function to execute on change.
+
+        """
+        self._key_watch(key, callback)
+
+    def __setitem__(self, key: str, value: T) -> None:
+        """Alias for update method.
+
+        Args:
+            key: The key to update.
+            value: The value to set.
+
+        """
+        self.update(key, value)
+
+    def __delitem__(self, key: str) -> None:
+        """Alias for delete method.
+
+        Args:
+            key: The key to delete.
+
+        """
+        self.delete(key)
+
+    def _key_watch(
+        self, key: str, callback: Callable[[int, str, str, str], None] | None = None
+    ) -> None:
+        if key not in self._monitoring:
+            self._monitoring[key] = []
+
+        if callback in self._monitoring[key]:
+            return
+
+        self._monitoring[key].append(callback)
+        callbacks = list(self._monitoring[key])
+        event_key = KEY_EVENT_FORMAT.format(DB=self._db, KEY=self._build_key(key))
+
+        def callback_wrapper(message: dict[str, bytes]) -> None:
+            event_db, event_key, event_type = self._decode_event(message)
+            if event_db is None or event_key is None:
+                return
+            int_event_db = int(event_db)
+            str_event_key = str(event_key)
+            logger.info(
+                "Redis: Handling `%s` for key `%s` (in db %d)...",
+                event_type,
+                event_key,
+                event_db,
+            )
+            key_value: str = self._key_fetch(str_event_key) or ""
+            # NOTE: Reimplement to non-blocking and parallel,
+            # instead of sequencial and blocking
+            # NOTE: pubsub requires synchronous callbacks
+            for callback in callbacks:
+                if callback is None:
+                    self._on_key_event(
+                        int_event_db, str_event_key, event_type, key_value
+                    )
+                else:
+                    callback(int_event_db, str_event_key, event_type, key_value)
+
+        self._pubsub.psubscribe(**{event_key: callback_wrapper})
+
+    def _key_versions(self, key: str) -> int | None:
+        try:
+            key = self._build_key(key)
+            key_type = self._key_type(key)
+
+            if key_type == self.TYPE_LIST:
+                return self._redis.llen(key)  # type: ignore
+            if key_type == self.TYPE_STRING:
+                return 1
+            return 0
+        except RedisConnectionError:
+            logger.error("Redis version: Failed - no connection")
+            return self._cache_verisons.get(key)
+
+    def _key_fetch(self, key: str) -> str | None:
+        try:
+            key = self._build_key(key)
+            key_type = self._key_type(key)
+            value = None
+
+            if key_type == self.TYPE_LIST:
+                value = self._redis.lindex(key, -1)
+            if key_type == self.TYPE_STRING:
+                value = self._redis.get(key)
+
+            if value is not None:
+                return value.decode()  # type: ignore
+
+            return value
+        except RedisConnectionError:
+            logger.error("Redis fetch: Failed - no connection")
+            return self._cache.get(key)
+
+    def _key_update(self, key: str, value: T) -> Any:
+        key = self._build_key(key)
+        key_type = self._key_type(key)
+        value_new = json.dumps(asdict(value))
+
+        try:
+            if value_new == json.dumps(self._key_fetch(key)):
+                logger.info("Redis update: Identical, skiping change...")
+                return value_new
+            if key_type in [self.TYPE_LIST, self.TYPE_NONE]:
+                self._redis.rpush(key, value_new)
+            elif key_type == self.TYPE_STRING:
+                value_old = self._redis.get(key)
+                self._redis.delete(key)
+                self._redis.rpush(key, value_old, value_new)  # type: ignore
+
+            return value_new
+        except RedisConnectionError:
+            logger.error("Redis update: Failed - no connection")
+            return self._cache[key]
+
+    def _key_delete(self, key: str) -> None:
+        self._redis.delete(self._build_key(key))
+
+    def _key_type(self, key: str) -> str | None:
+        try:
+            key_type = self._redis.type(key)
+            if key_type is not None:
+                return str(key_type.decode())  # type: ignore
+            return key_type
+        except RedisConnectionError:
+            return None
+
+    def _build_key(self, key: str) -> str:
+        return f"{self._prefix}{key}"
+
+    def _watcher(self) -> None:
+        # This thread seems unecessary, but without it messages aren't updated.
+        # Even if we're not receiving any 'pmessage' messages (???)
+        while True:
+            for message in self._pubsub.listen():
+                logger.info(
+                    "Redis watcher: Received event: %s - %s",
+                    message["channel"],
+                    message["data"],
+                )
+                if message["type"] == "pmessage":
+                    event_db, event_key, event_type = self._decode_event(message)
+                    if event_db is None or event_key is None:
+                        continue
+                    int_event_db = int(event_db)
+                    str_event_key = str(event_key)
+                    key_value = self._key_fetch(str_event_key) or ""
+
+                    self._on_key_event(
+                        int_event_db, str_event_key, event_type, key_value
+                    )
+            time.sleep(1)
+        logger.info("Redis watcher is shutting down...")
+
+    def _decode_event(
+        self, message: dict[str, bytes]
+    ) -> tuple[int | None, str | None, str]:
+        channel = message["channel"].decode()
+
+        match = re.match(KEY_EVENT_REGEX, channel)
+        if not match:
+            return None, None, message["data"].decode()
+
+        event_key = match.group("key")
+        event_db = int(match.group("db"))
+        event_type = message["data"].decode()
+        return event_db, event_key, event_type
+
+    def _on_key_event(self, db: int, key: str, event: str, value: str) -> None:
+        if db != self._db:
+            return
+
+        if event in [self.EVENT_SET, self.EVENT_RPUSH, self.EVENT_LSET]:
+            self._on_key_update(key, value)
+        elif event == self.EVENT_DELETE:
+            self._on_key_delete(key)
+        else:
+            logger.info(f"Redis _on_key_event: Ignoring operation: {event}")
+
+    def _on_key_update(self, key: str, value: str) -> None:
+        logger.info(f"Redis: Updating local cache: {key}")
+        self._cache[key] = value
+        self._cache_verisons[key] = self._key_versions(key) or -1
+
+    def _on_key_delete(self, key: str) -> None:
+        del self._cache[key]

cledar/redis/tests/test_async_integration_redis.py

@@ -0,0 +1,158 @@
+# mypy: disable-error-code=no-untyped-def
+import json
+from collections.abc import AsyncGenerator
+from datetime import datetime
+from enum import Enum
+
+import pytest
+import pytest_asyncio
+from pydantic import BaseModel
+from testcontainers.redis import RedisContainer
+
+from cledar.redis import AsyncRedisService, FailedValue, RedisServiceConfig
+
+
+class UserModel(BaseModel):
+    user_id: int
+    name: str
+
+
+class Color(Enum):
+    RED = 1
+    BLUE = 2
+
+
+@pytest.fixture(scope="module")
+def redis_container():
+    """Start a Redis container for testing."""
+    with RedisContainer("redis:7.2-alpine") as redis_db:
+        yield redis_db
+
+
+@pytest_asyncio.fixture(scope="function")
+async def async_redis_service(
+    redis_container: RedisContainer,
+) -> AsyncGenerator[AsyncRedisService, None]:
+    host = redis_container.get_container_host_ip()
+    port = int(redis_container.get_exposed_port(6379))
+
+    config = RedisServiceConfig(redis_host=host, redis_port=port, redis_db=0)
+    service = AsyncRedisService(config)
+    await service.connect()
+    yield service
+    await service.close()
+
+
+@pytest.mark.asyncio
+async def test_is_alive(async_redis_service: AsyncRedisService) -> None:
+    assert await async_redis_service.is_alive() is True
+
+
+@pytest.mark.asyncio
+async def test_set_and_get_pydantic_model(
+    async_redis_service: AsyncRedisService,
+) -> None:
+    key = "async:user:1"
+    model = UserModel(user_id=1, name="Alice")
+    assert await async_redis_service.set(key, model) is True
+    got = await async_redis_service.get(key, UserModel)
+    assert isinstance(got, UserModel)
+    assert got == model
+
+
+@pytest.mark.asyncio
+async def test_set_plain_string_and_get_raw(
+    async_redis_service: AsyncRedisService,
+) -> None:
+    key = "async:greeting"
+    assert await async_redis_service.set(key, "hello") is True
+    assert await async_redis_service.get_raw(key) == "hello"
+
+
+@pytest.mark.asyncio
+async def test_set_with_enum_and_datetime_uses_custom_encoder(
+    async_redis_service: AsyncRedisService,
+) -> None:
+    key = "async:meta"
+    now = datetime(2024, 1, 2, 3, 4, 5)
+    payload = {"color": Color.RED, "when": now}
+    assert await async_redis_service.set(key, payload) is True
+
+    raw = await async_redis_service.get_raw(key)
+    data = json.loads(raw)  # type: ignore
+    assert data["color"] == "red"
+    assert data["when"] == now.isoformat()
+
+
+@pytest.mark.asyncio
+async def test_list_keys(async_redis_service: AsyncRedisService) -> None:
+    prefix = "async:listkeys:test:"
+    keys = [f"{prefix}{i}" for i in range(3)]
+    for k in keys:
+        assert await async_redis_service.set(k, {"i": 1}) is True
+
+    listed = await async_redis_service.list_keys(f"{prefix}*")
+    for k in keys:
+        assert k in listed
+
+
+@pytest.mark.asyncio
+async def test_mget_mixed_results(async_redis_service: AsyncRedisService) -> None:
+    ok = UserModel(user_id=2, name="Bob")
+    k1 = "async:mget:ok"
+    k2 = "async:mget:not_json"
+    k3 = "async:mget:bad_validation"
+    k4 = "async:mget:none"
+
+    assert await async_redis_service.set(k1, ok) is True
+    assert await async_redis_service.set(k2, "{not-json}") is True
+    assert await async_redis_service.set(k3, json.dumps({"user_id": 3})) is True
+
+    results = await async_redis_service.mget([k1, k2, k3, k4], UserModel)
+
+    assert isinstance(results[0], UserModel)
+    assert isinstance(results[1], FailedValue)
+    assert isinstance(results[2], FailedValue)
+    assert results[3] is None
+
+
+@pytest.mark.asyncio
+async def test_delete(async_redis_service: AsyncRedisService) -> None:
+    key = "async:delete:test"
+    assert await async_redis_service.set(key, {"x": 1}) is True
+    assert await async_redis_service.delete(key) is True
+    assert await async_redis_service.get_raw(key) is None
+
+
+@pytest.mark.asyncio
+async def test_context_manager_pattern(redis_container: RedisContainer) -> None:
+    """Test that service can be used with proper async context management."""
+    host = redis_container.get_container_host_ip()
+    port = int(redis_container.get_exposed_port(6379))
+
+    config = RedisServiceConfig(redis_host=host, redis_port=port, redis_db=0)
+    service = AsyncRedisService(config)
+
+    try:
+        await service.connect()
+        assert await service.is_alive() is True
+        await service.set("test:key", "test:value")
+        assert await service.get_raw("test:key") == "test:value"
+    finally:
+        await service.close()
+
+
+@pytest.mark.asyncio
+async def test_concurrent_operations(async_redis_service: AsyncRedisService) -> None:
+    """Test multiple concurrent async operations."""
+    import asyncio
+
+    async def set_and_get(key: str, value: str) -> str | None:
+        await async_redis_service.set(key, value)
+        return await async_redis_service.get_raw(key)
+
+    tasks = [set_and_get(f"async:concurrent:{i}", f"value:{i}") for i in range(10)]
+    results = await asyncio.gather(*tasks)
+
+    for i, result in enumerate(results):
+        assert result == f"value:{i}"