cache-sync 0.3.1__py3-none-any.whl

cache_sync/providers/kafka/invalidation_bus.py

@@ -0,0 +1,173 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import socket
+import uuid
+from collections.abc import Sequence
+from contextlib import suppress
+from typing import Any
+
+from cache_sync.invalidation import (
+    ClearLocal,
+    InvalidationMessage,
+    RemoveLocal,
+)
+
+
+class KafkaInvalidationBus:
+    """Invalidation bus backed by a Kafka topic."""
+
+    def __init__(
+        self,
+        *,
+        bootstrap_servers: str | Sequence[str],
+        topic: str = "cache-sync-invalidations",
+        node_name: str | None = None,
+        group_id: str | None = None,
+    ) -> None:
+        """Create a Kafka invalidation bus.
+
+        By default, each node gets a unique consumer group so every node receives
+        every invalidation. Supplying the same `group_id` for multiple nodes will
+        load-balance messages and is usually wrong for cache invalidation.
+        """
+
+        self._bootstrap_servers = bootstrap_servers
+        self._topic = topic
+        self._source_id = str(uuid.uuid4())
+        self._node_name = node_name or f"{socket.gethostname()}-{self._source_id}"
+        self._group_id = group_id or f"cache-sync-node:{self._node_name}"
+        self._remove_local: RemoveLocal | None = None
+        self._clear_local: ClearLocal | None = None
+        self._producer: Any | None = None
+        self._consumer: Any | None = None
+        self._listener_task: asyncio.Task[None] | None = None
+
+    async def start(
+        self,
+        *,
+        remove_local: RemoveLocal,
+        clear_local: ClearLocal,
+    ) -> None:
+        """Start the Kafka producer, consumer, and listener task."""
+
+        if self._listener_task is not None:
+            return
+
+        try:
+            from aiokafka import AIOKafkaConsumer, AIOKafkaProducer
+        except ImportError as ex:  # pragma: no cover - exercised only without optional deps
+            msg = "Install cache-sync with the kafka dependency group to use Kafka."
+            raise RuntimeError(msg) from ex
+
+        self._remove_local = remove_local
+        self._clear_local = clear_local
+        self._producer = AIOKafkaProducer(bootstrap_servers=self._bootstrap_servers)
+        self._consumer = AIOKafkaConsumer(
+            self._topic,
+            bootstrap_servers=self._bootstrap_servers,
+            group_id=self._group_id,
+            auto_offset_reset="latest",
+        )
+        await self._producer.start()
+        await self._consumer.start()
+        self._listener_task = asyncio.create_task(self._listen())
+
+    async def stop(self) -> None:
+        """Stop the listener task and close Kafka clients."""
+
+        if self._listener_task is not None:
+            self._listener_task.cancel()
+
+            with suppress(asyncio.CancelledError):
+                await self._listener_task
+
+        if self._consumer is not None:
+            await self._consumer.stop()
+
+        if self._producer is not None:
+            await self._producer.stop()
+
+        self._listener_task = None
+        self._consumer = None
+        self._producer = None
+        self._remove_local = None
+        self._clear_local = None
+
+    async def invalidate(self, key: str) -> None:
+        """Publish a key-removal message to the Kafka topic."""
+
+        await self._publish(InvalidationMessage.remove(key))
+
+    async def clear(self) -> None:
+        """Publish a clear-all message to the Kafka topic."""
+
+        await self._publish(InvalidationMessage.clear())
+
+    async def _publish(self, message: InvalidationMessage) -> None:
+        if self._producer is None:
+            msg = "KafkaInvalidationBus must be started before publishing."
+            raise RuntimeError(msg)
+
+        await self._producer.send_and_wait(
+            self._topic,
+            self._encode_message(message),
+        )
+
+    async def _listen(self) -> None:
+        consumer = self._consumer
+        if consumer is None:
+            return
+
+        async for record in consumer:
+            self._apply_payload(record.value)
+
+    def _apply_payload(self, payload: bytes | str) -> None:
+        message = self._decode_message(payload)
+
+        if message is not None:
+            self._apply_message(message)
+
+    def _apply_message(self, message: InvalidationMessage) -> None:
+        if message.action == "remove" and message.key is not None:
+            remove_local = self._remove_local
+            if remove_local is not None:
+                remove_local(message.key)
+            return
+
+        if message.action == "clear":
+            clear_local = self._clear_local
+            if clear_local is not None:
+                clear_local()
+
+    def _encode_message(self, message: InvalidationMessage) -> bytes:
+        payload: dict[str, str] = {
+            "action": message.action,
+            "source_id": self._source_id,
+        }
+
+        if message.key is not None:
+            payload["key"] = message.key
+
+        return json.dumps(payload, separators=(",", ":")).encode("utf-8")
+
+    def _decode_message(self, payload: bytes | str) -> InvalidationMessage | None:
+        if isinstance(payload, bytes):
+            payload = payload.decode("utf-8")
+
+        try:
+            data = json.loads(payload)
+        except (TypeError, ValueError, UnicodeDecodeError):
+            return None
+
+        if not isinstance(data, dict) or data.get("source_id") == self._source_id:
+            return None
+
+        if data.get("action") == "remove" and isinstance(data.get("key"), str):
+            return InvalidationMessage.remove(data["key"])
+
+        if data.get("action") == "clear":
+            return InvalidationMessage.clear()
+
+        return None

cache_sync/providers/postgres/__init__.py

@@ -0,0 +1,7 @@
+"""PostgreSQL provider exports."""
+
+from cache_sync.providers.postgres.invalidation_bus import PostgresNotifyInvalidationBus
+
+__all__ = [
+    "PostgresNotifyInvalidationBus",
+]

cache_sync/providers/postgres/invalidation_bus.py

@@ -0,0 +1,129 @@
+from __future__ import annotations
+
+import json
+import socket
+import uuid
+from typing import Any
+
+from cache_sync.invalidation import (
+    ClearLocal,
+    InvalidationMessage,
+    RemoveLocal,
+)
+
+
+class PostgresNotifyInvalidationBus:
+    """Invalidation bus backed by PostgreSQL LISTEN/NOTIFY."""
+
+    def __init__(
+        self,
+        connection: Any,
+        *,
+        channel: str = "cache_sync_invalidations",
+        node_name: str | None = None,
+    ) -> None:
+        """Create a Postgres notification invalidation bus."""
+
+        self._connection = connection
+        self._channel = channel
+        self._source_id = str(uuid.uuid4())
+        self._node_name = node_name or f"{socket.gethostname()}-{self._source_id}"
+        self._remove_local: RemoveLocal | None = None
+        self._clear_local: ClearLocal | None = None
+        self._started = False
+
+    async def start(
+        self,
+        *,
+        remove_local: RemoveLocal,
+        clear_local: ClearLocal,
+    ) -> None:
+        """Register a notification listener on the configured channel."""
+
+        if self._started:
+            return
+
+        self._remove_local = remove_local
+        self._clear_local = clear_local
+        await self._connection.add_listener(self._channel, self._handle_notification)
+        self._started = True
+
+    async def stop(self) -> None:
+        """Remove the notification listener and clear local callbacks."""
+
+        if self._started:
+            await self._connection.remove_listener(self._channel, self._handle_notification)
+
+        self._started = False
+        self._remove_local = None
+        self._clear_local = None
+
+    async def invalidate(self, key: str) -> None:
+        """Publish a key-removal notification."""
+
+        await self._publish(InvalidationMessage.remove(key))
+
+    async def clear(self) -> None:
+        """Publish a clear-all notification."""
+
+        await self._publish(InvalidationMessage.clear())
+
+    async def _publish(self, message: InvalidationMessage) -> None:
+        await self._connection.execute(
+            "select pg_notify($1, $2)",
+            self._channel,
+            self._encode_message(message),
+        )
+
+    def _handle_notification(
+        self,
+        connection: Any,
+        pid: int,
+        channel: str,
+        payload: str,
+    ) -> None:
+        del connection, pid, channel
+        message = self._decode_message(payload)
+
+        if message is not None:
+            self._apply_message(message)
+
+    def _apply_message(self, message: InvalidationMessage) -> None:
+        if message.action == "remove" and message.key is not None:
+            remove_local = self._remove_local
+            if remove_local is not None:
+                remove_local(message.key)
+            return
+
+        if message.action == "clear":
+            clear_local = self._clear_local
+            if clear_local is not None:
+                clear_local()
+
+    def _encode_message(self, message: InvalidationMessage) -> str:
+        payload: dict[str, str] = {
+            "action": message.action,
+            "source_id": self._source_id,
+        }
+
+        if message.key is not None:
+            payload["key"] = message.key
+
+        return json.dumps(payload, separators=(",", ":"))
+
+    def _decode_message(self, payload: str) -> InvalidationMessage | None:
+        try:
+            data = json.loads(payload)
+        except (TypeError, ValueError):
+            return None
+
+        if not isinstance(data, dict) or data.get("source_id") == self._source_id:
+            return None
+
+        if data.get("action") == "remove" and isinstance(data.get("key"), str):
+            return InvalidationMessage.remove(data["key"])
+
+        if data.get("action") == "clear":
+            return InvalidationMessage.clear()
+
+        return None

cache_sync/providers/rabbitmq/__init__.py

@@ -0,0 +1,7 @@
+"""RabbitMQ provider exports."""
+
+from cache_sync.providers.rabbitmq.invalidation_bus import RabbitMQInvalidationBus
+
+__all__ = [
+    "RabbitMQInvalidationBus",
+]

cache_sync/providers/rabbitmq/invalidation_bus.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import json
+import socket
+import uuid
+from contextlib import suppress
+from typing import Any
+
+from cache_sync.invalidation import (
+    ClearLocal,
+    InvalidationMessage,
+    RemoveLocal,
+)
+
+
+class RabbitMQInvalidationBus:
+    """Invalidation bus backed by a RabbitMQ fanout exchange."""
+
+    def __init__(
+        self,
+        connection: Any,
+        *,
+        exchange_name: str = "cache-sync-invalidations",
+        node_name: str | None = None,
+    ) -> None:
+        """Create a RabbitMQ invalidation bus using an existing connection."""
+
+        self._connection = connection
+        self._exchange_name = exchange_name
+        self._source_id = str(uuid.uuid4())
+        self._node_name = node_name or f"{socket.gethostname()}-{self._source_id}"
+        self._remove_local: RemoveLocal | None = None
+        self._clear_local: ClearLocal | None = None
+        self._channel: Any | None = None
+        self._exchange: Any | None = None
+        self._queue: Any | None = None
+        self._consumer_tag: str | None = None
+
+    async def start(
+        self,
+        *,
+        remove_local: RemoveLocal,
+        clear_local: ClearLocal,
+    ) -> None:
+        """Declare the fanout exchange, bind an exclusive queue, and consume."""
+
+        if self._channel is not None:
+            return
+
+        try:
+            from aio_pika import ExchangeType
+        except ImportError as ex:  # pragma: no cover - exercised only without optional deps
+            msg = "Install cache-sync with the rabbitmq dependency group to use RabbitMQ."
+            raise RuntimeError(msg) from ex
+
+        self._remove_local = remove_local
+        self._clear_local = clear_local
+        self._channel = await self._connection.channel()
+        self._exchange = await self._channel.declare_exchange(
+            self._exchange_name,
+            ExchangeType.FANOUT,
+        )
+        self._queue = await self._channel.declare_queue(
+            exclusive=True,
+            auto_delete=True,
+        )
+        await self._queue.bind(self._exchange)
+        self._consumer_tag = await self._queue.consume(self._handle_incoming_message)
+
+    async def stop(self) -> None:
+        """Cancel consumption and close the created channel."""
+
+        if self._queue is not None and self._consumer_tag is not None:
+            with suppress(Exception):
+                await self._queue.cancel(self._consumer_tag)
+
+        if self._channel is not None:
+            with suppress(Exception):
+                await self._channel.close()
+
+        self._channel = None
+        self._exchange = None
+        self._queue = None
+        self._consumer_tag = None
+        self._remove_local = None
+        self._clear_local = None
+
+    async def invalidate(self, key: str) -> None:
+        """Publish a key-removal message to the fanout exchange."""
+
+        await self._publish(InvalidationMessage.remove(key))
+
+    async def clear(self) -> None:
+        """Publish a clear-all message to the fanout exchange."""
+
+        await self._publish(InvalidationMessage.clear())
+
+    async def _publish(self, message: InvalidationMessage) -> None:
+        if self._exchange is None:
+            msg = "RabbitMQInvalidationBus must be started before publishing."
+            raise RuntimeError(msg)
+
+        try:
+            from aio_pika import Message
+        except ImportError as ex:  # pragma: no cover - exercised only without optional deps
+            msg = "Install cache-sync with the rabbitmq dependency group to use RabbitMQ."
+            raise RuntimeError(msg) from ex
+
+        await self._exchange.publish(
+            Message(
+                body=self._encode_message(message),
+                content_type="application/json",
+            ),
+            routing_key="",
+        )
+
+    async def _handle_incoming_message(self, incoming_message: Any) -> None:
+        async with incoming_message.process():
+            self._apply_payload(incoming_message.body)
+
+    def _apply_payload(self, payload: bytes | str) -> None:
+        message = self._decode_message(payload)
+
+        if message is not None:
+            self._apply_message(message)
+
+    def _apply_message(self, message: InvalidationMessage) -> None:
+        if message.action == "remove" and message.key is not None:
+            remove_local = self._remove_local
+            if remove_local is not None:
+                remove_local(message.key)
+            return
+
+        if message.action == "clear":
+            clear_local = self._clear_local
+            if clear_local is not None:
+                clear_local()
+
+    def _encode_message(self, message: InvalidationMessage) -> bytes:
+        payload: dict[str, str] = {
+            "action": message.action,
+            "source_id": self._source_id,
+        }
+
+        if message.key is not None:
+            payload["key"] = message.key
+
+        return json.dumps(payload, separators=(",", ":")).encode("utf-8")
+
+    def _decode_message(self, payload: bytes | str) -> InvalidationMessage | None:
+        if isinstance(payload, bytes):
+            payload = payload.decode("utf-8")
+
+        try:
+            data = json.loads(payload)
+        except (TypeError, ValueError, UnicodeDecodeError):
+            return None
+
+        if not isinstance(data, dict) or data.get("source_id") == self._source_id:
+            return None
+
+        if data.get("action") == "remove" and isinstance(data.get("key"), str):
+            return InvalidationMessage.remove(data["key"])
+
+        if data.get("action") == "clear":
+            return InvalidationMessage.clear()
+
+        return None

cache_sync/providers/redis/__init__.py

@@ -0,0 +1,9 @@
+"""Redis provider exports."""
+
+from cache_sync.providers.redis.cache import RedisDistributedCache
+from cache_sync.providers.redis.invalidation_bus import RedisStreamsInvalidationBus
+
+__all__ = [
+    "RedisDistributedCache",
+    "RedisStreamsInvalidationBus",
+]

cache_sync/providers/redis/cache.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+from redis.asyncio import Redis
+
+from cache_sync.serializers import PickleSerializer, Serializer
+
+
+class RedisDistributedCache:
+    """Distributed cache implementation backed by Redis string keys."""
+
+    def __init__(
+        self,
+        redis: Redis,
+        *,
+        prefix: str = "cache-sync:",
+        serializer: Serializer | None = None,
+    ) -> None:
+        """Create a Redis distributed cache with an optional key prefix."""
+
+        self._redis = redis
+        self._prefix = prefix
+        self._serializer = serializer or PickleSerializer()
+
+    async def get(self, key: str) -> object | None:
+        """Return a deserialized value or `None` when the key is missing."""
+
+        value = await self._redis.get(self._key(key))
+
+        if value is None:
+            return None
+
+        if isinstance(value, str):
+            value = value.encode("utf-8")
+
+        return self._serializer.loads(value)
+
+    async def set(self, key: str, value: object, ttl_seconds: float) -> None:
+        """Serialize and store a value with a Redis expiration."""
+
+        await self._redis.set(
+            self._key(key),
+            self._serializer.dumps(value),
+            ex=max(1, int(ttl_seconds)),
+        )
+
+    async def delete(self, key: str) -> None:
+        """Delete a key from Redis."""
+
+        await self._redis.delete(self._key(key))
+
+    def _key(self, key: str) -> str:
+        return f"{self._prefix}{key}"