redqueue 0.10.0__py3-none-any.whl

redqueue/backends/stream.py

@@ -0,0 +1,370 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Synchronous Redis Streams backend."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from redqueue.backends.base import BaseMessageBackend
+from redqueue.compat import RedisCapabilities, RedisVersion
+from redqueue.config import QueueConfig
+from redqueue.exceptions import AckError, BackendUnavailableError, RetryExceededError
+from redqueue.message import Message, new_message_id
+from redqueue.monitoring import MonitoringEventType
+
+
+class SyncStreamRedis(Protocol):
+    """Redis command subset required by the synchronous Streams backend."""
+
+    def xadd(self, name: str, fields: dict[str, bytes | str], id: str = "*") -> str: ...
+
+    def xgroup_create(
+        self,
+        name: str,
+        groupname: str,
+        id: str = "0",
+        mkstream: bool = True,
+    ) -> bool: ...
+
+    def xreadgroup(
+        self,
+        groupname: str,
+        consumername: str,
+        streams: dict[str, str],
+        count: int | None = None,
+        block: int | None = None,
+    ) -> list[Any]: ...
+
+    def xack(self, name: str, groupname: str, *ids: str) -> int: ...
+
+    def xautoclaim(
+        self,
+        name: str,
+        groupname: str,
+        consumername: str,
+        min_idle_time: int,
+        start_id: str,
+        count: int | None = None,
+    ) -> Any: ...
+
+    def xpending_range(
+        self,
+        name: str,
+        groupname: str,
+        min: str,
+        max: str,
+        count: int,
+    ) -> list[Any]: ...
+
+    def xclaim(
+        self,
+        name: str,
+        groupname: str,
+        consumername: str,
+        min_idle_time: int,
+        message_ids: list[str],
+    ) -> list[Any]: ...
+
+
+class StreamBackend(BaseMessageBackend):
+    """Reliable queue backend implemented with Redis Streams."""
+
+    backend_name = "stream"
+
+    def __init__(
+        self,
+        redis: SyncStreamRedis,
+        config: QueueConfig,
+        capabilities: RedisCapabilities,
+    ) -> None:
+        capabilities.require_streams()
+        super().__init__(config)
+        self.redis = redis
+        self.capabilities = capabilities
+        self._ensure_group()
+
+    @classmethod
+    def for_modern_redis(
+        cls,
+        redis: SyncStreamRedis,
+        config: QueueConfig,
+    ) -> StreamBackend:
+        return cls(redis, config, RedisCapabilities(RedisVersion(7, 0, 0)))
+
+    @property
+    def stream_key(self) -> str:
+        return self.config.key("stream")
+
+    @property
+    def dead_key(self) -> str:
+        return self.config.key("dead")
+
+    def publish(
+        self,
+        payload: Any,
+        *,
+        headers: dict[str, Any] | None = None,
+        message_id: str | None = None,
+    ) -> str:
+        message = Message(
+            id=message_id or new_message_id(),
+            queue=self.config.queue,
+            payload=payload,
+            headers=headers or {},
+            backend=self.backend_name,
+        )
+        self._publish_message(message)
+        return message.id
+
+    def consume(
+        self,
+        *,
+        timeout: float | None = None,
+        batch_size: int = 1,
+    ) -> Message | list[Message] | None:
+        block = int(timeout * 1000) if timeout is not None else None
+        response = self._execute(
+            "redis.xreadgroup",
+            self.redis.xreadgroup,
+            self.config.consumer_group,
+            self._consumer_name(),
+            {self.stream_key: ">"},
+            batch_size,
+            block,
+        )
+        messages = self._parse_read_response(response)
+        if batch_size <= 1:
+            if not messages:
+                return None
+            self._emit(MonitoringEventType.MESSAGE_CONSUMED, messages[0])
+            return messages[0]
+        for message in messages:
+            self._emit(MonitoringEventType.MESSAGE_CONSUMED, message)
+        return messages
+
+    def ack(self, message: Message) -> None:
+        if not message.raw_id:
+            raise AckError(
+                "stream message is missing raw Redis stream id",
+                action="message.ack",
+                queue=self.config.queue,
+                details={"message_id": message.id},
+            )
+        removed = self._execute(
+            "redis.xack",
+            self.redis.xack,
+            self.stream_key,
+            self.config.consumer_group,
+            message.raw_id,
+        )
+        if removed < 1:
+            raise AckError(
+                "stream message was not acknowledged",
+                action="message.ack",
+                queue=self.config.queue,
+                details={"message_id": message.id, "raw_id": message.raw_id},
+            )
+        self._emit(MonitoringEventType.MESSAGE_ACKED, message)
+
+    def nack(self, message: Message, *, requeue: bool = True) -> None:
+        if requeue:
+            self.publish(
+                message.payload,
+                headers=message.headers,
+                message_id=message.id,
+            )
+            self.ack(message)
+        else:
+            self._move_to_dead(message)
+        self._emit(
+            MonitoringEventType.MESSAGE_NACKED,
+            message,
+            attributes={"requeue": requeue},
+        )
+
+    def retry(
+        self,
+        message: Message,
+        *,
+        delay: float | None = None,
+        reason: str | None = None,
+    ) -> None:
+        if message.attempts >= self.config.retry.max_retries:
+            self._move_to_dead(message)
+            self._emit(
+                MonitoringEventType.MESSAGE_DEAD_LETTERED,
+                message,
+                attributes={"reason": reason, "attempts": message.attempts},
+            )
+            raise RetryExceededError(
+                "stream message exceeded max retries and was moved to dead letter",
+                action="message.retry",
+                queue=self.config.queue,
+                details={
+                    "message_id": message.id,
+                    "attempts": message.attempts,
+                    "max_retries": self.config.retry.max_retries,
+                },
+            )
+        retried = message.with_attempt()
+        self._publish_message(retried)
+        self.ack(message)
+        self._emit(
+            MonitoringEventType.MESSAGE_RETRIED,
+            retried,
+            attributes={"delay": delay, "reason": reason},
+        )
+
+    def recover_pending(self, *, min_idle_ms: int, limit: int = 100) -> list[Message]:
+        if not self.capabilities.supports_streams_auto_claim:
+            pending = self._execute(
+                "redis.xpending_range",
+                self.redis.xpending_range,
+                self.stream_key,
+                self.config.consumer_group,
+                "-",
+                "+",
+                limit,
+            )
+            message_ids = [self._pending_id(item) for item in pending]
+            if not message_ids:
+                return []
+            claimed = self._execute(
+                "redis.xclaim",
+                self.redis.xclaim,
+                self.stream_key,
+                self.config.consumer_group,
+                self._consumer_name(),
+                min_idle_ms,
+                message_ids,
+            )
+            return [
+                self._decode_stream_entry(str(raw_id), fields)
+                for raw_id, fields in claimed
+            ]
+        response = self._execute(
+            "redis.xautoclaim",
+            self.redis.xautoclaim,
+            self.stream_key,
+            self.config.consumer_group,
+            self._consumer_name(),
+            min_idle_ms,
+            "0-0",
+            limit,
+        )
+        return self._parse_autoclaim_response(response)
+
+    def dead_letters(self, *, limit: int = 100) -> list[Message]:
+        response = self._execute(
+            "redis.xreadgroup",
+            self.redis.xreadgroup,
+            self.config.consumer_group,
+            self._consumer_name(),
+            {self.dead_key: ">"},
+            limit,
+            None,
+        )
+        return self._parse_read_response(response)
+
+    def requeue_dead(self, message: Message) -> None:
+        self.publish(message.payload, headers=message.headers, message_id=message.id)
+        if message.raw_id:
+            self._execute(
+                "redis.xack",
+                self.redis.xack,
+                self.dead_key,
+                self.config.consumer_group,
+                message.raw_id,
+            )
+
+    def _publish_message(self, message: Message) -> str:
+        raw_id = self._execute(
+            "redis.xadd",
+            self.redis.xadd,
+            self.stream_key,
+            {"payload": self._encode(message)},
+        )
+        published = message.with_backend(self.backend_name, raw_id=str(raw_id))
+        self._emit(
+            MonitoringEventType.MESSAGE_PUBLISHED,
+            published,
+            attributes={"key": self.stream_key},
+        )
+        return str(raw_id)
+
+    def _ensure_group(self) -> None:
+        try:
+            self.redis.xgroup_create(
+                self.stream_key,
+                self.config.consumer_group,
+                id="0",
+                mkstream=True,
+            )
+        except Exception as exc:
+            if "BUSYGROUP" in str(exc):
+                return
+            self._emit_backend_error("redis.xgroup_create", str(exc))
+            raise BackendUnavailableError(
+                "Redis Streams group initialization failed",
+                action="redis.xgroup_create",
+                queue=self.config.queue,
+            ) from exc
+
+    def _move_to_dead(self, message: Message) -> None:
+        self._execute(
+            "redis.xadd",
+            self.redis.xadd,
+            self.dead_key,
+            {"payload": self._encode(message)},
+        )
+        self.ack(message)
+
+    def _parse_read_response(self, response: list[Any]) -> list[Message]:
+        messages: list[Message] = []
+        for _stream, entries in response or []:
+            for raw_id, fields in entries:
+                messages.append(self._decode_stream_entry(str(raw_id), fields))
+        return messages
+
+    def _parse_autoclaim_response(self, response: Any) -> list[Message]:
+        if not response:
+            return []
+        entries = response[1] if len(response) > 1 else []
+        return [
+            self._decode_stream_entry(str(raw_id), fields)
+            for raw_id, fields in entries
+        ]
+
+    def _decode_stream_entry(self, raw_id: str, fields: dict[Any, Any]) -> Message:
+        payload = fields.get("payload") or fields.get(b"payload")
+        if payload is None:
+            raise BackendUnavailableError(
+                "stream entry is missing payload field",
+                action="message.decode",
+                queue=self.config.queue,
+                details={"raw_id": raw_id},
+            )
+        return self._decode(payload).with_backend(self.backend_name, raw_id=raw_id)
+
+    def _consumer_name(self) -> str:
+        return self.config.consumer_name or "redqueue-consumer"
+
+    @staticmethod
+    def _pending_id(item: Any) -> str:
+        if isinstance(item, dict):
+            value = item.get("message_id") or item.get("message-id") or item.get("id")
+            return str(value)
+        return str(item[0])
+
+    def _execute(self, action: str, func: Any, *args: Any) -> Any:
+        try:
+            return func(*args)
+        except Exception as exc:
+            self._emit_backend_error(action, str(exc))
+            raise BackendUnavailableError(
+                "Redis Streams backend command failed",
+                action=action,
+                queue=self.config.queue,
+            ) from exc

redqueue/client.py

@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Synchronous RedQueue client."""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+from redis import Redis
+
+from redqueue.backends import DelayBackend, ListBackend, StreamBackend
+from redqueue.compat import RedisCapabilities, RedisInfoClient, detect_capabilities
+from redqueue.config import BackendType, QueueConfig
+from redqueue.message import Message
+from redqueue.monitoring import MonitoringEvent, MonitoringEventType
+
+
+class QueueClient:
+    """Synchronous queue client."""
+
+    def __init__(
+        self,
+        config: QueueConfig,
+        *,
+        redis: Any | None = None,
+        capabilities: RedisCapabilities | None = None,
+    ) -> None:
+        self.config = config
+        self.redis = redis
+        self.capabilities = capabilities
+        self.backend = self._create_backend()
+        self.delay_backend = self._create_delay_backend()
+        self.config.monitoring.emit(
+            MonitoringEvent(
+                type=MonitoringEventType.CLIENT_CREATED,
+                queue=config.queue,
+                backend=config.backend_type.value,
+            )
+        )
+
+    @classmethod
+    def from_url(
+        cls,
+        url: str,
+        *,
+        queue: str,
+        backend: str | BackendType = BackendType.LIST,
+        **options: Any,
+    ) -> QueueClient:
+        redis = options.pop("redis", None) or Redis.from_url(url)
+        capabilities = options.pop("capabilities", None) or detect_capabilities(
+            cast(RedisInfoClient, redis)
+        )
+        config = QueueConfig(queue=queue, backend=backend, **options)
+        return cls(config=config, redis=redis, capabilities=capabilities)
+
+    def publish(
+        self,
+        payload: Any,
+        *,
+        delay: float | None = None,
+        headers: dict[str, Any] | None = None,
+        message_id: str | None = None,
+    ) -> str:
+        if delay is not None:
+            return self.delay(payload, delay_seconds=delay, headers=headers)
+        return self.backend.publish(
+            payload,
+            headers=headers,
+            message_id=message_id,
+        )
+
+    def consume(
+        self,
+        *,
+        timeout: float | None = None,
+        batch_size: int = 1,
+    ) -> Message | list[Message] | None:
+        return self.backend.consume(timeout=timeout, batch_size=batch_size)
+
+    def ack(self, message: Message) -> None:
+        self.backend.ack(message)
+
+    def nack(self, message: Message, *, requeue: bool = True) -> None:
+        self.backend.nack(message, requeue=requeue)
+
+    def retry(
+        self,
+        message: Message,
+        *,
+        delay: float | None = None,
+        reason: str | None = None,
+    ) -> None:
+        self.backend.retry(message, delay=delay, reason=reason)
+
+    def delay(
+        self,
+        payload: Any,
+        *,
+        delay_seconds: float | None = None,
+        run_at: float | None = None,
+        headers: dict[str, Any] | None = None,
+    ) -> str:
+        message_id = self.delay_backend.delay(
+            payload,
+            delay_seconds=delay_seconds,
+            run_at=run_at,
+            headers=headers,
+        )
+        self.config.monitoring.emit(
+            MonitoringEvent(
+                type=MonitoringEventType.DELAY_SCHEDULED,
+                queue=self.config.queue,
+                message_id=message_id,
+                backend=self.config.backend_type.value,
+                attributes={"delay_seconds": delay_seconds, "run_at": run_at},
+            )
+        )
+        return message_id
+
+    def schedule_due(self, *, limit: int = 100, now: float | None = None) -> int:
+        return self.delay_backend.schedule_due(limit=limit, now=now)
+
+    def recover_stale(self, *, min_idle_ms: int | None = None, limit: int = 100) -> int:
+        if isinstance(self.backend, StreamBackend):
+            idle = min_idle_ms or int(self.config.visibility_timeout_seconds * 1000)
+            return len(self.backend.recover_pending(min_idle_ms=idle, limit=limit))
+        return self.backend.recover_stale(limit=limit)
+
+    def dead_letters(self, *, limit: int = 100) -> list[Message]:
+        return self.backend.dead_letters(limit=limit)
+
+    def requeue_dead(self, message: Message) -> None:
+        self.backend.requeue_dead(message)
+
+    def close(self) -> None:
+        close = getattr(self.redis, "close", None)
+        if close is not None:
+            close()
+
+    def _create_backend(self) -> ListBackend | StreamBackend:
+        if self.config.backend_type is BackendType.LIST:
+            if self.redis is None:
+                raise TypeError("redis client is required for List backend")
+            capabilities = self.capabilities or detect_capabilities(
+                cast(RedisInfoClient, self.redis)
+            )
+            return ListBackend(self.redis, self.config, capabilities)
+        if self.config.backend_type is BackendType.STREAM:
+            if self.redis is None:
+                raise TypeError("redis client is required for Streams backend")
+            capabilities = self.capabilities or detect_capabilities(
+                cast(RedisInfoClient, self.redis)
+            )
+            return StreamBackend(self.redis, self.config, capabilities)
+        raise NotImplementedError(
+            f"backend {self.config.backend_type.value!r} is not implemented"
+        )
+
+    def _create_delay_backend(self) -> DelayBackend:
+        if self.redis is None:
+            raise TypeError("redis client is required for delayed tasks")
+        capabilities = self.capabilities or detect_capabilities(
+            cast(RedisInfoClient, self.redis)
+        )
+        capabilities.require_delay_sorted_set()
+        return DelayBackend(self.redis, self.config, self.backend)