redqueue 0.10.0__py3-none-any.whl

redqueue/backends/base.py

@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Shared backend helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from redqueue.config import QueueConfig
+from redqueue.exceptions import BackendUnavailableError
+from redqueue.message import Message
+from redqueue.monitoring import MonitoringEvent, MonitoringEventType
+
+
+class BaseMessageBackend:
+    """Shared message envelope and monitoring behavior."""
+
+    backend_name = "backend"
+
+    def __init__(self, config: QueueConfig) -> None:
+        self.config = config
+
+    def _encode(self, message: Message) -> bytes:
+        envelope = {
+            "id": message.id,
+            "queue": message.queue,
+            "payload": message.payload,
+            "headers": message.headers,
+            "attempts": message.attempts,
+            "created_at": message.created_at,
+            "available_at": message.available_at,
+            "backend": self.backend_name,
+            "raw_id": message.raw_id,
+        }
+        return self.config.serializer.encode(envelope, queue=self.config.queue)
+
+    def _decode(self, payload: bytes) -> Message:
+        envelope = self.config.serializer.decode(payload, queue=self.config.queue)
+        if not isinstance(envelope, dict):
+            raise BackendUnavailableError(
+                "decoded message envelope must be a mapping",
+                action="message.decode",
+                queue=self.config.queue,
+                details={"payload_type": type(envelope).__name__},
+            )
+        return Message(
+            id=str(envelope["id"]),
+            queue=str(envelope["queue"]),
+            payload=envelope["payload"],
+            headers=dict(envelope.get("headers") or {}),
+            attempts=int(envelope.get("attempts") or 0),
+            created_at=float(envelope["created_at"]),
+            available_at=envelope.get("available_at"),
+            backend=self.backend_name,
+            raw_id=envelope.get("raw_id"),
+        )
+
+    def _emit(
+        self,
+        event_type: MonitoringEventType,
+        message: Message,
+        *,
+        attributes: dict[str, Any] | None = None,
+    ) -> None:
+        self.config.monitoring.emit(
+            MonitoringEvent(
+                type=event_type,
+                queue=self.config.queue,
+                message_id=message.id,
+                backend=self.backend_name,
+                attributes=attributes or {},
+            )
+        )
+
+    def _emit_backend_error(self, action: str, error: str | None = None) -> None:
+        self.config.monitoring.emit(
+            MonitoringEvent(
+                type=MonitoringEventType.BACKEND_ERROR,
+                queue=self.config.queue,
+                backend=self.backend_name,
+                error=error,
+                attributes={"action": action},
+            )
+        )
+
+
+class BaseListBackend(BaseMessageBackend):
+    """Shared Redis List backend behavior."""
+
+    backend_name = "list"
+
+    @property
+    def ready_key(self) -> str:
+        return self.config.key("ready")
+
+    @property
+    def processing_key(self) -> str:
+        return self.config.key("processing")
+
+    @property
+    def dead_key(self) -> str:
+        return self.config.key("dead")

redqueue/backends/delay.py

@@ -0,0 +1,243 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Synchronous delayed task backend based on Redis Sorted Set."""
+
+from __future__ import annotations
+
+from time import time
+from typing import Any, Protocol
+
+from redqueue.config import QueueConfig
+from redqueue.exceptions import BackendUnavailableError, QueueConfigError
+from redqueue.message import Message, new_message_id
+from redqueue.monitoring import MonitoringEvent, MonitoringEventType
+
+
+class SyncDelayRedis(Protocol):
+    """Redis command subset required by the delayed task backend."""
+
+    def set(self, name: str, value: bytes) -> bool: ...
+
+    def get(self, name: str) -> bytes | None: ...
+
+    def delete(self, *names: str) -> int: ...
+
+    def zadd(self, name: str, mapping: dict[str, float]) -> int: ...
+
+    def zrangebyscore(
+        self,
+        name: str,
+        min: float | str,
+        max: float | str,
+        start: int | None = None,
+        num: int | None = None,
+    ) -> list[str | bytes]: ...
+
+    def zrem(self, name: str, *values: str) -> int: ...
+
+
+class DelayBackend:
+    """Delayed task scheduler implemented with Redis Sorted Set."""
+
+    backend_name = "delay"
+
+    def __init__(
+        self,
+        redis: SyncDelayRedis,
+        config: QueueConfig,
+        publisher: Any,
+    ) -> None:
+        self.redis = redis
+        self.config = config
+        self.publisher = publisher
+
+    @property
+    def delayed_key(self) -> str:
+        return self.config.key("delayed")
+
+    def payload_key(self, message_id: str) -> str:
+        return self.config.key(f"payload:{message_id}")
+
+    def delay(
+        self,
+        payload: Any,
+        *,
+        delay_seconds: float | None = None,
+        run_at: float | None = None,
+        headers: dict[str, Any] | None = None,
+        message_id: str | None = None,
+    ) -> str:
+        available_at = self._available_at(delay_seconds=delay_seconds, run_at=run_at)
+        message = Message(
+            id=message_id or new_message_id(),
+            queue=self.config.queue,
+            payload=payload,
+            headers=headers or {},
+            available_at=available_at,
+            backend=self.backend_name,
+        )
+        encoded = self._encode(message)
+        self._execute(
+            "redis.set",
+            self.redis.set,
+            self.payload_key(message.id),
+            encoded,
+        )
+        self._execute(
+            "redis.zadd",
+            self.redis.zadd,
+            self.delayed_key,
+            {message.id: available_at},
+        )
+        self._emit(MonitoringEventType.DELAY_SCHEDULED, message)
+        return message.id
+
+    def schedule_due(self, *, limit: int = 100, now: float | None = None) -> int:
+        now_value = time() if now is None else now
+        due_ids = self._execute(
+            "redis.zrangebyscore",
+            self.redis.zrangebyscore,
+            self.delayed_key,
+            "-inf",
+            now_value,
+            0,
+            limit,
+        )
+        released = 0
+        for raw_message_id in due_ids:
+            message_id = self._to_text(raw_message_id)
+            removed = self._execute(
+                "redis.zrem",
+                self.redis.zrem,
+                self.delayed_key,
+                message_id,
+            )
+            if removed < 1:
+                continue
+            message = self._load_message(message_id)
+            try:
+                self.publisher.publish(
+                    message.payload,
+                    headers=message.headers,
+                    message_id=message.id,
+                )
+            except Exception:
+                self._execute(
+                    "redis.zadd",
+                    self.redis.zadd,
+                    self.delayed_key,
+                    {message_id: message.available_at or now_value},
+                )
+                raise
+            self._execute(
+                "redis.delete",
+                self.redis.delete,
+                self.payload_key(message_id),
+            )
+            self._emit(MonitoringEventType.DELAY_RELEASED, message)
+            released += 1
+        return released
+
+    def _load_message(self, message_id: str) -> Message:
+        payload = self._execute(
+            "redis.get",
+            self.redis.get,
+            self.payload_key(message_id),
+        )
+        if payload is None:
+            raise BackendUnavailableError(
+                "delayed payload is missing",
+                action="delay.load",
+                queue=self.config.queue,
+                details={"message_id": message_id},
+            )
+        return self._decode(payload)
+
+    def _available_at(
+        self,
+        *,
+        delay_seconds: float | None,
+        run_at: float | None,
+    ) -> float:
+        if delay_seconds is not None and run_at is not None:
+            raise QueueConfigError("delay_seconds and run_at cannot both be set")
+        if delay_seconds is not None:
+            if delay_seconds < 0:
+                raise QueueConfigError(
+                    "delay_seconds must be greater than or equal to 0"
+                )
+            return time() + delay_seconds
+        if run_at is not None:
+            if run_at < 0:
+                raise QueueConfigError("run_at must be greater than or equal to 0")
+            return run_at
+        return time()
+
+    def _encode(self, message: Message) -> bytes:
+        envelope = {
+            "id": message.id,
+            "queue": message.queue,
+            "payload": message.payload,
+            "headers": message.headers,
+            "attempts": message.attempts,
+            "created_at": message.created_at,
+            "available_at": message.available_at,
+            "backend": self.backend_name,
+            "raw_id": message.raw_id,
+        }
+        return self.config.serializer.encode(envelope, queue=self.config.queue)
+
+    def _decode(self, payload: bytes) -> Message:
+        envelope = self.config.serializer.decode(payload, queue=self.config.queue)
+        if not isinstance(envelope, dict):
+            raise BackendUnavailableError(
+                "decoded delayed message envelope must be a mapping",
+                action="delay.decode",
+                queue=self.config.queue,
+                details={"payload_type": type(envelope).__name__},
+            )
+        return Message(
+            id=str(envelope["id"]),
+            queue=str(envelope["queue"]),
+            payload=envelope["payload"],
+            headers=dict(envelope.get("headers") or {}),
+            attempts=int(envelope.get("attempts") or 0),
+            created_at=float(envelope["created_at"]),
+            available_at=envelope.get("available_at"),
+            backend=self.backend_name,
+            raw_id=envelope.get("raw_id"),
+        )
+
+    def _execute(self, action: str, func: Any, *args: Any) -> Any:
+        try:
+            return func(*args)
+        except Exception as exc:
+            self.config.monitoring.emit(
+                MonitoringEvent(
+                    type=MonitoringEventType.BACKEND_ERROR,
+                    queue=self.config.queue,
+                    backend=self.backend_name,
+                    error=str(exc),
+                    attributes={"action": action},
+                )
+            )
+            raise BackendUnavailableError(
+                "Redis delayed task backend command failed",
+                action=action,
+                queue=self.config.queue,
+            ) from exc
+
+    def _emit(self, event_type: MonitoringEventType, message: Message) -> None:
+        self.config.monitoring.emit(
+            MonitoringEvent(
+                type=event_type,
+                queue=self.config.queue,
+                message_id=message.id,
+                backend=self.backend_name,
+            )
+        )
+
+    @staticmethod
+    def _to_text(value: str | bytes) -> str:
+        return value.decode() if isinstance(value, bytes) else value

redqueue/backends/list.py

@@ -0,0 +1,303 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Synchronous Redis List backend."""
+
+from __future__ import annotations
+
+from typing import Any, Protocol
+
+from redqueue.backends.base import BaseListBackend
+from redqueue.compat import RedisCapabilities, RedisVersion
+from redqueue.config import QueueConfig
+from redqueue.exceptions import AckError, BackendUnavailableError, RetryExceededError
+from redqueue.message import Message, new_message_id
+from redqueue.monitoring import MonitoringEventType
+
+
+class SyncListRedis(Protocol):
+    """Redis command subset required by the synchronous List backend."""
+
+    def lpush(self, name: str, *values: bytes) -> int: ...
+
+    def lrem(self, name: str, count: int, value: bytes) -> int: ...
+
+    def lrange(self, name: str, start: int, end: int) -> list[bytes]: ...
+
+    def blmove(
+        self,
+        first_list: str,
+        second_list: str,
+        timeout: float,
+        src: str = "RIGHT",
+        dest: str = "LEFT",
+    ) -> bytes | None: ...
+
+    def brpoplpush(
+        self,
+        src: str,
+        dst: str,
+        timeout: float,
+    ) -> bytes | None: ...
+
+
+class ListBackend(BaseListBackend):
+    """Reliable queue backend implemented with Redis List commands."""
+
+    backend_name = "list"
+
+    def __init__(
+        self,
+        redis: SyncListRedis,
+        config: QueueConfig,
+        capabilities: RedisCapabilities,
+    ) -> None:
+        capabilities.require_list_reliable()
+        super().__init__(config)
+        self.redis = redis
+        self.capabilities = capabilities
+
+    @classmethod
+    def for_modern_redis(
+        cls,
+        redis: SyncListRedis,
+        config: QueueConfig,
+    ) -> ListBackend:
+        """Create a backend for tests or callers that already know Redis is modern."""
+
+        return cls(redis, config, RedisCapabilities(RedisVersion(7, 0, 0)))
+
+    def publish(
+        self,
+        payload: Any,
+        *,
+        headers: dict[str, Any] | None = None,
+        message_id: str | None = None,
+    ) -> str:
+        message = Message(
+            id=message_id or new_message_id(),
+            queue=self.config.queue,
+            payload=payload,
+            headers=headers or {},
+            backend=self.backend_name,
+        )
+        encoded = self._encode(message)
+        self._execute("redis.lpush", self.redis.lpush, self.ready_key, encoded)
+        self._emit(
+            MonitoringEventType.MESSAGE_PUBLISHED,
+            message,
+            attributes={"key": self.ready_key},
+        )
+        return message.id
+
+    def consume(
+        self,
+        *,
+        timeout: float | None = None,
+        batch_size: int = 1,
+    ) -> Message | list[Message] | None:
+        timeout_value = timeout or 0
+        if batch_size <= 1:
+            payload = self._move_to_processing(timeout_value)
+            if payload is None:
+                return None
+            message = self._decode(payload)
+            self._emit(MonitoringEventType.MESSAGE_CONSUMED, message)
+            return message
+
+        messages: list[Message] = []
+        for _ in range(batch_size):
+            payload = self._move_to_processing(timeout_value)
+            if payload is None:
+                break
+            message = self._decode(payload)
+            self._emit(MonitoringEventType.MESSAGE_CONSUMED, message)
+            messages.append(message)
+        return messages
+
+    def ack(self, message: Message) -> None:
+        encoded = self._encode(message)
+        removed = self._execute(
+            "redis.lrem",
+            self.redis.lrem,
+            self.processing_key,
+            1,
+            encoded,
+        )
+        if removed < 1:
+            raise AckError(
+                "message was not found in processing queue",
+                action="message.ack",
+                queue=self.config.queue,
+                details={"message_id": message.id, "key": self.processing_key},
+            )
+        self._emit(MonitoringEventType.MESSAGE_ACKED, message)
+
+    def nack(self, message: Message, *, requeue: bool = True) -> None:
+        encoded = self._encode(message)
+        removed = self._execute(
+            "redis.lrem",
+            self.redis.lrem,
+            self.processing_key,
+            1,
+            encoded,
+        )
+        if removed < 1:
+            raise AckError(
+                "message was not found in processing queue",
+                action="message.nack",
+                queue=self.config.queue,
+                details={"message_id": message.id, "key": self.processing_key},
+            )
+        target_key = self.ready_key if requeue else self.dead_key
+        self._execute("redis.lpush", self.redis.lpush, target_key, encoded)
+        self._emit(
+            MonitoringEventType.MESSAGE_NACKED,
+            message,
+            attributes={"requeue": requeue, "target_key": target_key},
+        )
+
+    def retry(
+        self,
+        message: Message,
+        *,
+        delay: float | None = None,
+        reason: str | None = None,
+    ) -> None:
+        if message.attempts >= self.config.retry.max_retries:
+            self.nack(message, requeue=False)
+            self._emit(
+                MonitoringEventType.MESSAGE_DEAD_LETTERED,
+                message,
+                attributes={"reason": reason, "attempts": message.attempts},
+            )
+            raise RetryExceededError(
+                "message exceeded max retries and was moved to dead letter queue",
+                action="message.retry",
+                queue=self.config.queue,
+                details={
+                    "message_id": message.id,
+                    "attempts": message.attempts,
+                    "max_retries": self.config.retry.max_retries,
+                },
+            )
+
+        retried = message.with_attempt()
+        old_encoded = self._encode(message)
+        removed = self._execute(
+            "redis.lrem",
+            self.redis.lrem,
+            self.processing_key,
+            1,
+            old_encoded,
+        )
+        if removed < 1:
+            raise AckError(
+                "message was not found in processing queue",
+                action="message.retry",
+                queue=self.config.queue,
+                details={"message_id": message.id, "key": self.processing_key},
+            )
+        self._execute(
+            "redis.lpush",
+            self.redis.lpush,
+            self.ready_key,
+            self._encode(retried),
+        )
+        self._emit(
+            MonitoringEventType.MESSAGE_RETRIED,
+            retried,
+            attributes={"delay": delay, "reason": reason},
+        )
+
+    def recover_stale(self, *, limit: int = 100) -> int:
+        recovered = 0
+        entries = self._execute(
+            "redis.lrange",
+            self.redis.lrange,
+            self.processing_key,
+            0,
+            max(limit - 1, 0),
+        )
+        for payload in entries:
+            message = self._decode(payload)
+            removed = self._execute(
+                "redis.lrem",
+                self.redis.lrem,
+                self.processing_key,
+                1,
+                payload,
+            )
+            if removed < 1:
+                continue
+            self._execute("redis.lpush", self.redis.lpush, self.ready_key, payload)
+            self._emit(
+                MonitoringEventType.MESSAGE_RETRIED,
+                message,
+                attributes={"reason": "stale_processing_recovered"},
+            )
+            recovered += 1
+        return recovered
+
+    def dead_letters(self, *, limit: int = 100) -> list[Message]:
+        entries = self._execute(
+            "redis.lrange",
+            self.redis.lrange,
+            self.dead_key,
+            0,
+            max(limit - 1, 0),
+        )
+        return [self._decode(payload) for payload in entries]
+
+    def requeue_dead(self, message: Message) -> None:
+        encoded = self._encode(message)
+        removed = self._execute(
+            "redis.lrem",
+            self.redis.lrem,
+            self.dead_key,
+            1,
+            encoded,
+        )
+        if removed < 1:
+            raise AckError(
+                "message was not found in dead letter queue",
+                action="message.requeue_dead",
+                queue=self.config.queue,
+                details={"message_id": message.id, "key": self.dead_key},
+            )
+        self._execute("redis.lpush", self.redis.lpush, self.ready_key, encoded)
+        self._emit(
+            MonitoringEventType.MESSAGE_RETRIED,
+            message,
+            attributes={"reason": "dead_letter_requeued"},
+        )
+
+    def _move_to_processing(self, timeout: float) -> bytes | None:
+        if self.capabilities.supports_list_reliable_blmove:
+            return self._execute(
+                "redis.blmove",
+                self.redis.blmove,
+                self.ready_key,
+                self.processing_key,
+                timeout,
+                "RIGHT",
+                "LEFT",
+            )
+        return self._execute(
+            "redis.brpoplpush",
+            self.redis.brpoplpush,
+            self.ready_key,
+            self.processing_key,
+            timeout,
+        )
+
+    def _execute(self, action: str, func: Any, *args: Any) -> Any:
+        try:
+            return func(*args)
+        except Exception as exc:
+            self._emit_backend_error(action, str(exc))
+            raise BackendUnavailableError(
+                "Redis List backend command failed",
+                action=action,
+                queue=self.config.queue,
+            ) from exc