redqueue 0.10.0__py3-none-any.whl

redqueue/__init__.py

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Redis-backed message queue library."""
+
+from redqueue._version import __version__
+from redqueue.async_client import AsyncQueueClient
+from redqueue.client import QueueClient
+from redqueue.compat import (
+    RedisCapabilities,
+    RedisVersion,
+    detect_capabilities,
+    detect_capabilities_async,
+    extract_redis_version,
+)
+from redqueue.config import BackendType, QueueConfig, RetryConfig
+from redqueue.exceptions import (
+    AckError,
+    BackendUnavailableError,
+    ErrorContext,
+    MessageDecodeError,
+    MessageEncodeError,
+    QueueConfigError,
+    RedisCompatibilityError,
+    RedQueueError,
+    RetryExceededError,
+)
+from redqueue.message import Message, new_message_id
+from redqueue.monitoring import (
+    CompositeMonitoringHook,
+    InMemoryMonitoringHook,
+    MonitoringEvent,
+    MonitoringEventType,
+    MonitoringHook,
+    NoopMonitoringHook,
+    SafeMonitoringHook,
+)
+from redqueue.serialization import JsonSerializer, Serializer
+
+__all__ = [
+    "__version__",
+    "AckError",
+    "AsyncQueueClient",
+    "BackendType",
+    "BackendUnavailableError",
+    "ErrorContext",
+    "CompositeMonitoringHook",
+    "InMemoryMonitoringHook",
+    "JsonSerializer",
+    "Message",
+    "MessageDecodeError",
+    "MessageEncodeError",
+    "MonitoringEvent",
+    "MonitoringEventType",
+    "MonitoringHook",
+    "NoopMonitoringHook",
+    "QueueClient",
+    "QueueConfig",
+    "QueueConfigError",
+    "RedisCapabilities",
+    "RedQueueError",
+    "RedisCompatibilityError",
+    "RedisVersion",
+    "RetryConfig",
+    "RetryExceededError",
+    "SafeMonitoringHook",
+    "Serializer",
+    "detect_capabilities",
+    "detect_capabilities_async",
+    "extract_redis_version",
+    "new_message_id",
+]

redqueue/_version.py

@@ -0,0 +1,6 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Package version."""
+
+__version__ = "0.10.0"

redqueue/async_client.py

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Asynchronous RedQueue client."""
+
+from __future__ import annotations
+
+from typing import Any, cast
+
+from redis.asyncio import Redis
+
+from redqueue.backends import AsyncDelayBackend, AsyncListBackend, AsyncStreamBackend
+from redqueue.compat import (
+    AsyncRedisInfoClient,
+    RedisCapabilities,
+    detect_capabilities_async,
+)
+from redqueue.config import BackendType, QueueConfig
+from redqueue.message import Message
+
+
+class AsyncQueueClient:
+    """Asynchronous queue client with API parity to QueueClient."""
+
+    def __init__(
+        self,
+        config: QueueConfig,
+        *,
+        redis: Any | None = None,
+        capabilities: RedisCapabilities | None = None,
+    ) -> None:
+        self.config = config
+        self.redis = redis
+        self.capabilities = capabilities
+        self.backend: AsyncListBackend | AsyncStreamBackend | None = None
+        self.delay_backend: AsyncDelayBackend | None = None
+
+    @classmethod
+    async def from_url(
+        cls,
+        url: str,
+        *,
+        queue: str,
+        backend: str | BackendType = BackendType.LIST,
+        **options: Any,
+    ) -> AsyncQueueClient:
+        redis = options.pop("redis", None) or Redis.from_url(url)
+        capabilities = options.pop(
+            "capabilities",
+            None,
+        ) or await detect_capabilities_async(cast(AsyncRedisInfoClient, redis))
+        config = QueueConfig(queue=queue, backend=backend, **options)
+        client = cls(config=config, redis=redis, capabilities=capabilities)
+        await client._ensure_backend()
+        return client
+
+    async def publish(
+        self,
+        payload: Any,
+        *,
+        delay: float | None = None,
+        headers: dict[str, Any] | None = None,
+        message_id: str | None = None,
+    ) -> str:
+        if delay is not None:
+            return await self.delay(payload, delay_seconds=delay, headers=headers)
+        backend = await self._ensure_backend()
+        return await backend.publish(
+            payload,
+            headers=headers,
+            message_id=message_id,
+        )
+
+    async def consume(
+        self,
+        *,
+        timeout: float | None = None,
+        batch_size: int = 1,
+    ) -> Message | list[Message] | None:
+        backend = await self._ensure_backend()
+        return await backend.consume(
+            timeout=timeout,
+            batch_size=batch_size,
+        )
+
+    async def ack(self, message: Message) -> None:
+        await (await self._ensure_backend()).ack(message)
+
+    async def nack(self, message: Message, *, requeue: bool = True) -> None:
+        await (await self._ensure_backend()).nack(message, requeue=requeue)
+
+    async def retry(
+        self,
+        message: Message,
+        *,
+        delay: float | None = None,
+        reason: str | None = None,
+    ) -> None:
+        await (await self._ensure_backend()).retry(message, delay=delay, reason=reason)
+
+    async def delay(
+        self,
+        payload: Any,
+        *,
+        delay_seconds: float | None = None,
+        run_at: float | None = None,
+        headers: dict[str, Any] | None = None,
+    ) -> str:
+        delay_backend = await self._ensure_delay_backend()
+        message_id = await delay_backend.delay(
+            payload,
+            delay_seconds=delay_seconds,
+            run_at=run_at,
+            headers=headers,
+        )
+        return message_id
+
+    async def schedule_due(self, *, limit: int = 100, now: float | None = None) -> int:
+        return await (await self._ensure_delay_backend()).schedule_due(
+            limit=limit,
+            now=now,
+        )
+
+    async def recover_stale(
+        self,
+        *,
+        min_idle_ms: int | None = None,
+        limit: int = 100,
+    ) -> int:
+        backend = await self._ensure_backend()
+        if isinstance(backend, AsyncStreamBackend):
+            idle = min_idle_ms or int(self.config.visibility_timeout_seconds * 1000)
+            return len(await backend.recover_pending(min_idle_ms=idle, limit=limit))
+        return await backend.recover_stale(limit=limit)
+
+    async def dead_letters(self, *, limit: int = 100) -> list[Message]:
+        return await (await self._ensure_backend()).dead_letters(limit=limit)
+
+    async def requeue_dead(self, message: Message) -> None:
+        await (await self._ensure_backend()).requeue_dead(message)
+
+    async def close(self) -> None:
+        close = getattr(self.redis, "aclose", None) or getattr(
+            self.redis,
+            "close",
+            None,
+        )
+        if close is not None:
+            result = close()
+            if hasattr(result, "__await__"):
+                await result
+
+    async def _ensure_backend(self) -> AsyncListBackend | AsyncStreamBackend:
+        if self.backend is not None:
+            return self.backend
+        if self.config.backend_type is BackendType.LIST:
+            if self.redis is None:
+                raise TypeError("redis client is required for async List backend")
+            capabilities = self.capabilities
+            if capabilities is None:
+                raise TypeError("Redis capabilities are required before backend use")
+            self.backend = AsyncListBackend(self.redis, self.config, capabilities)
+            return self.backend
+        if self.config.backend_type is BackendType.STREAM:
+            if self.redis is None:
+                raise TypeError("redis client is required for async Streams backend")
+            capabilities = self.capabilities
+            if capabilities is None:
+                raise TypeError("Redis capabilities are required before backend use")
+            self.backend = await AsyncStreamBackend.create(
+                self.redis,
+                self.config,
+                capabilities,
+            )
+            return self.backend
+        raise NotImplementedError(
+            f"backend {self.config.backend_type.value!r} is not implemented"
+        )
+
+    async def _ensure_delay_backend(self) -> AsyncDelayBackend:
+        if self.delay_backend is not None:
+            return self.delay_backend
+        if self.redis is None:
+            raise TypeError("redis client is required for async delayed tasks")
+        capabilities = self.capabilities
+        if capabilities is None:
+            raise TypeError("Redis capabilities are required before delay backend use")
+        capabilities.require_delay_sorted_set()
+        self.delay_backend = AsyncDelayBackend(
+            self.redis,
+            self.config,
+            await self._ensure_backend(),
+        )
+        return self.delay_backend

redqueue/backends/__init__.py

@@ -0,0 +1,20 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Backend implementations for RedQueue."""
+
+from redqueue.backends.async_delay import AsyncDelayBackend
+from redqueue.backends.async_list import AsyncListBackend
+from redqueue.backends.async_stream import AsyncStreamBackend
+from redqueue.backends.delay import DelayBackend
+from redqueue.backends.list import ListBackend
+from redqueue.backends.stream import StreamBackend
+
+__all__ = [
+    "AsyncDelayBackend",
+    "AsyncListBackend",
+    "AsyncStreamBackend",
+    "DelayBackend",
+    "ListBackend",
+    "StreamBackend",
+]

redqueue/backends/async_delay.py

@@ -0,0 +1,242 @@
+# SPDX-License-Identifier: Apache-2.0
+# Author: SpringMirror-pear
+
+"""Asynchronous delayed task backend based on Redis Sorted Set."""
+
+from __future__ import annotations
+
+from time import time
+from typing import Any, Protocol
+
+from redqueue.config import QueueConfig
+from redqueue.exceptions import BackendUnavailableError, QueueConfigError
+from redqueue.message import Message, new_message_id
+from redqueue.monitoring import MonitoringEvent, MonitoringEventType
+
+
+class AsyncDelayRedis(Protocol):
+    """Redis command subset required by the async delayed task backend."""
+
+    async def set(self, name: str, value: bytes) -> bool: ...
+
+    async def get(self, name: str) -> bytes | None: ...
+
+    async def delete(self, *names: str) -> int: ...
+
+    async def zadd(self, name: str, mapping: dict[str, float]) -> int: ...
+
+    async def zrangebyscore(
+        self,
+        name: str,
+        min: float | str,
+        max: float | str,
+        start: int | None = None,
+        num: int | None = None,
+    ) -> list[str | bytes]: ...
+
+    async def zrem(self, name: str, *values: str) -> int: ...
+
+
+class AsyncDelayBackend:
+    """Async delayed task scheduler implemented with Redis Sorted Set."""
+
+    backend_name = "delay"
+
+    def __init__(
+        self,
+        redis: AsyncDelayRedis,
+        config: QueueConfig,
+        publisher: Any,
+    ) -> None:
+        self.redis = redis
+        self.config = config
+        self.publisher = publisher
+
+    @property
+    def delayed_key(self) -> str:
+        return self.config.key("delayed")
+
+    def payload_key(self, message_id: str) -> str:
+        return self.config.key(f"payload:{message_id}")
+
+    async def delay(
+        self,
+        payload: Any,
+        *,
+        delay_seconds: float | None = None,
+        run_at: float | None = None,
+        headers: dict[str, Any] | None = None,
+        message_id: str | None = None,
+    ) -> str:
+        available_at = self._available_at(delay_seconds=delay_seconds, run_at=run_at)
+        message = Message(
+            id=message_id or new_message_id(),
+            queue=self.config.queue,
+            payload=payload,
+            headers=headers or {},
+            available_at=available_at,
+            backend=self.backend_name,
+        )
+        await self._execute(
+            "redis.set",
+            self.redis.set,
+            self.payload_key(message.id),
+            self._encode(message),
+        )
+        await self._execute(
+            "redis.zadd",
+            self.redis.zadd,
+            self.delayed_key,
+            {message.id: available_at},
+        )
+        self._emit(MonitoringEventType.DELAY_SCHEDULED, message)
+        return message.id
+
+    async def schedule_due(self, *, limit: int = 100, now: float | None = None) -> int:
+        now_value = time() if now is None else now
+        due_ids = await self._execute(
+            "redis.zrangebyscore",
+            self.redis.zrangebyscore,
+            self.delayed_key,
+            "-inf",
+            now_value,
+            0,
+            limit,
+        )
+        released = 0
+        for raw_message_id in due_ids:
+            message_id = self._to_text(raw_message_id)
+            removed = await self._execute(
+                "redis.zrem",
+                self.redis.zrem,
+                self.delayed_key,
+                message_id,
+            )
+            if removed < 1:
+                continue
+            message = await self._load_message(message_id)
+            try:
+                await self.publisher.publish(
+                    message.payload,
+                    headers=message.headers,
+                    message_id=message.id,
+                )
+            except Exception:
+                await self._execute(
+                    "redis.zadd",
+                    self.redis.zadd,
+                    self.delayed_key,
+                    {message_id: message.available_at or now_value},
+                )
+                raise
+            await self._execute(
+                "redis.delete",
+                self.redis.delete,
+                self.payload_key(message_id),
+            )
+            self._emit(MonitoringEventType.DELAY_RELEASED, message)
+            released += 1
+        return released
+
+    async def _load_message(self, message_id: str) -> Message:
+        payload = await self._execute(
+            "redis.get",
+            self.redis.get,
+            self.payload_key(message_id),
+        )
+        if payload is None:
+            raise BackendUnavailableError(
+                "delayed payload is missing",
+                action="delay.load",
+                queue=self.config.queue,
+                details={"message_id": message_id},
+            )
+        return self._decode(payload)
+
+    def _available_at(
+        self,
+        *,
+        delay_seconds: float | None,
+        run_at: float | None,
+    ) -> float:
+        if delay_seconds is not None and run_at is not None:
+            raise QueueConfigError("delay_seconds and run_at cannot both be set")
+        if delay_seconds is not None:
+            if delay_seconds < 0:
+                raise QueueConfigError(
+                    "delay_seconds must be greater than or equal to 0"
+                )
+            return time() + delay_seconds
+        if run_at is not None:
+            if run_at < 0:
+                raise QueueConfigError("run_at must be greater than or equal to 0")
+            return run_at
+        return time()
+
+    def _encode(self, message: Message) -> bytes:
+        envelope = {
+            "id": message.id,
+            "queue": message.queue,
+            "payload": message.payload,
+            "headers": message.headers,
+            "attempts": message.attempts,
+            "created_at": message.created_at,
+            "available_at": message.available_at,
+            "backend": self.backend_name,
+            "raw_id": message.raw_id,
+        }
+        return self.config.serializer.encode(envelope, queue=self.config.queue)
+
+    def _decode(self, payload: bytes) -> Message:
+        envelope = self.config.serializer.decode(payload, queue=self.config.queue)
+        if not isinstance(envelope, dict):
+            raise BackendUnavailableError(
+                "decoded delayed message envelope must be a mapping",
+                action="delay.decode",
+                queue=self.config.queue,
+                details={"payload_type": type(envelope).__name__},
+            )
+        return Message(
+            id=str(envelope["id"]),
+            queue=str(envelope["queue"]),
+            payload=envelope["payload"],
+            headers=dict(envelope.get("headers") or {}),
+            attempts=int(envelope.get("attempts") or 0),
+            created_at=float(envelope["created_at"]),
+            available_at=envelope.get("available_at"),
+            backend=self.backend_name,
+            raw_id=envelope.get("raw_id"),
+        )
+
+    async def _execute(self, action: str, func: Any, *args: Any) -> Any:
+        try:
+            return await func(*args)
+        except Exception as exc:
+            self.config.monitoring.emit(
+                MonitoringEvent(
+                    type=MonitoringEventType.BACKEND_ERROR,
+                    queue=self.config.queue,
+                    backend=self.backend_name,
+                    error=str(exc),
+                    attributes={"action": action},
+                )
+            )
+            raise BackendUnavailableError(
+                "Redis async delayed task backend command failed",
+                action=action,
+                queue=self.config.queue,
+            ) from exc
+
+    def _emit(self, event_type: MonitoringEventType, message: Message) -> None:
+        self.config.monitoring.emit(
+            MonitoringEvent(
+                type=event_type,
+                queue=self.config.queue,
+                message_id=message.id,
+                backend=self.backend_name,
+            )
+        )
+
+    @staticmethod
+    def _to_text(value: str | bytes) -> str:
+        return value.decode() if isinstance(value, bytes) else value