rabbitkit 0.9.0__py3-none-any.whl

rabbitkit/dlq.py

@@ -0,0 +1,320 @@
+"""DLQ Inspector — peek, replay, and purge dead-letter queues.
+
+Provides inspection and recovery tools for messages stuck in DLQs.
+
+**Operational realism:**
+- ``peek()`` returns materialized snapshots, not live references
+- ``peek()`` may affect message ordering (basic.get + requeue changes position)
+- ``replay()`` preserves original headers (pass ``reset_retry_count=True`` to
+  grant the replayed message a fresh retry ladder)
+- ``replay()`` acks a DLQ original only after the republish outcome is OK;
+  failed republishes are nack-requeued so they stay on the DLQ
+- ``purge()`` is immediate and unfiltered — use ``replay()`` for selective recovery
+"""
+
+from __future__ import annotations
+
+import logging
+from collections.abc import Callable
+from typing import Any
+
+from rabbitkit.core.message import RabbitMessage
+from rabbitkit.core.types import MessageEnvelope
+
+logger = logging.getLogger(__name__)
+
+# Matches RetryConfig.retry_header's default. If you customized retry_header,
+# strip your header via a predicate/pre-processing step instead.
+_RETRY_COUNT_HEADER = "x-rabbitkit-retry-count"
+
+
+class ReplayResult(int):
+    """Replay report that IS the replayed count (int-compatible, so existing
+    ``count = inspector.replay(...)`` callers keep working), with extras:
+
+    - ``failed``: messages whose republish outcome was not OK — they were
+      nack-requeued and REMAIN ON THE DLQ.
+    - ``requeued``: non-matching messages returned to the DLQ (predicate
+      returned False).
+    """
+
+    failed: int
+    requeued: int
+
+    def __new__(cls, replayed: int, failed: int = 0, requeued: int = 0) -> ReplayResult:
+        obj = super().__new__(cls, replayed)
+        obj.failed = failed
+        obj.requeued = requeued
+        return obj
+
+    def __repr__(self) -> str:
+        return f"ReplayResult(replayed={int(self)}, failed={self.failed}, requeued={self.requeued})"
+
+
+class DLQInspector:
+    """Dead-letter queue inspection and replay.
+
+    Accepts a transport that supports ``basic_get``, ``publish``,
+    ``purge_queue`` methods. Works with both sync and async transports.
+
+    Usage::
+
+        inspector = DLQInspector(transport)
+
+        # Peek at messages without consuming them
+        messages = inspector.peek("orders-queue.dlq", limit=5)
+
+        # Replay matching messages back to source queue
+        count = inspector.replay(
+            "orders-queue.dlq",
+            predicate=lambda msg: msg.headers.get("x-error") == "timeout",
+            target_queue="orders-queue",
+        )
+
+        # Purge entire DLQ
+        count = inspector.purge("orders-queue.dlq")
+    """
+
+    def __init__(self, transport: Any) -> None:
+        self._transport = transport
+
+    # ── Sync methods ─────────────────────────────────────────────────────
+
+    def peek(self, queue: str, limit: int = 10) -> list[RabbitMessage]:
+        """Fetch up to ``limit`` messages from the queue, then requeue them.
+
+        Messages are nacked with ``requeue=True`` so they return to the queue.
+        Ordering may change after this operation.
+
+        Returns a list of message snapshots.
+        """
+        messages: list[RabbitMessage] = []
+
+        for _ in range(limit):
+            msg = self._transport.basic_get(queue)
+            if msg is None:
+                break
+            messages.append(msg)
+
+        # Requeue all peeked messages
+        for msg in messages:
+            if not msg.is_settled:
+                msg.nack(requeue=True)
+
+        return messages
+
+    @staticmethod
+    def _build_replay_envelope(
+        msg: RabbitMessage,
+        target_queue: str | None,
+        target_exchange: str | None,
+        reset_retry_count: bool,
+    ) -> MessageEnvelope:
+        """Build the republish envelope for one DLQ message.
+
+        ``mandatory=True`` so an unroutable target comes back as a
+        ``RETURNED`` outcome instead of being broker-confirmed into the void.
+        """
+        rk = target_queue or msg.headers.get("x-rabbitkit-original-queue", msg.routing_key)
+        headers = dict(msg.headers)
+        if reset_retry_count:
+            headers.pop(_RETRY_COUNT_HEADER, None)
+        return MessageEnvelope(
+            routing_key=rk,
+            body=msg.body,
+            exchange=target_exchange if target_exchange is not None else "",
+            headers=headers,
+            message_id=msg.message_id or "",
+            correlation_id=msg.correlation_id,
+            content_type=msg.content_type or "application/octet-stream",
+            content_encoding=msg.content_encoding,
+            # Preserve the remaining original message properties -- these used
+            # to be silently dropped on replay, e.g. a priority-queue message
+            # lost its priority, and an RPC request's reply_to/type/app_id/
+            # user_id never survived the replay for the reply to route back.
+            reply_to=msg.reply_to,
+            priority=msg.priority,
+            expiration=msg.expiration,
+            type=msg.type,
+            app_id=msg.app_id,
+            user_id=msg.user_id,
+            mandatory=True,
+        )
+
+    def replay(
+        self,
+        queue: str,
+        predicate: Callable[[RabbitMessage], bool] | None = None,
+        target_queue: str | None = None,
+        target_exchange: str | None = None,
+        *,
+        reset_retry_count: bool = False,
+    ) -> ReplayResult:
+        """Replay messages from the DLQ.
+
+        Fetches messages, applies optional predicate filter, publishes
+        matching messages to the target, and acks each original **only after
+        its republish outcome is OK**. A failed republish (NACKED / TIMEOUT /
+        RETURNED / ERROR) is nack-requeued, so the message stays on the DLQ
+        instead of being lost.
+
+        Non-matching messages are nacked with ``requeue=True``.
+
+        Args:
+            queue: Source DLQ to replay from.
+            predicate: Optional filter — only replay messages where
+                predicate returns True. All messages replayed if None.
+            target_queue: Target queue routing key. Defaults to the
+                original queue from message headers.
+            target_exchange: Target exchange. Defaults to "".
+            reset_retry_count: Strip the ``x-rabbitkit-retry-count`` header
+                so the replayed message gets a fresh retry ladder. Default
+                False preserves headers verbatim — meaning a previously
+                max-retried message is terminal after ONE failed attempt and
+                returns to the DLQ.
+
+        Returns:
+            :class:`ReplayResult` — int-compatible replayed count, with
+            ``.failed`` (left on the DLQ) and ``.requeued`` (non-matching).
+
+        Loop Engineering Review, Reliability: a non-matching or
+        failed-publish message is **not** nacked (requeued) until after this
+        method's fetch loop has fully exhausted the queue. ``basic_get`` has
+        no natural "already seen this delivery" tracking of its own -- if
+        such a message were requeued immediately, and nothing else is
+        consuming from this queue, the very next ``basic_get`` call in this
+        same loop could immediately re-fetch that exact message, forever.
+        Held-but-unsettled messages are invisible to further ``basic_get``
+        calls (the broker still considers them delivered-but-unacked), so
+        deferring the nack until after the loop truly exits guarantees
+        termination regardless of how many messages the predicate rejects or
+        the publisher fails.
+        """
+        replayed = 0
+        held_for_requeue: list[RabbitMessage] = []
+        failed = 0
+        requeued = 0
+
+        while True:
+            msg = self._transport.basic_get(queue)
+            if msg is None:
+                break
+
+            # Apply predicate filter -- hold, don't nack yet (see docstring).
+            if predicate is not None and not predicate(msg):
+                held_for_requeue.append(msg)
+                requeued += 1
+                continue
+
+            envelope = self._build_replay_envelope(msg, target_queue, target_exchange, reset_retry_count)
+            outcome = self._transport.publish(envelope)
+            if outcome is not None and not outcome.ok:
+                # Republish failed — DO NOT ack, or the message is lost
+                # forever. Hold for nack-requeue so it stays on the DLQ.
+                logger.error(
+                    "DLQ replay publish failed (status=%s); message stays on %r: routing_key=%s message_id=%s",
+                    getattr(outcome, "status", "unknown"),
+                    queue,
+                    envelope.routing_key,
+                    envelope.message_id,
+                )
+                held_for_requeue.append(msg)
+                failed += 1
+                continue
+
+            # Ack the original — it is safely republished now
+            if not msg.is_settled:
+                msg.ack()
+
+            replayed += 1
+
+        # The fetch loop is done (basic_get returned None) -- now it's safe
+        # to requeue held messages; this loop can no longer re-fetch them.
+        for msg in held_for_requeue:
+            if not msg.is_settled:
+                msg.nack(requeue=True)
+
+        return ReplayResult(replayed, failed=failed, requeued=requeued)
+
+    def purge(self, queue: str) -> int:
+        """Purge all messages from the queue.
+
+        Returns the number of messages purged.
+        """
+        return int(self._transport.purge_queue(queue))
+
+    # ── Async methods ────────────────────────────────────────────────────
+
+    async def peek_async(self, queue: str, limit: int = 10) -> list[RabbitMessage]:
+        """Async variant of ``peek``."""
+        messages: list[RabbitMessage] = []
+
+        for _ in range(limit):
+            msg = await self._transport.basic_get(queue)
+            if msg is None:
+                break
+            messages.append(msg)
+
+        # Requeue all peeked messages
+        for msg in messages:
+            if not msg.is_settled:
+                await msg.nack_async(requeue=True)
+
+        return messages
+
+    async def replay_async(
+        self,
+        queue: str,
+        predicate: Callable[[RabbitMessage], bool] | None = None,
+        target_queue: str | None = None,
+        target_exchange: str | None = None,
+        *,
+        reset_retry_count: bool = False,
+    ) -> ReplayResult:
+        """Async variant of ``replay`` -- see its docstring for the
+        outcome-checked ack, ``reset_retry_count``, and why non-matching /
+        failed messages are held, not nacked, until the fetch loop has fully
+        exhausted the queue (termination guarantee)."""
+        replayed = 0
+        held_for_requeue: list[RabbitMessage] = []
+        failed = 0
+        requeued = 0
+
+        while True:
+            msg = await self._transport.basic_get(queue)
+            if msg is None:
+                break
+
+            if predicate is not None and not predicate(msg):
+                held_for_requeue.append(msg)
+                requeued += 1
+                continue
+
+            envelope = self._build_replay_envelope(msg, target_queue, target_exchange, reset_retry_count)
+            outcome = await self._transport.publish(envelope)
+            if outcome is not None and not outcome.ok:
+                logger.error(
+                    "DLQ replay publish failed (status=%s); message stays on %r: routing_key=%s message_id=%s",
+                    getattr(outcome, "status", "unknown"),
+                    queue,
+                    envelope.routing_key,
+                    envelope.message_id,
+                )
+                held_for_requeue.append(msg)
+                failed += 1
+                continue
+
+            if not msg.is_settled:
+                await msg.ack_async()
+
+            replayed += 1
+
+        for msg in held_for_requeue:
+            if not msg.is_settled:
+                await msg.nack_async(requeue=True)
+
+        return ReplayResult(replayed, failed=failed, requeued=requeued)
+
+    async def purge_async(self, queue: str) -> int:
+        """Async variant of ``purge``."""
+        return int(await self._transport.purge_queue(queue))

rabbitkit/experimental/__init__.py

@@ -0,0 +1,50 @@
+"""rabbitkit.experimental — features under active development.
+
+These APIs are NOT covered by the stability guarantee. They may change or
+be removed in any release without a deprecation period.
+
+Stable APIs are in the top-level ``rabbitkit`` package.
+
+Experimental features:
+- RPC (tight coupling, use with care)
+- Dashboard (web UI for local development)
+- Stream queues
+- Distributed locking
+- Message signing
+- Result backends
+
+Usage::
+
+    from rabbitkit.experimental import AsyncRPCClient, RPCClient
+    from rabbitkit.experimental import create_dashboard_app
+    from rabbitkit.experimental import DistributedLock, RedisLock
+    from rabbitkit.experimental import SigningMiddleware
+    from rabbitkit.experimental import RedisResultBackend
+"""
+
+from rabbitkit.dashboard.app import create_dashboard_app
+from rabbitkit.locking import DistributedLock, LockMiddleware, RedisLock
+from rabbitkit.middleware.signing import InvalidSignatureError, SigningConfig, SigningMiddleware
+from rabbitkit.results.backend import RedisResultBackend, ResultBackend
+from rabbitkit.results.middleware import ResultMiddleware
+from rabbitkit.rpc import AsyncRPCClient, RPCClient, RPCTimeoutError
+from rabbitkit.streams import StreamConsumerConfig, StreamOffset, StreamOffsetType
+
+__all__ = [
+    "AsyncRPCClient",
+    "DistributedLock",
+    "InvalidSignatureError",
+    "LockMiddleware",
+    "RPCClient",
+    "RPCTimeoutError",
+    "RedisLock",
+    "RedisResultBackend",
+    "ResultBackend",
+    "ResultMiddleware",
+    "SigningConfig",
+    "SigningMiddleware",
+    "StreamConsumerConfig",
+    "StreamOffset",
+    "StreamOffsetType",
+    "create_dashboard_app",
+]

rabbitkit/fastapi.py

@@ -0,0 +1,91 @@
+"""FastAPI integration — lifespan context manager for rabbitkit.
+
+Thin module that wires broker startup/shutdown to FastAPI's lifespan.
+
+Usage::
+
+    from fastapi import FastAPI
+    from rabbitkit.fastapi import rabbitkit_lifespan
+
+    app = FastAPI(lifespan=rabbitkit_lifespan(broker=broker, rabbit_app=rabbit_app))
+
+Or as a decorator-style::
+
+    @asynccontextmanager
+    async def lifespan(app):
+        async with rabbitkit_lifespan(broker=broker):
+            yield
+
+    app = FastAPI(lifespan=lifespan)
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from collections.abc import AsyncIterator
+from contextlib import asynccontextmanager
+from typing import Any
+
+logger = logging.getLogger(__name__)
+
+
+@asynccontextmanager
+async def rabbitkit_lifespan(
+    app: Any = None,
+    *,
+    broker: Any | None = None,
+    rabbit_app: Any | None = None,
+) -> AsyncIterator[None]:
+    """Async context manager that starts/stops rabbitkit components.
+
+    Suitable for use as FastAPI's ``lifespan`` parameter or standalone.
+
+    Start order: rabbit_app.start_async() → broker.start() / broker.start_async()
+    Stop order (in finally): broker.stop() / broker.stop_async() → rabbit_app.stop_async()
+
+    Duck-types sync vs async: if the method is a coroutine, it is awaited.
+
+    Args:
+        app: FastAPI app instance (passed by FastAPI lifespan protocol, may be None).
+        broker: Optional rabbitkit broker (SyncBroker or AsyncBroker).
+        rabbit_app: Optional RabbitApp for lifecycle hooks.
+    """
+    try:
+        # Start rabbit_app first (startup hooks)
+        if rabbit_app is not None:
+            if hasattr(rabbit_app, "start_async"):
+                await rabbit_app.start_async()
+            elif hasattr(rabbit_app, "start"):
+                result = rabbit_app.start()
+                if asyncio.iscoroutine(result):
+                    await result
+
+        # Start broker
+        if broker is not None:
+            if asyncio.iscoroutinefunction(getattr(broker, "start", None)):
+                await broker.start()
+            elif hasattr(broker, "start"):
+                broker.start()
+
+        logger.info("rabbitkit lifespan started")
+        yield
+
+    finally:
+        # Stop broker first
+        if broker is not None:
+            if asyncio.iscoroutinefunction(getattr(broker, "stop", None)):
+                await broker.stop()
+            elif hasattr(broker, "stop"):
+                broker.stop()
+
+        # Stop rabbit_app (shutdown hooks)
+        if rabbit_app is not None:
+            if hasattr(rabbit_app, "stop_async"):
+                await rabbit_app.stop_async()
+            elif hasattr(rabbit_app, "stop"):
+                result = rabbit_app.stop()
+                if asyncio.iscoroutine(result):
+                    await result
+
+        logger.info("rabbitkit lifespan stopped")