redis-message-queue 8.0.3__tar.gz → 8.2.0__tar.gz

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis-message-queue
-Version: 8.0.3
+Version: 8.2.0
 Summary: Python message queuing with Redis and message deduplication
 License: MIT
 License-File: LICENSE
@@ -392,6 +392,32 @@ while not interrupt.is_interrupted():
 > processes, or install one top-level signal owner that calls `queue.drain()`
 > / `queue.aclose()` or sets an application stop event.
 
+If another library owns SIGTERM/SIGINT in the same process, adapt its shutdown
+signal to rmq with a user-owned event instead of installing rmq signal handlers:
+
+```python
+import threading
+
+from redis_message_queue import EventDrivenInterruptHandler, RedisMessageQueue
+
+stop_event = threading.Event()
+interrupt = EventDrivenInterruptHandler(stop_event)
+queue = RedisMessageQueue("q", client=client, interrupt=interrupt)
+
+while not interrupt.is_interrupted():
+    with queue.process_message() as message:
+        if message is not None:
+            process(message)
+
+# In the sibling library's shutdown hook:
+stop_event.set()
+queue.drain(timeout=25)
+```
+
+The caller MUST set `stop_event` before exiting. rmq observes
+`is_interrupted()` and exits cooperatively; it does not call `sys.exit()` or
+otherwise force process shutdown.
+
 There are three distinct shutdown shapes; pick the one that matches your runtime:
 
 | Shape | Trigger | In-flight handler | Pending claim IDs |
@@ -589,6 +615,11 @@ using `":queue:"` with a queue name that overlaps RQ keys. rmq has no fixed
 library prefix; generated keys share the Redis DB namespace with every other
 Redis user.
 
+Set `strict_envelope_decoding=True` if this Redis is shared with sibling task
+libraries (Celery, RQ, Dramatiq) to fail-fast on foreign payloads. With the
+default `False`, non-rmq values that do not start with the rmq envelope prefix
+remain backward-compatible raw messages and are yielded to the handler.
+
 ## Production notes
 
 ### Fork safety and pre-fork servers

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/README.md

@@ -366,6 +366,32 @@ while not interrupt.is_interrupted():
 > processes, or install one top-level signal owner that calls `queue.drain()`
 > / `queue.aclose()` or sets an application stop event.
 
+If another library owns SIGTERM/SIGINT in the same process, adapt its shutdown
+signal to rmq with a user-owned event instead of installing rmq signal handlers:
+
+```python
+import threading
+
+from redis_message_queue import EventDrivenInterruptHandler, RedisMessageQueue
+
+stop_event = threading.Event()
+interrupt = EventDrivenInterruptHandler(stop_event)
+queue = RedisMessageQueue("q", client=client, interrupt=interrupt)
+
+while not interrupt.is_interrupted():
+    with queue.process_message() as message:
+        if message is not None:
+            process(message)
+
+# In the sibling library's shutdown hook:
+stop_event.set()
+queue.drain(timeout=25)
+```
+
+The caller MUST set `stop_event` before exiting. rmq observes
+`is_interrupted()` and exits cooperatively; it does not call `sys.exit()` or
+otherwise force process shutdown.
+
 There are three distinct shutdown shapes; pick the one that matches your runtime:
 
 | Shape | Trigger | In-flight handler | Pending claim IDs |
@@ -563,6 +589,11 @@ using `":queue:"` with a queue name that overlaps RQ keys. rmq has no fixed
 library prefix; generated keys share the Redis DB namespace with every other
 Redis user.
 
+Set `strict_envelope_decoding=True` if this Redis is shared with sibling task
+libraries (Celery, RQ, Dramatiq) to fail-fast on foreign payloads. With the
+default `False`, non-rmq values that do not start with the rmq envelope prefix
+remain backward-compatible raw messages and are yielded to the handler.
+
 ## Production notes
 
 ### Fork safety and pre-fork servers

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-message-queue"
-version = "8.0.3"
+version = "8.2.0"
 description = "Python message queuing with Redis and message deduplication"
 authors = ["Elijas <4084885+Elijas@users.noreply.github.com>"]
 readme = "README.md"

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/redis_message_queue/__init__.py

@@ -1,20 +1,25 @@
 from redis_message_queue._abstract_redis_gateway import AbstractRedisGateway
 from redis_message_queue._event import EventOperation, EventOutcome, QueueEvent
 from redis_message_queue._exceptions import (
+    ClaimStoreFailedError,
     CleanupFailedError,
     ConfigurationError,
+    DrainFailedError,
     GatewayContractError,
     LuaScriptError,
     MalformedStoredMessageError,
+    PayloadTooDeepError,
+    PayloadTooLargeError,
     QueueBackpressureError,
     QueueDrainedError,
     RedisMessageQueueError,
     RetryBudgetExhaustedError,
 )
 from redis_message_queue._redis_gateway import RedisGateway
-from redis_message_queue._stored_message import ClaimedMessage, MessageData
+from redis_message_queue._stored_message import ClaimedMessage, MessageData, MessagePayload
 from redis_message_queue.interrupt_handler import (
     BaseGracefulInterruptHandler,
+    EventDrivenInterruptHandler,
     GracefulInterruptHandler,
 )
 from redis_message_queue.redis_message_queue import RedisMessageQueue
@@ -25,16 +30,22 @@ __all__ = [
     "AbstractRedisGateway",
     "ClaimedMessage",
     "MessageData",
+    "MessagePayload",
+    "EventDrivenInterruptHandler",
     "GracefulInterruptHandler",
     "BaseGracefulInterruptHandler",
     "QueueEvent",
     "EventOperation",
     "EventOutcome",
     "RedisMessageQueueError",
+    "ClaimStoreFailedError",
     "ConfigurationError",
+    "DrainFailedError",
     "GatewayContractError",
     "LuaScriptError",
     "MalformedStoredMessageError",
+    "PayloadTooLargeError",
+    "PayloadTooDeepError",
     "QueueBackpressureError",
     "QueueDrainedError",
     "CleanupFailedError",

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/redis_message_queue/_abstract_redis_gateway.py

@@ -12,13 +12,12 @@ class AbstractRedisGateway(ABC):
     gateways MUST uphold the same behavioral contracts documented on each method
     to avoid phantom heartbeats, undetected lease conflicts, or silent data loss.
 
-    Gateways that support visibility timeouts (lease-based claiming) MUST expose
-    a ``message_visibility_timeout_seconds`` property (int or None). This is not
-    abstract because it is configuration rather than protocol, but it is required
-    when the queue is configured with ``heartbeat_interval_seconds``.
-    Lease-capable custom gateways MUST expose this property; omitting it
-    silently disables heartbeat validation and lease-token safety checks,
-    causing the queue to treat the gateway as a non-lease implementation.
+    Gateways that support visibility timeouts (lease-based claiming) MUST
+    override the ``message_visibility_timeout_seconds`` property with a positive
+    int. The abstract base declares this property with a ``None`` default so
+    non-lease custom gateways keep the existing behavior, while lease-capable
+    custom gateways have a typeable contract to override. A positive value is
+    required when the queue is configured with ``heartbeat_interval_seconds``.
 
     The queue also reads ``max_delivery_count`` and ``dead_letter_queue``
     from the gateway. The abstract base provides ``None`` defaults via
@@ -65,6 +64,14 @@ class AbstractRedisGateway(ABC):
     def dead_letter_queue(self) -> str | None:
         return None
 
+    @property
+    def message_visibility_timeout_seconds(self) -> int | None:
+        """Visibility timeout (lease duration) in seconds. Override to enable
+        lease-based crash recovery; return None to disable. Required when the
+        queue is configured with ``heartbeat_interval_seconds``.
+        """
+        return None
+
     @abstractmethod
     def publish_message(self, queue: str, message: str, dedup_key: str) -> bool:
         """Publish a message with deduplication.

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/redis_message_queue/_config.py

@@ -31,6 +31,7 @@ DEFAULT_RETRY_INITIAL_DELAY_SECONDS = 0.01
 DEFAULT_PENDING_OVERLOAD_BLOCK_TIMEOUT_SECONDS = 1.0
 INTERRUPTIBLE_RETRY_SLEEP_POLL_SECONDS = 0.05
 PENDING_OVERLOAD_LUA_SENTINEL = -1
+CLAIM_STORE_FAILED_LUA_SENTINEL = "\0__rmq_claim_store_failed__"
 PENDING_OVERLOAD_POLICIES = ("raise", "drop_oldest", "block")
 DEDUPLICATION_REQUIRES_KEY_MESSAGE = (
     "deduplication=True requires get_deduplication_key (callable returning a non-empty str). "
@@ -53,6 +54,13 @@ def is_redis_retryable_exception(exception):
     if isinstance(exception, redis.exceptions.ClusterError) and "TTL exhausted" in str(exception):
         return True
 
+    no_script_error = getattr(redis.exceptions, "NoScriptError", None)
+    if no_script_error is not None and isinstance(exception, no_script_error):
+        return True
+
+    if isinstance(exception, redis.exceptions.ResponseError) and str(exception).startswith("NOSCRIPT"):
+        return True
+
     # 2. Explicit retryable exceptions (BusyLoadingError is a ConnectionError
     #    subclass, so it is already handled by branch 1 above)
     return isinstance(
@@ -832,11 +840,13 @@ if #to_requeue > 0 then
     redis.call('RPUSH', KEYS[1], unpack(to_requeue))
 end
 local dead_lettered_events = {}
+local claim_store_failed_sentinel = string.char(0) .. '__rmq_claim_store_failed__'
 
 local function store_claim_and_return(stored)
     -- pcall guards against OOM mid-write: compensate by returning message to pending
     local ok, result = pcall(function()
-        local lease_token = tostring(redis.call('INCR', KEYS[5]))
+        redis.call('INCR', KEYS[5])
+        local lease_token = redis.call('GET', KEYS[5])
         local claim_payload = cjson.encode({stored, lease_token})
         redis.call('ZADD', KEYS[3], now_ms + tonumber(ARGV[1]), stored)
         redis.call('HSET', KEYS[4], stored, lease_token)
@@ -847,9 +857,10 @@ local function store_claim_and_return(stored)
         return {stored, lease_token, reclaimed_events, dead_lettered_events}
     end)
     if not ok then
+        redis.call('HINCRBY', KEYS[6], stored, -1)
         redis.call('LREM', KEYS[2], 1, stored)
         redis.pcall('RPUSH', KEYS[1], stored)
-        return false
+        return {claim_store_failed_sentinel, tostring(result), stored}
     end
     return result
 end

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/redis_message_queue/_event.py

@@ -24,6 +24,7 @@ class EventOperation(StrEnum):
     TRIM_FAILED = "trim_failed"
     RETRY_ATTEMPT = "retry_attempt"
     RETRY_EXHAUSTED = "retry_exhausted"
+    DRAIN = "drain"
 
 
 class EventOutcome(StrEnum):
@@ -32,6 +33,7 @@ class EventOutcome(StrEnum):
     SUCCESS = "success"
     FAILURE = "failure"
     SKIPPED = "skipped"
+    START = "start"
 
 
 @dataclass(frozen=True)
@@ -68,3 +70,7 @@ class QueueEvent:
     the actual exception object when one was raised; pass to OpenTelemetry
     `span.record_exception(...)` for full trace attribution
     """
+    timeout_seconds: float | None = None
+    """the caller-requested timeout for drain operations, when applicable"""
+    pending_claim_ids: int | None = None
+    """number of unresolved pending claim ids for drain operations, when applicable"""

redis_message_queue-8.2.0/redis_message_queue/_exceptions.py

@@ -0,0 +1,165 @@
+import redis.exceptions
+
+
+class RedisMessageQueueError(Exception):
+    """Base class for redis-message-queue specific errors."""
+
+    def __init__(
+        self,
+        *args: object,
+        queue: str | None = None,
+        message_id: str | None = None,
+        operation: str | None = None,
+    ) -> None:
+        super().__init__(*args)
+        self.queue = queue
+        self.message_id = message_id
+        self.operation = operation
+
+
+class ConfigurationError(RedisMessageQueueError, ValueError):
+    """Bad parameter values or combinations."""
+
+
+class GatewayContractError(RedisMessageQueueError, TypeError):
+    """Custom gateway returned wrong type or violated contract."""
+
+
+class LuaScriptError(redis.exceptions.ResponseError, RedisMessageQueueError):
+    """A Lua script returned an unexpected error_reply."""
+
+    def __init__(
+        self,
+        *args: object,
+        queue: str | None = None,
+        message_id: str | None = None,
+        operation: str | None = None,
+    ) -> None:
+        RedisMessageQueueError.__init__(
+            self,
+            *args,
+            queue=queue,
+            message_id=message_id,
+            operation=operation,
+        )
+
+
+class CleanupFailedError(RedisMessageQueueError):
+    """Cleanup after handler completion failed."""
+
+
+class ClaimStoreFailedError(RedisMessageQueueError):
+    """Raised when the VT-claim Lua store_claim_and_return pcall failed.
+
+    The script decremented the speculative delivery_count increment and
+    compensated by returning the message to pending before surfacing this error.
+    """
+
+
+class DrainFailedError(RedisMessageQueueError):
+    """Wraps a non-RMQ exception caught during drain pending-claim recovery.
+
+    drain() returns False as the bool result; this exception carries
+    F7 context (queue, operation="drain") into the drain/failure event
+    payload so users diagnosing drain incidents via on_event see the
+    same structured attrs as elsewhere.
+    """
+
+
+class MalformedStoredMessageError(RedisMessageQueueError):
+    """Stored value is not a valid RMQ envelope for the configured decode mode."""
+
+
+class PayloadTooLargeError(RedisMessageQueueError, ValueError):
+    """Publish payload exceeds the configured serialized byte limit."""
+
+
+class PayloadTooDeepError(RedisMessageQueueError, ValueError):
+    """Publish payload exceeds the configured nesting-depth limit."""
+
+
+class QueueBackpressureError(RedisMessageQueueError):
+    """Publish rejected because the pending queue is at its configured limit."""
+
+    _REMEDIATION = (
+        "consider increasing `max_pending_length`, switching to "
+        "`pending_overload_policy='block'`, or adding consumer capacity."
+    )
+
+    def __init__(
+        self,
+        *args: object,
+        queue: str | None = None,
+        message_id: str | None = None,
+        operation: str | None = None,
+    ) -> None:
+        message = "Pending queue reached its configured limit" if not args else args[0]
+        if not isinstance(message, str) or len(args) > 1:
+            super().__init__(*args, queue=queue, message_id=message_id, operation=operation)
+            return
+        if self._REMEDIATION not in message:
+            message = f"{message}; {self._REMEDIATION}"
+        super().__init__(message, queue=queue, message_id=message_id, operation=operation)
+
+
+class QueueDrainedError(RedisMessageQueueError):
+    """Raised when publish() is called after drain() or aclose()."""
+
+
+class RetryBudgetExhaustedError(redis.exceptions.RedisError, RedisMessageQueueError):
+    """Tenacity retry budget exhausted; underlying redis-py exception is .__cause__."""
+
+    _REMEDIATION = (
+        "verify Redis connectivity and consider increasing `retry_budget_seconds` if transient failures are expected."
+    )
+
+    def __init__(
+        self,
+        *args: object,
+        queue: str | None = None,
+        message_id: str | None = None,
+        operation: str | None = None,
+    ) -> None:
+        message = "Redis retry budget exhausted" if not args else args[0]
+        if not isinstance(message, str) or len(args) > 1:
+            RedisMessageQueueError.__init__(
+                self,
+                *args,
+                queue=queue,
+                message_id=message_id,
+                operation=operation,
+            )
+            return
+        if self._REMEDIATION not in message:
+            message = f"{message}; {self._REMEDIATION}"
+        RedisMessageQueueError.__init__(
+            self,
+            message,
+            queue=queue,
+            message_id=message_id,
+            operation=operation,
+        )
+
+
+def _set_exception_context(
+    exc: BaseException,
+    *,
+    queue: str | None = None,
+    message_id: str | None = None,
+    operation: str | None = None,
+) -> None:
+    if not isinstance(exc, RedisMessageQueueError):
+        return
+    if queue is not None:
+        exc.queue = queue
+    if message_id is not None:
+        exc.message_id = message_id
+    if operation is not None:
+        exc.operation = operation
+
+
+def wrap_lua_response_error(exc: redis.exceptions.ResponseError) -> LuaScriptError | None:
+    message = str(exc)
+    if message.startswith("WRONGTYPE ") or message.startswith("OOM during publish;"):
+        return LuaScriptError(message)
+    return None

redis_message_queue-8.2.0/redis_message_queue/_payload_limits.py

@@ -0,0 +1,72 @@
+import json
+
+from redis_message_queue._exceptions import ConfigurationError, PayloadTooDeepError, PayloadTooLargeError
+
+
+def validate_payload_limit_parameter(name: str, value: int | None) -> int | None:
+    if value is None:
+        return None
+    if not isinstance(value, int) or isinstance(value, bool):
+        bool_hint = " (use a positive int or None, not True/False)" if isinstance(value, bool) else ""
+        raise TypeError(f"'{name}' must be an int or None, got {type(value).__name__}{bool_hint}")
+    if value <= 0:
+        raise ConfigurationError(f"'{name}' must be positive when provided, got {value}")
+    return value
+
+
+def validate_max_payload_depth(message: dict, max_payload_depth: int | None) -> None:
+    if max_payload_depth is None:
+        return
+
+    stack: list[tuple[object, str, int]] = [(message, "message", 0)]
+    seen: set[int] = set()
+    while stack:
+        value, path, depth = stack.pop()
+        if depth > max_payload_depth:
+            raise PayloadTooDeepError(
+                f"max_payload_depth={max_payload_depth} exceeded: depth {depth} reached at {path}"
+            )
+        if isinstance(value, dict):
+            current_id = id(value)
+            if current_id in seen:
+                continue
+            seen.add(current_id)
+            children = list(value.items())
+            for key, child in reversed(children):
+                stack.append((child, f"{path}[{key!r}]", depth + 1))
+        elif isinstance(value, (list, tuple)):
+            current_id = id(value)
+            if current_id in seen:
+                continue
+            seen.add(current_id)
+            for index in range(len(value) - 1, -1, -1):
+                stack.append((value[index], f"{path}[{index}]", depth + 1))
+
+
+def serialize_dict_payload_with_limit(message: dict, max_payload_bytes: int | None) -> str:
+    message_str = json.dumps(message, sort_keys=True, allow_nan=False)
+    if max_payload_bytes is not None:
+        validate_max_payload_bytes(
+            len(message_str.encode("utf-8")),
+            max_payload_bytes,
+            payload_type="dict message",
+        )
+    return message_str
+
+
+def validate_str_payload_size(message: str, max_payload_bytes: int | None) -> None:
+    if max_payload_bytes is None:
+        return
+    validate_max_payload_bytes(
+        len(message.encode("utf-8")),
+        max_payload_bytes,
+        payload_type="str message",
+    )
+
+
+def validate_max_payload_bytes(size_bytes: int, max_payload_bytes: int | None, *, payload_type: str) -> None:
+    if max_payload_bytes is None or size_bytes <= max_payload_bytes:
+        return
+    raise PayloadTooLargeError(
+        f"max_payload_bytes={max_payload_bytes} exceeded: payload is {size_bytes} bytes ({payload_type})"
+    )

{redis_message_queue-8.0.3 → redis_message_queue-8.2.0}/redis_message_queue/_queue_key_manager.py

@@ -1,7 +1,8 @@
 from redis_message_queue._exceptions import ConfigurationError
+from redis_message_queue._stored_message import MessagePayload
 
 
-def validate_callable_deduplication_key(dedup_key: object, message: str | dict) -> str:
+def validate_callable_deduplication_key(dedup_key: object, message: MessagePayload) -> str:
     if dedup_key is None:
         raise ConfigurationError(
             f"get_deduplication_key returned None for message {message!r}; the callable must return a non-empty string"