redis-message-queue 8.0.1__tar.gz → 8.0.2__tar.gz

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis-message-queue
-Version: 8.0.1
+Version: 8.0.2
 Summary: Python message queuing with Redis and message deduplication
 License: MIT
 License-File: LICENSE

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "redis-message-queue"
-version = "8.0.1"
+version = "8.0.2"
 description = "Python message queuing with Redis and message deduplication"
 authors = ["Elijas <4084885+Elijas@users.noreply.github.com>"]
 readme = "README.md"

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/__init__.py

@@ -5,6 +5,7 @@ from redis_message_queue._exceptions import (
     ConfigurationError,
     GatewayContractError,
     LuaScriptError,
+    MalformedStoredMessageError,
     QueueBackpressureError,
     QueueDrainedError,
     RedisMessageQueueError,
@@ -33,6 +34,7 @@ __all__ = [
     "ConfigurationError",
     "GatewayContractError",
     "LuaScriptError",
+    "MalformedStoredMessageError",
     "QueueBackpressureError",
     "QueueDrainedError",
     "CleanupFailedError",

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/_config.py

@@ -739,6 +739,26 @@ local function redis_message_queue_decode_claim(cached_claim)
     return nil
 end
 
+local function redis_message_queue_decode_envelope(stored)
+    local prefix = string.char(30) .. 'RMQ1:'
+    if type(stored) ~= 'string' or string.sub(stored, 1, string.len(prefix)) ~= prefix then
+        return nil
+    end
+    local ok, envelope = pcall(cjson.decode, string.sub(stored, string.len(prefix) + 1))
+    if ok and type(envelope) == 'table' and type(envelope['id']) == 'string' then
+        return envelope
+    end
+    return nil
+end
+
+local function redis_message_queue_message_id(stored)
+    local envelope = redis_message_queue_decode_envelope(stored)
+    if envelope then
+        return envelope['id']
+    end
+    return ''
+end
+
 local time = redis.call('TIME')
 local now_ms = tonumber(time[1]) * 1000 + math.floor(tonumber(time[2]) / 1000)
 
@@ -779,6 +799,7 @@ end
 -- With a single consumer polling at default interval, 1000 expired leases drain in ~2.5s.
 local expired = redis.call('ZRANGEBYSCORE', KEYS[3], '-inf', now_ms, 'LIMIT', 0, 100)
 local to_requeue = {}
+local reclaimed_events = {}
 for i = #expired, 1, -1 do
     local expired_lease_token = redis.call('HGET', KEYS[4], expired[i])
     redis.call('ZREM', KEYS[3], expired[i])
@@ -800,13 +821,14 @@ for i = #expired, 1, -1 do
     end
     if redis.call('LREM', KEYS[2], 1, expired[i]) == 1 then
         table.insert(to_requeue, expired[i])
+        local delivery_count = redis.call('HGET', KEYS[6], expired[i])
+        table.insert(reclaimed_events, {redis_message_queue_message_id(expired[i]), tostring(delivery_count or '0')})
     end
 end
 if #to_requeue > 0 then
     redis.call('RPUSH', KEYS[1], unpack(to_requeue))
 end
-local reclaimed_count = #to_requeue
-local dead_lettered_count = 0
+local dead_lettered_events = {}
 
 local function store_claim_and_return(stored)
     -- pcall guards against OOM mid-write: compensate by returning message to pending
@@ -819,7 +841,7 @@ local function store_claim_and_return(stored)
         redis.call('HSET', KEYS[9], lease_token, KEYS[8])
         redis.call('HSET', KEYS[10], ARGV[4], claim_payload)
         redis.call('HSET', KEYS[11], lease_token, ARGV[4])
-        return {stored, lease_token, tostring(reclaimed_count), tostring(dead_lettered_count)}
+        return {stored, lease_token, reclaimed_events, dead_lettered_events}
     end)
     if not ok then
         redis.call('LREM', KEYS[2], 1, stored)
@@ -835,36 +857,28 @@ while claim_attempts < 100 do
 
     local stored = redis.call('LMOVE', KEYS[1], KEYS[2], 'RIGHT', 'LEFT')
     if not stored then
-        return {'', '', tostring(reclaimed_count), tostring(dead_lettered_count)}
+        return {'', '', reclaimed_events, dead_lettered_events}
     end
 
-    if max_delivery_count > 0 then
-        local count = redis.call('HINCRBY', KEYS[6], stored, 1)
-        if count > max_delivery_count then
-            redis.call('LREM', KEYS[2], 1, stored)
-            redis.call('HDEL', KEYS[6], stored)
-            -- Strip envelope to store raw payload in DLQ, consistent with completed/failed queues.
-            -- The per-delivery UUID in the envelope is lost; see README dead-letter notes.
-            local dead_letter_value = stored
-            local prefix = string.char(30) .. 'RMQ1:'
-            if string.sub(stored, 1, string.len(prefix)) == prefix then
-                local ok, envelope = pcall(cjson.decode, string.sub(stored, string.len(prefix) + 1))
-                if ok and type(envelope) == 'table' and type(envelope['id']) == 'string'
-                        and type(envelope['payload']) == 'string' then
-                    dead_letter_value = envelope['payload']
-                end
-            end
-            redis.call('LPUSH', KEYS[7], dead_letter_value)
-            dead_lettered_count = dead_lettered_count + 1
-        else
-            return store_claim_and_return(stored)
+    local count = redis.call('HINCRBY', KEYS[6], stored, 1)
+    if max_delivery_count > 0 and count > max_delivery_count then
+        redis.call('LREM', KEYS[2], 1, stored)
+        redis.call('HDEL', KEYS[6], stored)
+        -- Strip envelope to store raw payload in DLQ, consistent with completed/failed queues.
+        -- The per-delivery UUID in the envelope is lost; see README dead-letter notes.
+        local dead_letter_value = stored
+        local envelope = redis_message_queue_decode_envelope(stored)
+        if envelope and type(envelope['payload']) == 'string' then
+            dead_letter_value = envelope['payload']
         end
+        redis.call('LPUSH', KEYS[7], dead_letter_value)
+        table.insert(dead_lettered_events, {redis_message_queue_message_id(stored), tostring(count)})
     else
         return store_claim_and_return(stored)
     end
 end
 
-return {'', '', tostring(reclaimed_count), tostring(dead_lettered_count)}
+return {'', '', reclaimed_events, dead_lettered_events}
 """
 )
 
@@ -1037,6 +1051,46 @@ return removed
 """
 )
 
+CLEANUP_DRAINED_LEASE_TOKEN_COUNTER_LUA_SCRIPT = (
+    _LUA_KEY_TYPE_GUARD
+    + """
+local err = redis_message_queue_require_type(KEYS[1], 'list')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[2], 'zset')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[3], 'hash')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[4], 'hash')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[5], 'string')
+if err then
+    return err
+end
+
+if redis.call('LLEN', KEYS[1]) == 0
+    and redis.call('ZCARD', KEYS[2]) == 0
+    and redis.call('HLEN', KEYS[3]) == 0
+    and redis.call('HLEN', KEYS[4]) == 0 then
+    redis.call('DEL', KEYS[5])
+    return 1
+end
+
+return 0
+"""
+)
+
 RENEW_MESSAGE_LEASE_LUA_SCRIPT = (
     _LUA_KEY_TYPE_GUARD
     + """

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/_event.py

@@ -52,6 +52,10 @@ class QueueEvent:
     """a diagnostic hash of the lease token when visibility timeout is enabled"""
     destination_queue: str | None = None
     """the queue a message was moved to, when applicable"""
+    delivery_count: int | None = None
+    """the number of delivery attempts recorded for this message, when applicable"""
+    max_delivery_count: int | None = None
+    """the configured delivery-attempt threshold, when applicable"""
     exception_type: str | None = None
     """
     type name of the raised exception for metrics labels (e.g., 'TimeoutError');

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/_exceptions.py

@@ -21,6 +21,10 @@ class CleanupFailedError(RedisMessageQueueError):
     """Cleanup after handler completion failed."""
 
 
+class MalformedStoredMessageError(RedisMessageQueueError):
+    """Stored value starts with the RMQ envelope prefix but is not a valid envelope."""
+
+
 class QueueBackpressureError(RedisMessageQueueError):
     """Publish rejected because the pending queue is at its configured limit."""
 

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/_redis_gateway.py

@@ -16,6 +16,7 @@ from redis_message_queue._config import (
     ADD_MESSAGE_LUA_SCRIPT,
     CLAIM_MESSAGE_LUA_SCRIPT,
     CLAIM_MESSAGE_WITH_VISIBILITY_TIMEOUT_LUA_SCRIPT,
+    CLEANUP_DRAINED_LEASE_TOKEN_COUNTER_LUA_SCRIPT,
     DEFAULT_MESSAGE_DEDUPLICATION_LOG_TTL,
     DEFAULT_MESSAGE_WAIT_INTERVAL_SECONDS,
     DEFAULT_PENDING_OVERLOAD_BLOCK_TIMEOUT_SECONDS,
@@ -56,6 +57,7 @@ from redis_message_queue.interrupt_handler._interface import (
 logger = logging.getLogger(__name__)
 _TClaim = TypeVar("_TClaim", bound=ClaimedMessage | MessageData)
 _TRedisCall = TypeVar("_TRedisCall")
+_MessageAttemptEvent = tuple[str | None, int]
 
 _LEASE_DEADLINES_SUFFIX = ":lease_deadlines"
 _LEASE_TOKENS_SUFFIX = ":lease_tokens"
@@ -125,6 +127,26 @@ def _coerce_lua_count(value: object) -> int:
         return 0
 
 
+def _decode_lua_text(value: object) -> str | None:
+    if isinstance(value, bytes):
+        value = value.decode("utf-8")
+    if isinstance(value, str) and value:
+        return value
+    return None
+
+
+def _coerce_lua_message_attempts(value: object) -> list[_MessageAttemptEvent]:
+    if not isinstance(value, list | tuple):
+        return []
+
+    attempts: list[_MessageAttemptEvent] = []
+    for item in value:
+        if not isinstance(item, list | tuple) or len(item) < 2:
+            continue
+        attempts.append((_decode_lua_text(item[0]), _coerce_lua_count(item[1])))
+    return attempts
+
+
 def _pending_overload_max_backoff_seconds(block_timeout_seconds: float) -> float:
     return min(_PENDING_OVERLOAD_MAX_BACKOFF_SECONDS, block_timeout_seconds / 10)
 
@@ -259,8 +281,11 @@ class RedisGateway(AbstractRedisGateway):
         operation: EventOperation,
         outcome: EventOutcome,
         *,
+        message_id: str | None = None,
         claim_id: str | None = None,
         destination_queue: str | None = None,
+        delivery_count: int | None = None,
+        max_delivery_count: int | None = None,
         exception_type: str | None = None,
         error: BaseException | None = None,
     ) -> None:
@@ -269,8 +294,11 @@ class RedisGateway(AbstractRedisGateway):
         self._event_emitter(
             operation,
             outcome,
+            message_id=message_id,
             claim_id=claim_id,
             destination_queue=destination_queue,
+            delivery_count=delivery_count,
+            max_delivery_count=max_delivery_count,
             exception_type=exception_type,
             error=error,
         )
@@ -278,12 +306,20 @@ class RedisGateway(AbstractRedisGateway):
     def _emit_repeated_event(
         self,
         operation: EventOperation,
-        count: int,
+        attempts: list[_MessageAttemptEvent],
         *,
         destination_queue: str | None = None,
+        max_delivery_count: int | None = None,
     ) -> None:
-        for _ in range(max(count, 0)):
-            self._emit_event(operation, "success", destination_queue=destination_queue)
+        for message_id, delivery_count in attempts:
+            self._emit_event(
+                operation,
+                "success",
+                message_id=message_id,
+                destination_queue=destination_queue,
+                delivery_count=delivery_count,
+                max_delivery_count=max_delivery_count,
+            )
 
     def _eval(self, *args: object) -> object:
         try:
@@ -839,10 +875,15 @@ class RedisGateway(AbstractRedisGateway):
             return None
 
         stored_message, lease_token = result[0], result[1]
-        reclaimed_count = _coerce_lua_count(result[2]) if len(result) > 2 else 0
-        dead_lettered_count = _coerce_lua_count(result[3]) if len(result) > 3 else 0
-        self._emit_repeated_event("claim_reclaim", reclaimed_count)
-        self._emit_repeated_event("dlq", dead_lettered_count, destination_queue=self._dead_letter_queue)
+        reclaimed_attempts = _coerce_lua_message_attempts(result[2]) if len(result) > 2 else []
+        dead_lettered_attempts = _coerce_lua_message_attempts(result[3]) if len(result) > 3 else []
+        self._emit_repeated_event("claim_reclaim", reclaimed_attempts)
+        self._emit_repeated_event(
+            "dlq",
+            dead_lettered_attempts,
+            destination_queue=self._dead_letter_queue,
+            max_delivery_count=self._max_delivery_count,
+        )
         if stored_message in ("", b"") and lease_token in ("", b""):
             return None
         if isinstance(lease_token, bytes):
@@ -912,6 +953,23 @@ class RedisGateway(AbstractRedisGateway):
     def _claim_result_ttl_ms(self) -> str:
         return str(max(self._message_wait_interval_seconds, 120) * 1000)
 
+    def _cleanup_drained_lease_token_counter(self, processing_queue: str) -> bool:
+        if self._message_visibility_timeout_seconds is None:
+            return False
+        return bool(
+            _coerce_lua_count(
+                self._eval(
+                    CLEANUP_DRAINED_LEASE_TOKEN_COUNTER_LUA_SCRIPT,
+                    5,
+                    processing_queue,
+                    self._lease_deadlines_key(processing_queue),
+                    self._lease_tokens_key(processing_queue),
+                    self._delivery_counts_key(processing_queue),
+                    self._lease_token_counter_key(processing_queue),
+                )
+            )
+        )
+
     def _delete_claim_result_key(
         self,
         claim_result_key: str,

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/_stored_message.py

@@ -2,9 +2,12 @@ import json
 import uuid
 from dataclasses import dataclass
 
+from redis_message_queue._exceptions import MalformedStoredMessageError
+
 MessageData = str | bytes
 
 _STORED_MESSAGE_PREFIX = "\x1eRMQ1:"
+_STORED_MESSAGE_PREFIX_BYTES = _STORED_MESSAGE_PREFIX.encode("utf-8")
 
 
 @dataclass(frozen=True)
@@ -50,52 +53,68 @@ def decode_stored_message(message: MessageData) -> MessageData:
     only when input came through ``encode_stored_message`` first. Built-in
     publish/consume always re-wraps so this footgun cannot fire end-to-end;
     custom gateways feeding raw input must wrap before decoding.
+
+    Raises ``MalformedStoredMessageError`` when the value starts with the RMQ
+    envelope prefix but is not a valid payload-bearing envelope.
     """
-    payload = _extract_payload(message)
-    if payload is None:
+    envelope = _decode_envelope(message)
+    if envelope is None:
         return message
+    _message_id, payload = envelope
     if isinstance(message, bytes):
         return payload.encode("utf-8")
     return payload
 
 
 def extract_stored_message_id(message: MessageData) -> str | None:
-    if isinstance(message, bytes):
-        try:
-            message = message.decode("utf-8")
-        except UnicodeDecodeError:
-            return None
-    if not message.startswith(_STORED_MESSAGE_PREFIX):
-        return None
-    try:
-        envelope = json.loads(message[len(_STORED_MESSAGE_PREFIX) :])
-    except json.JSONDecodeError:
+    """Return the RMQ envelope id, or None for values that are not RMQ envelopes.
+
+    Raises ``MalformedStoredMessageError`` when the value starts with the RMQ
+    envelope prefix but is not a valid payload-bearing envelope.
+    """
+    envelope = _decode_envelope(message)
+    if envelope is None:
         return None
-    if isinstance(envelope, dict) and isinstance(envelope.get("id"), str):
-        return envelope["id"]
-    return None
+    message_id, _payload = envelope
+    return message_id
 
 
-def _extract_payload(message: MessageData) -> str | None:
+def _decode_envelope(message: MessageData) -> tuple[str, str] | None:
     if isinstance(message, bytes):
+        if not message.startswith(_STORED_MESSAGE_PREFIX_BYTES):
+            return None
         try:
             message = message.decode("utf-8")
-        except UnicodeDecodeError:
-            return None
-
-    if not message.startswith(_STORED_MESSAGE_PREFIX):
+        except UnicodeDecodeError as exc:
+            raise MalformedStoredMessageError(
+                "Stored message starts with the RMQ envelope prefix but is not valid UTF-8"
+            ) from exc
+    elif not message.startswith(_STORED_MESSAGE_PREFIX):
         return None
 
+    envelope_body = message[len(_STORED_MESSAGE_PREFIX) :]
+
     try:
-        envelope = json.loads(message[len(_STORED_MESSAGE_PREFIX) :])
-    except json.JSONDecodeError:
-        return None
+        envelope = json.loads(envelope_body)
+    except json.JSONDecodeError as exc:
+        raise MalformedStoredMessageError(
+            "Stored message starts with the RMQ envelope prefix but does not contain valid JSON"
+        ) from exc
 
     if not isinstance(envelope, dict):
-        return None
-
-    payload = envelope.get("payload")
-    envelope_id = envelope.get("id")
-    if not isinstance(payload, str) or not isinstance(envelope_id, str):
-        return None
-    return payload
+        raise MalformedStoredMessageError(
+            "Stored message starts with the RMQ envelope prefix but does not contain a JSON object"
+        )
+
+    if "id" not in envelope:
+        raise MalformedStoredMessageError("Stored RMQ envelope is missing required 'id' field")
+    if "payload" not in envelope:
+        raise MalformedStoredMessageError("Stored RMQ envelope is missing required 'payload' field")
+
+    envelope_id = envelope["id"]
+    payload = envelope["payload"]
+    if not isinstance(envelope_id, str):
+        raise MalformedStoredMessageError("Stored RMQ envelope 'id' field must be a string")
+    if not isinstance(payload, str):
+        raise MalformedStoredMessageError("Stored RMQ envelope 'payload' field must be a string")
+    return envelope_id, payload

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/asyncio/__init__.py

@@ -4,6 +4,7 @@ from redis_message_queue._exceptions import (
     ConfigurationError,
     GatewayContractError,
     LuaScriptError,
+    MalformedStoredMessageError,
     QueueBackpressureError,
     QueueDrainedError,
     RedisMessageQueueError,
@@ -30,6 +31,7 @@ __all__ = [
     "ConfigurationError",
     "GatewayContractError",
     "LuaScriptError",
+    "MalformedStoredMessageError",
     "QueueBackpressureError",
     "QueueDrainedError",
     "CleanupFailedError",

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/asyncio/_redis_gateway.py

@@ -14,6 +14,7 @@ from redis_message_queue._config import (
     ADD_MESSAGE_LUA_SCRIPT,
     CLAIM_MESSAGE_LUA_SCRIPT,
     CLAIM_MESSAGE_WITH_VISIBILITY_TIMEOUT_LUA_SCRIPT,
+    CLEANUP_DRAINED_LEASE_TOKEN_COUNTER_LUA_SCRIPT,
     DEFAULT_MESSAGE_DEDUPLICATION_LOG_TTL,
     DEFAULT_MESSAGE_WAIT_INTERVAL_SECONDS,
     DEFAULT_PENDING_OVERLOAD_BLOCK_TIMEOUT_SECONDS,
@@ -55,6 +56,7 @@ from redis_message_queue.interrupt_handler._interface import (
 logger = logging.getLogger(__name__)
 _TClaim = TypeVar("_TClaim", bound=ClaimedMessage | MessageData)
 _TRedisCall = TypeVar("_TRedisCall")
+_MessageAttemptEvent = tuple[str | None, int]
 
 _LEASE_DEADLINES_SUFFIX = ":lease_deadlines"
 _LEASE_TOKENS_SUFFIX = ":lease_tokens"
@@ -108,6 +110,26 @@ def _coerce_lua_count(value: object) -> int:
         return 0
 
 
+def _decode_lua_text(value: object) -> str | None:
+    if isinstance(value, bytes):
+        value = value.decode("utf-8")
+    if isinstance(value, str) and value:
+        return value
+    return None
+
+
+def _coerce_lua_message_attempts(value: object) -> list[_MessageAttemptEvent]:
+    if not isinstance(value, list | tuple):
+        return []
+
+    attempts: list[_MessageAttemptEvent] = []
+    for item in value:
+        if not isinstance(item, list | tuple) or len(item) < 2:
+            continue
+        attempts.append((_decode_lua_text(item[0]), _coerce_lua_count(item[1])))
+    return attempts
+
+
 def _pending_overload_max_backoff_seconds(block_timeout_seconds: float) -> float:
     return min(_PENDING_OVERLOAD_MAX_BACKOFF_SECONDS, block_timeout_seconds / 10)
 
@@ -233,8 +255,11 @@ class RedisGateway(AbstractRedisGateway):
         operation: EventOperation,
         outcome: EventOutcome,
         *,
+        message_id: str | None = None,
         claim_id: str | None = None,
         destination_queue: str | None = None,
+        delivery_count: int | None = None,
+        max_delivery_count: int | None = None,
         exception_type: str | None = None,
         error: BaseException | None = None,
     ) -> None:
@@ -243,8 +268,11 @@ class RedisGateway(AbstractRedisGateway):
         await self._event_emitter(
             operation,
             outcome,
+            message_id=message_id,
             claim_id=claim_id,
             destination_queue=destination_queue,
+            delivery_count=delivery_count,
+            max_delivery_count=max_delivery_count,
             exception_type=exception_type,
             error=error,
         )
@@ -252,12 +280,20 @@ class RedisGateway(AbstractRedisGateway):
     async def _emit_repeated_event(
         self,
         operation: EventOperation,
-        count: int,
+        attempts: list[_MessageAttemptEvent],
         *,
         destination_queue: str | None = None,
+        max_delivery_count: int | None = None,
     ) -> None:
-        for _ in range(max(count, 0)):
-            await self._emit_event(operation, "success", destination_queue=destination_queue)
+        for message_id, delivery_count in attempts:
+            await self._emit_event(
+                operation,
+                "success",
+                message_id=message_id,
+                destination_queue=destination_queue,
+                delivery_count=delivery_count,
+                max_delivery_count=max_delivery_count,
+            )
 
     async def _eval(self, *args: object) -> object:
         try:
@@ -819,10 +855,15 @@ class RedisGateway(AbstractRedisGateway):
             return None
 
         stored_message, lease_token = result[0], result[1]
-        reclaimed_count = _coerce_lua_count(result[2]) if len(result) > 2 else 0
-        dead_lettered_count = _coerce_lua_count(result[3]) if len(result) > 3 else 0
-        await self._emit_repeated_event("claim_reclaim", reclaimed_count)
-        await self._emit_repeated_event("dlq", dead_lettered_count, destination_queue=self._dead_letter_queue)
+        reclaimed_attempts = _coerce_lua_message_attempts(result[2]) if len(result) > 2 else []
+        dead_lettered_attempts = _coerce_lua_message_attempts(result[3]) if len(result) > 3 else []
+        await self._emit_repeated_event("claim_reclaim", reclaimed_attempts)
+        await self._emit_repeated_event(
+            "dlq",
+            dead_lettered_attempts,
+            destination_queue=self._dead_letter_queue,
+            max_delivery_count=self._max_delivery_count,
+        )
         if stored_message in ("", b"") and lease_token in ("", b""):
             return None
         if isinstance(lease_token, bytes):
@@ -892,6 +933,23 @@ class RedisGateway(AbstractRedisGateway):
     def _claim_result_ttl_ms(self) -> str:
         return str(max(self._message_wait_interval_seconds, 120) * 1000)
 
+    async def _cleanup_drained_lease_token_counter(self, processing_queue: str) -> bool:
+        if self._message_visibility_timeout_seconds is None:
+            return False
+        return bool(
+            _coerce_lua_count(
+                await self._eval(
+                    CLEANUP_DRAINED_LEASE_TOKEN_COUNTER_LUA_SCRIPT,
+                    5,
+                    processing_queue,
+                    self._lease_deadlines_key(processing_queue),
+                    self._lease_tokens_key(processing_queue),
+                    self._delivery_counts_key(processing_queue),
+                    self._lease_token_counter_key(processing_queue),
+                )
+            )
+        )
+
     async def _delete_claim_result_key(
         self,
         claim_result_key: str,

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/asyncio/redis_message_queue.py

@@ -23,6 +23,7 @@ from redis_message_queue._exceptions import (
     CleanupFailedError,
     ConfigurationError,
     GatewayContractError,
+    MalformedStoredMessageError,
     QueueDrainedError,
 )
 from redis_message_queue._queue_key_manager import QueueKeyManager, validate_callable_deduplication_key
@@ -750,6 +751,8 @@ class RedisMessageQueue:
         claim_id: str | None = None,
         lease_token_hash: str | None = None,
         destination_queue: str | None = None,
+        delivery_count: int | None = None,
+        max_delivery_count: int | None = None,
         exception_type: str | None = None,
         error: BaseException | None = None,
         duration_ms: float | None = None,
@@ -764,6 +767,8 @@ class RedisMessageQueue:
             claim_id=claim_id,
             lease_token_hash=lease_token_hash,
             destination_queue=destination_queue,
+            delivery_count=delivery_count,
+            max_delivery_count=max_delivery_count,
             exception_type=exception_type,
             error=error,
             duration_ms=duration_ms,
@@ -801,48 +806,40 @@ class RedisMessageQueue:
             return await self._publish_unlocked(message)
 
     async def _publish_unlocked(self, message: str | dict) -> bool:
-        if not isinstance(message, (str, dict)):
-            raise TypeError(f"'message' must be a str or dict, got {type(message).__name__}")
-        if isinstance(message, dict):
-            non_str_keys = _find_non_string_dict_keys(message)
-            if non_str_keys:
-                raise TypeError(
-                    "'message' dict keys must all be strings; "
-                    f"got non-string keys: {non_str_keys[:3]}" + (" (and more)" if len(non_str_keys) > 3 else "")
-                )
-            message_str = json.dumps(message, sort_keys=True, allow_nan=False)
-        else:
-            message_str = message
-
         started_at = time.perf_counter()
-        if not self._deduplication:
-            try:
-                result = await self._redis.add_message(self.key.pending, message_str)
-            except Exception as exc:
-                await self._emit_event(
-                    "publish",
-                    "failure",
-                    exception_type=type(exc).__name__,
-                    error=exc,
-                    duration_ms=_duration_ms(started_at),
-                )
-                raise
-            if result is not None:
-                raise GatewayContractError(
-                    f"gateway.add_message() must return None, got {type(result).__name__}. "
-                    "See AbstractRedisGateway.add_message for the full contract."
-                )
-            await self._emit_event("publish", "success", duration_ms=_duration_ms(started_at))
-            return True
-
-        dedup_key = self._get_deduplication_key(message)
-        if inspect.isawaitable(dedup_key):
-            dedup_key = await dedup_key
-        dedup_key = validate_callable_deduplication_key(dedup_key, message)
-        dedup_key = self.key.deduplication(dedup_key)
-
         try:
-            result = await self._redis.publish_message(self.key.pending, message_str, dedup_key)
+            if not isinstance(message, (str, dict)):
+                raise TypeError(f"'message' must be a str or dict, got {type(message).__name__}")
+            if isinstance(message, dict):
+                non_str_keys = _find_non_string_dict_keys(message)
+                if non_str_keys:
+                    raise TypeError(
+                        "'message' dict keys must all be strings; "
+                        f"got non-string keys: {non_str_keys[:3]}" + (" (and more)" if len(non_str_keys) > 3 else "")
+                    )
+                message_str = json.dumps(message, sort_keys=True, allow_nan=False)
+            else:
+                message_str = message
+
+            if not self._deduplication:
+                result = await self._redis.add_message(self.key.pending, message_str)
+                if result is not None:
+                    raise GatewayContractError(
+                        f"gateway.add_message() must return None, got {type(result).__name__}. "
+                        "See AbstractRedisGateway.add_message for the full contract."
+                    )
+            else:
+                dedup_key = self._get_deduplication_key(message)
+                if inspect.isawaitable(dedup_key):
+                    dedup_key = await dedup_key
+                dedup_key = validate_callable_deduplication_key(dedup_key, message)
+                dedup_key = self.key.deduplication(dedup_key)
+                result = await self._redis.publish_message(self.key.pending, message_str, dedup_key)
+                if not isinstance(result, bool):
+                    raise GatewayContractError(
+                        f"gateway.publish_message() must return bool, got {type(result).__name__}. "
+                        "See AbstractRedisGateway.publish_message for the full contract."
+                    )
         except Exception as exc:
             await self._emit_event(
                 "publish",
@@ -852,11 +849,10 @@ class RedisMessageQueue:
                 duration_ms=_duration_ms(started_at),
             )
             raise
-        if not isinstance(result, bool):
-            raise GatewayContractError(
-                f"gateway.publish_message() must return bool, got {type(result).__name__}. "
-                "See AbstractRedisGateway.publish_message for the full contract."
-            )
+
+        if not self._deduplication:
+            await self._emit_event("publish", "success", duration_ms=_duration_ms(started_at))
+            return True
         await self._emit_event(
             "publish" if result else "publish_dedup_hit",
             "success" if result else "skipped",
@@ -878,6 +874,38 @@ class RedisMessageQueue:
             return
         try:
             claimed_message = await self._wait_for_message_and_move()
+            if claimed_message is not None:
+                if not isinstance(claimed_message, (ClaimedMessage, str, bytes)):
+                    raise GatewayContractError(
+                        f"gateway.wait_for_message_and_move() must return ClaimedMessage, str, bytes, or None; "
+                        f"got {type(claimed_message).__name__}. "
+                        "See AbstractRedisGateway.wait_for_message_and_move for the full contract."
+                    )
+
+                stored_message = claimed_message
+                lease_token = None
+                if isinstance(claimed_message, ClaimedMessage):
+                    stored_message = claimed_message.stored_message
+                    lease_token = claimed_message.lease_token
+                lease_token_hash = _hash_lease_token(lease_token)
+
+                if lease_token is None and self._heartbeat_interval_seconds is not None:
+                    if not self._warned_no_lease_for_heartbeat:
+                        self._warned_no_lease_for_heartbeat = True
+                        no_lease_token_warning = (
+                            "Heartbeat is configured but the gateway returned no lease token. "
+                            "The heartbeat will not run for this message. Ensure the gateway "
+                            "returns ClaimedMessage from wait_for_message_and_move() when "
+                            "visibility timeouts are in use."
+                        )
+                        logger.warning(no_lease_token_warning)
+                        warnings.warn(no_lease_token_warning, RuntimeWarning, stacklevel=2)
+
+                if lease_token is None and self._requires_claimed_message:
+                    raise GatewayContractError(
+                        "gateways with visibility timeouts must return ClaimedMessage from "
+                        "wait_for_message_and_move(); got plain MessageData without a lease token"
+                    )
         except Exception as exc:
             await self._emit_event(
                 "claim",
@@ -892,20 +920,21 @@ class RedisMessageQueue:
             yield None
             return
 
-        if not isinstance(claimed_message, (ClaimedMessage, str, bytes)):
-            raise GatewayContractError(
-                f"gateway.wait_for_message_and_move() must return ClaimedMessage, str, bytes, or None; "
-                f"got {type(claimed_message).__name__}. "
-                "See AbstractRedisGateway.wait_for_message_and_move for the full contract."
+        message_id = None
+        try:
+            message_id = extract_stored_message_id(stored_message)
+            message = decode_stored_message(stored_message)
+        except MalformedStoredMessageError as exc:
+            await self._emit_event(
+                "claim",
+                "failure",
+                message_id=message_id,
+                lease_token_hash=lease_token_hash,
+                exception_type=type(exc).__name__,
+                error=exc,
+                duration_ms=_duration_ms(claim_started_at),
             )
-
-        stored_message = claimed_message
-        lease_token = None
-        if isinstance(claimed_message, ClaimedMessage):
-            stored_message = claimed_message.stored_message
-            lease_token = claimed_message.lease_token
-        message_id = extract_stored_message_id(stored_message)
-        lease_token_hash = _hash_lease_token(lease_token)
+            raise
         await self._emit_event(
             "claim",
             "success",
@@ -914,25 +943,6 @@ class RedisMessageQueue:
             duration_ms=_duration_ms(claim_started_at),
         )
 
-        if lease_token is None and self._heartbeat_interval_seconds is not None:
-            if not self._warned_no_lease_for_heartbeat:
-                self._warned_no_lease_for_heartbeat = True
-                no_lease_token_warning = (
-                    "Heartbeat is configured but the gateway returned no lease token. "
-                    "The heartbeat will not run for this message. Ensure the gateway "
-                    "returns ClaimedMessage from wait_for_message_and_move() when "
-                    "visibility timeouts are in use."
-                )
-                logger.warning(no_lease_token_warning)
-                warnings.warn(no_lease_token_warning, RuntimeWarning, stacklevel=2)
-
-        if lease_token is None and self._requires_claimed_message:
-            raise GatewayContractError(
-                "gateways with visibility timeouts must return ClaimedMessage from "
-                "wait_for_message_and_move(); got plain MessageData without a lease token"
-            )
-
-        message = decode_stored_message(stored_message)
         lease_heartbeat = self._build_lease_heartbeat(stored_message, lease_token, message_id, lease_token_hash)
         if lease_heartbeat is not None:
             lease_heartbeat.start()
@@ -1153,7 +1163,10 @@ class RedisMessageQueue:
         if timeout is not None and timeout < 0:
             raise ConfigurationError(f"'timeout' must be non-negative when provided, got {timeout}")
         async with self._aclose_lock:
+            cleanup_lease_counter = getattr(self._redis, "_cleanup_drained_lease_token_counter", None)
             if self._aclose_result is not None:
+                if cleanup_lease_counter is not None:
+                    await _await_preserving_cancellation(cleanup_lease_counter(self.key.processing))
                 return self._aclose_result
 
             async with self._publish_lock:
@@ -1161,6 +1174,8 @@ class RedisMessageQueue:
                 self._drained = True
             drainer = getattr(self._redis, "_drain_pending_claim_ids", None)
             if drainer is None:
+                if cleanup_lease_counter is not None:
+                    await _await_preserving_cancellation(cleanup_lease_counter(self.key.processing))
                 self._aclose_result = True
                 return self._aclose_result
             loop = asyncio.get_running_loop()
@@ -1169,6 +1184,8 @@ class RedisMessageQueue:
                 drainer(self.key.processing, deadline_monotonic=deadline_monotonic)
             )
             if result is True:
+                if cleanup_lease_counter is not None:
+                    await _await_preserving_cancellation(cleanup_lease_counter(self.key.processing))
                 self._aclose_result = True
             else:
                 self._aclose_result = None

{redis_message_queue-8.0.1 → redis_message_queue-8.0.2}/redis_message_queue/redis_message_queue.py

@@ -24,6 +24,7 @@ from redis_message_queue._exceptions import (
     CleanupFailedError,
     ConfigurationError,
     GatewayContractError,
+    MalformedStoredMessageError,
     QueueDrainedError,
 )
 from redis_message_queue._queue_key_manager import QueueKeyManager, validate_callable_deduplication_key
@@ -703,6 +704,8 @@ class RedisMessageQueue:
         claim_id: str | None = None,
         lease_token_hash: str | None = None,
         destination_queue: str | None = None,
+        delivery_count: int | None = None,
+        max_delivery_count: int | None = None,
         exception_type: str | None = None,
         error: BaseException | None = None,
         duration_ms: float | None = None,
@@ -717,6 +720,8 @@ class RedisMessageQueue:
             claim_id=claim_id,
             lease_token_hash=lease_token_hash,
             destination_queue=destination_queue,
+            delivery_count=delivery_count,
+            max_delivery_count=max_delivery_count,
             exception_type=exception_type,
             error=error,
             duration_ms=duration_ms,
@@ -754,56 +759,50 @@ class RedisMessageQueue:
             return self._publish_unlocked(message)
 
     def _publish_unlocked(self, message: str | dict) -> bool:
-        if not isinstance(message, (str, dict)):
-            raise TypeError(f"'message' must be a str or dict, got {type(message).__name__}")
-        if isinstance(message, dict):
-            non_str_keys = _find_non_string_dict_keys(message)
-            if non_str_keys:
-                raise TypeError(
-                    "'message' dict keys must all be strings; "
-                    f"got non-string keys: {non_str_keys[:3]}" + (" (and more)" if len(non_str_keys) > 3 else "")
-                )
-            message_str = json.dumps(message, sort_keys=True, allow_nan=False)
-        else:
-            message_str = message
-
         started_at = time.perf_counter()
-        if not self._deduplication:
-            try:
-                result = self._redis.add_message(self.key.pending, message_str)
-            except Exception as exc:
-                self._emit_event(
-                    "publish",
-                    "failure",
-                    exception_type=type(exc).__name__,
-                    error=exc,
-                    duration_ms=_duration_ms(started_at),
-                )
-                raise
-            if result is not None:
-                raise GatewayContractError(
-                    f"gateway.add_message() must return None, got {type(result).__name__}. "
-                    "See AbstractRedisGateway.add_message for the full contract."
-                )
-            self._emit_event("publish", "success", duration_ms=_duration_ms(started_at))
-            return True
-
-        dedup_key = self._get_deduplication_key(message)
-        if inspect.isawaitable(dedup_key):
-            is_coroutine = inspect.iscoroutine(dedup_key)
-            _close_or_cancel_awaitable(dedup_key)
-            if is_coroutine:
-                raise TypeError(
-                    "'get_deduplication_key' returned a coroutine; use the async RedisMessageQueue for async callables"
-                )
-            raise TypeError(
-                "'get_deduplication_key' returned an awaitable; use the async RedisMessageQueue for async callables"
-            )
-        dedup_key = validate_callable_deduplication_key(dedup_key, message)
-        dedup_key = self.key.deduplication(dedup_key)
-
         try:
-            result = self._redis.publish_message(self.key.pending, message_str, dedup_key)
+            if not isinstance(message, (str, dict)):
+                raise TypeError(f"'message' must be a str or dict, got {type(message).__name__}")
+            if isinstance(message, dict):
+                non_str_keys = _find_non_string_dict_keys(message)
+                if non_str_keys:
+                    raise TypeError(
+                        "'message' dict keys must all be strings; "
+                        f"got non-string keys: {non_str_keys[:3]}" + (" (and more)" if len(non_str_keys) > 3 else "")
+                    )
+                message_str = json.dumps(message, sort_keys=True, allow_nan=False)
+            else:
+                message_str = message
+
+            if not self._deduplication:
+                result = self._redis.add_message(self.key.pending, message_str)
+                if result is not None:
+                    raise GatewayContractError(
+                        f"gateway.add_message() must return None, got {type(result).__name__}. "
+                        "See AbstractRedisGateway.add_message for the full contract."
+                    )
+            else:
+                dedup_key = self._get_deduplication_key(message)
+                if inspect.isawaitable(dedup_key):
+                    is_coroutine = inspect.iscoroutine(dedup_key)
+                    _close_or_cancel_awaitable(dedup_key)
+                    if is_coroutine:
+                        raise TypeError(
+                            "'get_deduplication_key' returned a coroutine; "
+                            "use the async RedisMessageQueue for async callables"
+                        )
+                    raise TypeError(
+                        "'get_deduplication_key' returned an awaitable; "
+                        "use the async RedisMessageQueue for async callables"
+                    )
+                dedup_key = validate_callable_deduplication_key(dedup_key, message)
+                dedup_key = self.key.deduplication(dedup_key)
+                result = self._redis.publish_message(self.key.pending, message_str, dedup_key)
+                if not isinstance(result, bool):
+                    raise GatewayContractError(
+                        f"gateway.publish_message() must return bool, got {type(result).__name__}. "
+                        "See AbstractRedisGateway.publish_message for the full contract."
+                    )
         except Exception as exc:
             self._emit_event(
                 "publish",
@@ -813,11 +812,10 @@ class RedisMessageQueue:
                 duration_ms=_duration_ms(started_at),
             )
             raise
-        if not isinstance(result, bool):
-            raise GatewayContractError(
-                f"gateway.publish_message() must return bool, got {type(result).__name__}. "
-                "See AbstractRedisGateway.publish_message for the full contract."
-            )
+
+        if not self._deduplication:
+            self._emit_event("publish", "success", duration_ms=_duration_ms(started_at))
+            return True
         self._emit_event(
             "publish" if result else "publish_dedup_hit",
             "success" if result else "skipped",
@@ -839,6 +837,38 @@ class RedisMessageQueue:
             return
         try:
             claimed_message = self._wait_for_message_and_move()
+            if claimed_message is not None:
+                if not isinstance(claimed_message, (ClaimedMessage, str, bytes)):
+                    raise GatewayContractError(
+                        f"gateway.wait_for_message_and_move() must return ClaimedMessage, str, bytes, or None; "
+                        f"got {type(claimed_message).__name__}. "
+                        "See AbstractRedisGateway.wait_for_message_and_move for the full contract."
+                    )
+
+                stored_message = claimed_message
+                lease_token = None
+                if isinstance(claimed_message, ClaimedMessage):
+                    stored_message = claimed_message.stored_message
+                    lease_token = claimed_message.lease_token
+                lease_token_hash = _hash_lease_token(lease_token)
+
+                if lease_token is None and self._heartbeat_interval_seconds is not None:
+                    if not self._warned_no_lease_for_heartbeat:
+                        self._warned_no_lease_for_heartbeat = True
+                        no_lease_token_warning = (
+                            "Heartbeat is configured but the gateway returned no lease token. "
+                            "The heartbeat will not run for this message. Ensure the gateway "
+                            "returns ClaimedMessage from wait_for_message_and_move() when "
+                            "visibility timeouts are in use."
+                        )
+                        logger.warning(no_lease_token_warning)
+                        warnings.warn(no_lease_token_warning, RuntimeWarning, stacklevel=2)
+
+                if lease_token is None and self._requires_claimed_message:
+                    raise GatewayContractError(
+                        "gateways with visibility timeouts must return ClaimedMessage from "
+                        "wait_for_message_and_move(); got plain MessageData without a lease token"
+                    )
         except Exception as exc:
             self._emit_event(
                 "claim",
@@ -853,20 +883,21 @@ class RedisMessageQueue:
             yield None
             return
 
-        if not isinstance(claimed_message, (ClaimedMessage, str, bytes)):
-            raise GatewayContractError(
-                f"gateway.wait_for_message_and_move() must return ClaimedMessage, str, bytes, or None; "
-                f"got {type(claimed_message).__name__}. "
-                "See AbstractRedisGateway.wait_for_message_and_move for the full contract."
+        message_id = None
+        try:
+            message_id = extract_stored_message_id(stored_message)
+            message = decode_stored_message(stored_message)
+        except MalformedStoredMessageError as exc:
+            self._emit_event(
+                "claim",
+                "failure",
+                message_id=message_id,
+                lease_token_hash=lease_token_hash,
+                exception_type=type(exc).__name__,
+                error=exc,
+                duration_ms=_duration_ms(claim_started_at),
             )
-
-        stored_message = claimed_message
-        lease_token = None
-        if isinstance(claimed_message, ClaimedMessage):
-            stored_message = claimed_message.stored_message
-            lease_token = claimed_message.lease_token
-        message_id = extract_stored_message_id(stored_message)
-        lease_token_hash = _hash_lease_token(lease_token)
+            raise
         self._emit_event(
             "claim",
             "success",
@@ -875,25 +906,6 @@ class RedisMessageQueue:
             duration_ms=_duration_ms(claim_started_at),
         )
 
-        if lease_token is None and self._heartbeat_interval_seconds is not None:
-            if not self._warned_no_lease_for_heartbeat:
-                self._warned_no_lease_for_heartbeat = True
-                no_lease_token_warning = (
-                    "Heartbeat is configured but the gateway returned no lease token. "
-                    "The heartbeat will not run for this message. Ensure the gateway "
-                    "returns ClaimedMessage from wait_for_message_and_move() when "
-                    "visibility timeouts are in use."
-                )
-                logger.warning(no_lease_token_warning)
-                warnings.warn(no_lease_token_warning, RuntimeWarning, stacklevel=2)
-
-        if lease_token is None and self._requires_claimed_message:
-            raise GatewayContractError(
-                "gateways with visibility timeouts must return ClaimedMessage from "
-                "wait_for_message_and_move(); got plain MessageData without a lease token"
-            )
-
-        message = decode_stored_message(stored_message)
         lease_heartbeat = self._build_lease_heartbeat(stored_message, lease_token, message_id, lease_token_hash)
         if lease_heartbeat is not None:
             lease_heartbeat.start()
@@ -1110,7 +1122,10 @@ class RedisMessageQueue:
         if timeout is not None and timeout < 0:
             raise ConfigurationError(f"'timeout' must be non-negative when provided, got {timeout}")
         with self._drain_lock:
+            cleanup_lease_counter = getattr(self._redis, "_cleanup_drained_lease_token_counter", None)
             if self._drain_result is True:
+                if cleanup_lease_counter is not None:
+                    cleanup_lease_counter(self.key.processing)
                 return True
 
             with self._publish_lock:
@@ -1118,11 +1133,15 @@ class RedisMessageQueue:
                 self._drained.set()
             drainer = getattr(self._redis, "_drain_pending_claim_ids", None)
             if drainer is None:
+                if cleanup_lease_counter is not None:
+                    cleanup_lease_counter(self.key.processing)
                 self._drain_result = True
                 return True
             deadline_monotonic = None if timeout is None else (time.monotonic() + float(timeout))
             result = drainer(self.key.processing, deadline_monotonic=deadline_monotonic)
             if result is True:
+                if cleanup_lease_counter is not None:
+                    cleanup_lease_counter(self.key.processing)
                 self._drain_result = True
             else:
                 self._drain_result = None