redis-message-queue 8.2.2__tar.gz → 8.2.3__tar.gz

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: redis-message-queue
-Version: 8.2.2
+Version: 8.2.3
 Summary: Python message queuing with Redis and message deduplication
 Project-URL: Homepage, https://github.com/Elijas/redis-message-queue
 Project-URL: Repository, https://github.com/Elijas/redis-message-queue
@@ -34,7 +34,7 @@ Description-Content-Type: text/markdown
 **Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
 
 ```bash
-pip install "redis-message-queue>=8.2.2,<9.0.0"
+pip install "redis-message-queue>=8.2.3,<9.0.0"
 ```
 
 Requires Redis server >= 6.2.

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/README.md

@@ -11,7 +11,7 @@
 **Lightweight Python message queuing with Redis and built-in publish-side deduplication.** Deduplicate publishes within a TTL window, with optional crash recovery — across any number of producers and consumers.
 
 ```bash
-pip install "redis-message-queue>=8.2.2,<9.0.0"
+pip install "redis-message-queue>=8.2.3,<9.0.0"
 ```
 
 Requires Redis server >= 6.2.

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "redis-message-queue"
-version = "8.2.2"
+version = "8.2.3"
 description = "Python message queuing with Redis and message deduplication"
 authors = [{ name = "Elijas", email = "4084885+Elijas@users.noreply.github.com" }]
 readme = "README.md"
@@ -48,7 +48,7 @@ default-groups = ["dev", "test"]
 ##############################
 
 [tool.bumpversion]
-current_version = "8.2.2"
+current_version = "8.2.3"
 parse = "(?P<major>\\d+)\\.(?P<minor>\\d+)\\.(?P<patch>\\d+)"
 serialize = ["{major}.{minor}.{patch}"]
 search = "{current_version}"

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/redis_message_queue/_config.py

@@ -585,6 +585,53 @@ return removed
 """
 )
 
+RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT = (
+    _LUA_KEY_TYPE_GUARD
+    + """
+local err = redis_message_queue_require_type(KEYS[1], 'list')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[2], 'list')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[3], 'hash')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[4], 'hash')
+if err then
+    return err
+end
+
+local err = redis_message_queue_require_type(KEYS[5], 'string')
+if err then
+    return err
+end
+
+local cached_result = redis.call('GET', KEYS[5])
+if cached_result then
+    return tonumber(cached_result)
+end
+
+redis.call('RPUSH', KEYS[2], ARGV[1])
+local removed = redis.call('LREM', KEYS[1], 1, ARGV[1])
+if removed == 1 then
+    redis.call('HDEL', KEYS[3], ARGV[2])
+    redis.call('HDEL', KEYS[4], ARGV[1])
+else
+    redis.call('LREM', KEYS[2], 1, ARGV[1])
+end
+
+redis.call('SET', KEYS[5], tostring(removed), 'PX', tonumber(ARGV[3]))
+return removed
+"""
+)
+
 REMOVE_MESSAGE_LUA_SCRIPT = (
     _LUA_KEY_TYPE_GUARD
     + """
@@ -876,8 +923,6 @@ while claim_attempts < 100 do
 
     local count = redis.call('HINCRBY', KEYS[6], stored, 1)
     if max_delivery_count > 0 and count > max_delivery_count then
-        redis.call('LREM', KEYS[2], 1, stored)
-        redis.call('HDEL', KEYS[6], stored)
         -- Strip envelope to store raw payload in DLQ, consistent with completed/failed queues.
         -- The per-delivery UUID in the envelope is lost; see README dead-letter notes.
         local dead_letter_value = stored
@@ -886,6 +931,8 @@ while claim_attempts < 100 do
             dead_letter_value = envelope['payload']
         end
         redis.call('LPUSH', KEYS[7], dead_letter_value)
+        redis.call('LREM', KEYS[2], 1, stored)
+        redis.call('HDEL', KEYS[6], stored)
         table.insert(dead_lettered_events, {redis_message_queue_message_id(stored), tostring(count)})
     else
         return store_claim_and_return(stored)

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/redis_message_queue/_redis_gateway.py

@@ -31,6 +31,7 @@ from redis_message_queue._config import (
     REMOVE_MESSAGE_LUA_SCRIPT,
     REMOVE_MESSAGE_WITH_LEASE_TOKEN_LUA_SCRIPT,
     RENEW_MESSAGE_LEASE_LUA_SCRIPT,
+    RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
     _ChainedInterrupt,
     build_retry_strategy,
     is_redis_retryable_exception,
@@ -79,6 +80,8 @@ _VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS = 0.25
 _PENDING_OVERLOAD_INITIAL_BACKOFF_SECONDS = 0.010
 _PENDING_OVERLOAD_MAX_BACKOFF_SECONDS = 0.500
 _CLAIM_STORE_FAILED_LUA_SENTINEL_BYTES = CLAIM_STORE_FAILED_LUA_SENTINEL.encode("utf-8")
+_PENDING_QUEUE_SUFFIX = "pending"
+_PROCESSING_QUEUE_SUFFIX = "processing"
 
 
 class _DrainDeadlineExceeded(Exception):
@@ -1225,6 +1228,53 @@ class RedisGateway(AbstractRedisGateway):
         _raise_if_drain_deadline_expired(deadline_monotonic)
         return ClaimedMessage(stored_message=stored_message, lease_token=lease_token)
 
+    def _pending_queue_from_processing_queue(self, processing_queue: str) -> str:
+        if not processing_queue.endswith(_PROCESSING_QUEUE_SUFFIX):
+            raise RuntimeError(f"cannot derive pending queue key from processing queue {processing_queue!r}")
+        return f"{processing_queue.removesuffix(_PROCESSING_QUEUE_SUFFIX)}{_PENDING_QUEUE_SUFFIX}"
+
+    def _return_recovered_non_visibility_timeout_claim_to_pending(
+        self,
+        processing_queue: str,
+        stored_message: MessageData,
+        claim_id: str,
+        *,
+        deadline_monotonic: float | None,
+    ) -> bool:
+        pending_queue = self._pending_queue_from_processing_queue(processing_queue)
+        operation_id = uuid.uuid4().hex
+        operation_result_key = self._operation_result_key(processing_queue, operation_id)
+
+        try:
+            _raise_if_drain_deadline_expired(deadline_monotonic)
+            result = _call_with_drain_deadline(
+                lambda: self._eval(
+                    RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
+                    5,
+                    processing_queue,
+                    pending_queue,
+                    self._claim_result_ids_key(processing_queue),
+                    self._claim_result_backrefs_key(processing_queue),
+                    operation_result_key,
+                    stored_message,
+                    claim_id,
+                    self._operation_result_ttl_ms(),
+                ),
+                deadline_monotonic=deadline_monotonic,
+            )
+            _raise_if_drain_deadline_expired(deadline_monotonic)
+            return bool(_coerce_lua_count(result))
+        except RedisMessageQueueError as exc:
+            _set_exception_context(
+                exc,
+                queue=processing_queue,
+                message_id=extract_stored_message_id(stored_message),
+                operation="drain",
+            )
+            raise
+        finally:
+            self._delete_operation_result_key(operation_result_key)
+
     def _is_interrupted(self, is_interrupted: BaseGracefulInterruptHandler | None = None) -> bool:
         return (self._interrupt is not None and self._interrupt.is_interrupted()) or (
             is_interrupted is not None and is_interrupted.is_interrupted()
@@ -1241,8 +1291,10 @@ class RedisGateway(AbstractRedisGateway):
         Walks the same recovery path as ``_wait_for_claim`` but without
         gating on the interrupt flag, so a soft shutdown can flush
         ambiguous-claim state that would otherwise be dropped on process
-        exit (AA-05-F2). Returns True if no pending ids remain; False if
-        the deadline fired or transient Redis errors prevented full drain.
+        exit (AA-05-F2). No-visibility-timeout recoveries are returned to
+        the pending queue before their claim ids are cleared. Returns True
+        if no pending ids remain; False if the deadline fired or Redis
+        errors prevented full drain.
         """
         with self._drain_pending_claim_ids_lock:
             self._last_drain_error = None
@@ -1258,11 +1310,8 @@ class RedisGateway(AbstractRedisGateway):
         deadline_monotonic: float | None,
     ) -> bool:
         """Recover every in-memory pending claim id for ``processing_queue``."""
-        if self._message_visibility_timeout_seconds is not None:
-            recover = self._recover_pending_visibility_timeout_claim
-        else:
-            recover = self._recover_pending_non_visibility_timeout_claim
-        skipped_transient: set[str] = set()
+        has_visibility_timeout = self._message_visibility_timeout_seconds is not None
+        skipped_unresolved: set[str] = set()
         last_error: BaseException | None = None
         while True:
             # ``>=`` (not ``>``) makes ``timeout=0`` deterministically take
@@ -1278,7 +1327,7 @@ class RedisGateway(AbstractRedisGateway):
                     return True
                 recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
                 claim_id = next(
-                    (cid for cid in pending if cid not in recovering and cid not in skipped_transient),
+                    (cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
                     None,
                 )
                 if claim_id is None:
@@ -1287,8 +1336,34 @@ class RedisGateway(AbstractRedisGateway):
             clear = False
             try:
                 try:
-                    recover(processing_queue, claim_id, deadline_monotonic=deadline_monotonic)
-                    clear = True
+                    if has_visibility_timeout:
+                        self._recover_pending_visibility_timeout_claim(
+                            processing_queue,
+                            claim_id,
+                            deadline_monotonic=deadline_monotonic,
+                        )
+                        clear = True
+                        continue
+
+                    recovered_claim = self._recover_pending_non_visibility_timeout_claim(
+                        processing_queue,
+                        claim_id,
+                        deadline_monotonic=deadline_monotonic,
+                    )
+                    if recovered_claim is None:
+                        clear = True
+                    elif self._return_recovered_non_visibility_timeout_claim_to_pending(
+                        processing_queue,
+                        recovered_claim,
+                        claim_id,
+                        deadline_monotonic=deadline_monotonic,
+                    ):
+                        clear = True
+                    else:
+                        last_error = RuntimeError(
+                            f"drain recovered claim {claim_id!r} but message was not present in processing queue"
+                        )
+                        skipped_unresolved.add(claim_id)
                 except _DrainDeadlineExceeded:
                     last_error = TimeoutError("drain pending-claim recovery deadline expired")
                     break
@@ -1301,7 +1376,7 @@ class RedisGateway(AbstractRedisGateway):
                         claim_id,
                         type(exc).__name__,
                     )
-                    skipped_transient.add(claim_id)
+                    skipped_unresolved.add(claim_id)
             finally:
                 self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
         with self._pending_claim_ids_lock:

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/redis_message_queue/asyncio/_redis_gateway.py

@@ -29,6 +29,7 @@ from redis_message_queue._config import (
     REMOVE_MESSAGE_LUA_SCRIPT,
     REMOVE_MESSAGE_WITH_LEASE_TOKEN_LUA_SCRIPT,
     RENEW_MESSAGE_LEASE_LUA_SCRIPT,
+    RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
     _ChainedInterrupt,
     build_retry_strategy,
     is_redis_retryable_exception,
@@ -78,6 +79,8 @@ _VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS = 0.25
 _PENDING_OVERLOAD_INITIAL_BACKOFF_SECONDS = 0.010
 _PENDING_OVERLOAD_MAX_BACKOFF_SECONDS = 0.500
 _CLAIM_STORE_FAILED_LUA_SENTINEL_BYTES = CLAIM_STORE_FAILED_LUA_SENTINEL.encode("utf-8")
+_PENDING_QUEUE_SUFFIX = "pending"
+_PROCESSING_QUEUE_SUFFIX = "processing"
 
 
 class _DrainDeadlineExceeded(Exception):
@@ -1206,6 +1209,53 @@ class RedisGateway(AbstractRedisGateway):
         _raise_if_drain_deadline_expired(deadline_monotonic)
         return ClaimedMessage(stored_message=stored_message, lease_token=lease_token)
 
+    def _pending_queue_from_processing_queue(self, processing_queue: str) -> str:
+        if not processing_queue.endswith(_PROCESSING_QUEUE_SUFFIX):
+            raise RuntimeError(f"cannot derive pending queue key from processing queue {processing_queue!r}")
+        return f"{processing_queue.removesuffix(_PROCESSING_QUEUE_SUFFIX)}{_PENDING_QUEUE_SUFFIX}"
+
+    async def _return_recovered_non_visibility_timeout_claim_to_pending(
+        self,
+        processing_queue: str,
+        stored_message: MessageData,
+        claim_id: str,
+        *,
+        deadline_monotonic: float | None,
+    ) -> bool:
+        pending_queue = self._pending_queue_from_processing_queue(processing_queue)
+        operation_id = uuid.uuid4().hex
+        operation_result_key = self._operation_result_key(processing_queue, operation_id)
+
+        try:
+            _raise_if_drain_deadline_expired(deadline_monotonic)
+            result = await _call_with_drain_deadline(
+                lambda: self._eval(
+                    RETURN_MESSAGE_TO_PENDING_LUA_SCRIPT,
+                    5,
+                    processing_queue,
+                    pending_queue,
+                    self._claim_result_ids_key(processing_queue),
+                    self._claim_result_backrefs_key(processing_queue),
+                    operation_result_key,
+                    stored_message,
+                    claim_id,
+                    self._operation_result_ttl_ms(),
+                ),
+                deadline_monotonic=deadline_monotonic,
+            )
+            _raise_if_drain_deadline_expired(deadline_monotonic)
+            return bool(_coerce_lua_count(result))
+        except RedisMessageQueueError as exc:
+            _set_exception_context(
+                exc,
+                queue=processing_queue,
+                message_id=extract_stored_message_id(stored_message),
+                operation="drain",
+            )
+            raise
+        finally:
+            await self._delete_operation_result_key(operation_result_key)
+
     def _is_interrupted(self, is_interrupted: BaseGracefulInterruptHandler | None = None) -> bool:
         return (self._interrupt is not None and self._interrupt.is_interrupted()) or (
             is_interrupted is not None and is_interrupted.is_interrupted()
@@ -1222,8 +1272,10 @@ class RedisGateway(AbstractRedisGateway):
         Kept separate from the sync implementation per AF11 (sync/async
         gateways stay duplicated). Walks the same recovery path as
         ``_wait_for_claim`` but without gating on the interrupt flag so a
-        soft shutdown can flush ambiguous-claim state. Returns True if no
-        pending ids remain; False on deadline expiry or transient errors.
+        soft shutdown can flush ambiguous-claim state. No-visibility-timeout
+        recoveries are returned to the pending queue before their claim ids
+        are cleared. Returns True if no pending ids remain; False on deadline
+        expiry or Redis errors.
         """
         async with self._drain_pending_claim_ids_lock:
             self._last_drain_error = None
@@ -1239,11 +1291,8 @@ class RedisGateway(AbstractRedisGateway):
         deadline_monotonic: float | None,
     ) -> bool:
         """Recover every in-memory pending claim id for ``processing_queue``."""
-        if self._message_visibility_timeout_seconds is not None:
-            recover = self._recover_pending_visibility_timeout_claim
-        else:
-            recover = self._recover_pending_non_visibility_timeout_claim
-        skipped_transient: set[str] = set()
+        has_visibility_timeout = self._message_visibility_timeout_seconds is not None
+        skipped_unresolved: set[str] = set()
         loop = asyncio.get_running_loop()
         last_error: BaseException | None = None
         while True:
@@ -1258,7 +1307,7 @@ class RedisGateway(AbstractRedisGateway):
                     return True
                 recovering = self._recovering_claim_ids.setdefault(processing_queue, set())
                 claim_id = next(
-                    (cid for cid in pending if cid not in recovering and cid not in skipped_transient),
+                    (cid for cid in pending if cid not in recovering and cid not in skipped_unresolved),
                     None,
                 )
                 if claim_id is None:
@@ -1267,8 +1316,34 @@ class RedisGateway(AbstractRedisGateway):
             clear = False
             try:
                 try:
-                    await recover(processing_queue, claim_id, deadline_monotonic=deadline_monotonic)
-                    clear = True
+                    if has_visibility_timeout:
+                        await self._recover_pending_visibility_timeout_claim(
+                            processing_queue,
+                            claim_id,
+                            deadline_monotonic=deadline_monotonic,
+                        )
+                        clear = True
+                        continue
+
+                    recovered_claim = await self._recover_pending_non_visibility_timeout_claim(
+                        processing_queue,
+                        claim_id,
+                        deadline_monotonic=deadline_monotonic,
+                    )
+                    if recovered_claim is None:
+                        clear = True
+                    elif await self._return_recovered_non_visibility_timeout_claim_to_pending(
+                        processing_queue,
+                        recovered_claim,
+                        claim_id,
+                        deadline_monotonic=deadline_monotonic,
+                    ):
+                        clear = True
+                    else:
+                        last_error = RuntimeError(
+                            f"drain recovered claim {claim_id!r} but message was not present in processing queue"
+                        )
+                        skipped_unresolved.add(claim_id)
                 except _DrainDeadlineExceeded:
                     last_error = TimeoutError("drain pending-claim recovery deadline expired")
                     break
@@ -1281,7 +1356,7 @@ class RedisGateway(AbstractRedisGateway):
                         claim_id,
                         type(exc).__name__,
                     )
-                    skipped_transient.add(claim_id)
+                    skipped_unresolved.add(claim_id)
             finally:
                 self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
         with self._pending_claim_ids_lock:

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/redis_message_queue/asyncio/redis_message_queue.py

@@ -920,6 +920,18 @@ class RedisMessageQueue:
                     "Use an async callable or return an awaitable."
                 )
             await result
+        except asyncio.CancelledError as exc:
+            current_task = asyncio.current_task()
+            if current_task is not None and current_task.cancelling() > 0:
+                raise
+            logger.exception("on_event callback raised an exception")
+            with warnings.catch_warnings():
+                warnings.simplefilter("always", RuntimeWarning)
+                warnings.warn(
+                    f"on_event callback raised {type(exc).__name__}",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
         except Exception as exc:
             logger.exception("on_event callback raised an exception")
             with warnings.catch_warnings():
@@ -1414,7 +1426,9 @@ class RedisMessageQueue:
         calls raise ``QueueDrainedError``. It then awaits the gateway's
         pending-claim-id recovery loop. Returns ``True`` if all pending claim
         ids were recovered, ``False`` if the deadline fired or a transient
-        Redis error left ids pending.
+        Redis error left ids pending. In no-visibility-timeout queues,
+        recovered messages are returned to pending before the claim id is
+        cleared.
 
         Unlike ``asyncio.CancelledError`` (hard-abort, leaves messages
         claimed for VT-reclaim), ``aclose()`` is the explicit-drain

{redis_message_queue-8.2.2 → redis_message_queue-8.2.3}/redis_message_queue/redis_message_queue.py

@@ -1,3 +1,4 @@
+import asyncio
 import hashlib
 import inspect
 import logging
@@ -883,6 +884,15 @@ class RedisMessageQueue:
                     "'on_event' returned an awaitable; use the async RedisMessageQueue "
                     "from redis_message_queue.asyncio instead"
                 )
+        except asyncio.CancelledError as exc:
+            logger.exception("on_event callback raised an exception")
+            with warnings.catch_warnings():
+                warnings.simplefilter("always", RuntimeWarning)
+                warnings.warn(
+                    f"on_event callback raised {type(exc).__name__}",
+                    RuntimeWarning,
+                    stacklevel=2,
+                )
         except Exception as exc:
             logger.exception("on_event callback raised an exception")
             with warnings.catch_warnings():
@@ -1374,7 +1384,8 @@ class RedisMessageQueue:
         calls raise ``QueueDrainedError``. It then walks the gateway's
         in-memory ``_pending_claim_ids`` to recover any ambiguous claims that
         an interrupt-aware shutdown would otherwise drop on process exit
-        (AA-05-F2).
+        (AA-05-F2). In no-visibility-timeout queues, recovered messages are
+        returned to pending before the claim id is cleared.
 
         ``timeout`` bounds the pending-claim-id recovery loop in seconds;
         ``None`` waits indefinitely, ``0`` skips the loop entirely. The