redis-message-queue 8.0.3__tar.gz → 8.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (26) hide show
  1. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/PKG-INFO +32 -1
  2. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/README.md +31 -0
  3. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/pyproject.toml +1 -1
  4. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/__init__.py +4 -0
  5. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_event.py +6 -0
  6. redis_message_queue-8.1.0/redis_message_queue/_exceptions.py +149 -0
  7. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_redis_gateway.py +81 -16
  8. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_stored_message.py +16 -7
  9. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/__init__.py +8 -1
  10. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/_redis_gateway.py +81 -16
  11. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/redis_message_queue.py +208 -8
  12. redis_message_queue-8.1.0/redis_message_queue/interrupt_handler/__init__.py +9 -0
  13. redis_message_queue-8.1.0/redis_message_queue/interrupt_handler/_event_driven.py +24 -0
  14. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/redis_message_queue.py +223 -10
  15. redis_message_queue-8.0.3/redis_message_queue/_exceptions.py +0 -71
  16. redis_message_queue-8.0.3/redis_message_queue/interrupt_handler/__init__.py +0 -4
  17. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/LICENSE +0 -0
  18. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_abstract_redis_gateway.py +0 -0
  19. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_callable_utils.py +0 -0
  20. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_config.py +0 -0
  21. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_queue_key_manager.py +0 -0
  22. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_redis_cluster.py +0 -0
  23. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/_abstract_redis_gateway.py +0 -0
  24. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/interrupt_handler/_implementation.py +0 -0
  25. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/interrupt_handler/_interface.py +0 -0
  26. {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/py.typed +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: redis-message-queue
3
- Version: 8.0.3
3
+ Version: 8.1.0
4
4
  Summary: Python message queuing with Redis and message deduplication
5
5
  License: MIT
6
6
  License-File: LICENSE
@@ -392,6 +392,32 @@ while not interrupt.is_interrupted():
392
392
  > processes, or install one top-level signal owner that calls `queue.drain()`
393
393
  > / `queue.aclose()` or sets an application stop event.
394
394
 
395
+ If another library owns SIGTERM/SIGINT in the same process, adapt its shutdown
396
+ signal to rmq with a user-owned event instead of installing rmq signal handlers:
397
+
398
+ ```python
399
+ import threading
400
+
401
+ from redis_message_queue import EventDrivenInterruptHandler, RedisMessageQueue
402
+
403
+ stop_event = threading.Event()
404
+ interrupt = EventDrivenInterruptHandler(stop_event)
405
+ queue = RedisMessageQueue("q", client=client, interrupt=interrupt)
406
+
407
+ while not interrupt.is_interrupted():
408
+ with queue.process_message() as message:
409
+ if message is not None:
410
+ process(message)
411
+
412
+ # In the sibling library's shutdown hook:
413
+ stop_event.set()
414
+ queue.drain(timeout=25)
415
+ ```
416
+
417
+ The caller MUST set `stop_event` before exiting. rmq observes
418
+ `is_interrupted()` and exits cooperatively; it does not call `sys.exit()` or
419
+ otherwise force process shutdown.
420
+
395
421
  There are three distinct shutdown shapes; pick the one that matches your runtime:
396
422
 
397
423
  | Shape | Trigger | In-flight handler | Pending claim IDs |
@@ -589,6 +615,11 @@ using `":queue:"` with a queue name that overlaps RQ keys. rmq has no fixed
589
615
  library prefix; generated keys share the Redis DB namespace with every other
590
616
  Redis user.
591
617
 
618
+ Set `strict_envelope_decoding=True` if this Redis is shared with sibling task
619
+ libraries (Celery, RQ, Dramatiq) to fail-fast on foreign payloads. With the
620
+ default `False`, non-rmq values that do not start with the rmq envelope prefix
621
+ remain backward-compatible raw messages and are yielded to the handler.
622
+
592
623
  ## Production notes
593
624
 
594
625
  ### Fork safety and pre-fork servers
@@ -366,6 +366,32 @@ while not interrupt.is_interrupted():
366
366
  > processes, or install one top-level signal owner that calls `queue.drain()`
367
367
  > / `queue.aclose()` or sets an application stop event.
368
368
 
369
+ If another library owns SIGTERM/SIGINT in the same process, adapt its shutdown
370
+ signal to rmq with a user-owned event instead of installing rmq signal handlers:
371
+
372
+ ```python
373
+ import threading
374
+
375
+ from redis_message_queue import EventDrivenInterruptHandler, RedisMessageQueue
376
+
377
+ stop_event = threading.Event()
378
+ interrupt = EventDrivenInterruptHandler(stop_event)
379
+ queue = RedisMessageQueue("q", client=client, interrupt=interrupt)
380
+
381
+ while not interrupt.is_interrupted():
382
+ with queue.process_message() as message:
383
+ if message is not None:
384
+ process(message)
385
+
386
+ # In the sibling library's shutdown hook:
387
+ stop_event.set()
388
+ queue.drain(timeout=25)
389
+ ```
390
+
391
+ The caller MUST set `stop_event` before exiting. rmq observes
392
+ `is_interrupted()` and exits cooperatively; it does not call `sys.exit()` or
393
+ otherwise force process shutdown.
394
+
369
395
  There are three distinct shutdown shapes; pick the one that matches your runtime:
370
396
 
371
397
  | Shape | Trigger | In-flight handler | Pending claim IDs |
@@ -563,6 +589,11 @@ using `":queue:"` with a queue name that overlaps RQ keys. rmq has no fixed
563
589
  library prefix; generated keys share the Redis DB namespace with every other
564
590
  Redis user.
565
591
 
592
+ Set `strict_envelope_decoding=True` if this Redis is shared with sibling task
593
+ libraries (Celery, RQ, Dramatiq) to fail-fast on foreign payloads. With the
594
+ default `False`, non-rmq values that do not start with the rmq envelope prefix
595
+ remain backward-compatible raw messages and are yielded to the handler.
596
+
566
597
  ## Production notes
567
598
 
568
599
  ### Fork safety and pre-fork servers
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "redis-message-queue"
3
- version = "8.0.3"
3
+ version = "8.1.0"
4
4
  description = "Python message queuing with Redis and message deduplication"
5
5
  authors = ["Elijas <4084885+Elijas@users.noreply.github.com>"]
6
6
  readme = "README.md"
@@ -3,6 +3,7 @@ from redis_message_queue._event import EventOperation, EventOutcome, QueueEvent
3
3
  from redis_message_queue._exceptions import (
4
4
  CleanupFailedError,
5
5
  ConfigurationError,
6
+ DrainFailedError,
6
7
  GatewayContractError,
7
8
  LuaScriptError,
8
9
  MalformedStoredMessageError,
@@ -15,6 +16,7 @@ from redis_message_queue._redis_gateway import RedisGateway
15
16
  from redis_message_queue._stored_message import ClaimedMessage, MessageData
16
17
  from redis_message_queue.interrupt_handler import (
17
18
  BaseGracefulInterruptHandler,
19
+ EventDrivenInterruptHandler,
18
20
  GracefulInterruptHandler,
19
21
  )
20
22
  from redis_message_queue.redis_message_queue import RedisMessageQueue
@@ -25,6 +27,7 @@ __all__ = [
25
27
  "AbstractRedisGateway",
26
28
  "ClaimedMessage",
27
29
  "MessageData",
30
+ "EventDrivenInterruptHandler",
28
31
  "GracefulInterruptHandler",
29
32
  "BaseGracefulInterruptHandler",
30
33
  "QueueEvent",
@@ -32,6 +35,7 @@ __all__ = [
32
35
  "EventOutcome",
33
36
  "RedisMessageQueueError",
34
37
  "ConfigurationError",
38
+ "DrainFailedError",
35
39
  "GatewayContractError",
36
40
  "LuaScriptError",
37
41
  "MalformedStoredMessageError",
@@ -24,6 +24,7 @@ class EventOperation(StrEnum):
24
24
  TRIM_FAILED = "trim_failed"
25
25
  RETRY_ATTEMPT = "retry_attempt"
26
26
  RETRY_EXHAUSTED = "retry_exhausted"
27
+ DRAIN = "drain"
27
28
 
28
29
 
29
30
  class EventOutcome(StrEnum):
@@ -32,6 +33,7 @@ class EventOutcome(StrEnum):
32
33
  SUCCESS = "success"
33
34
  FAILURE = "failure"
34
35
  SKIPPED = "skipped"
36
+ START = "start"
35
37
 
36
38
 
37
39
  @dataclass(frozen=True)
@@ -68,3 +70,7 @@ class QueueEvent:
68
70
  the actual exception object when one was raised; pass to OpenTelemetry
69
71
  `span.record_exception(...)` for full trace attribution
70
72
  """
73
+ timeout_seconds: float | None = None
74
+ """the caller-requested timeout for drain operations, when applicable"""
75
+ pending_claim_ids: int | None = None
76
+ """number of unresolved pending claim ids for drain operations, when applicable"""
@@ -0,0 +1,149 @@
1
+ import redis.exceptions
2
+
3
+
4
+ class RedisMessageQueueError(Exception):
5
+ """Base class for redis-message-queue specific errors."""
6
+
7
+ def __init__(
8
+ self,
9
+ *args: object,
10
+ queue: str | None = None,
11
+ message_id: str | None = None,
12
+ operation: str | None = None,
13
+ ) -> None:
14
+ super().__init__(*args)
15
+ self.queue = queue
16
+ self.message_id = message_id
17
+ self.operation = operation
18
+
19
+
20
+ class ConfigurationError(RedisMessageQueueError, ValueError):
21
+ """Bad parameter values or combinations."""
22
+
23
+
24
+ class GatewayContractError(RedisMessageQueueError, TypeError):
25
+ """Custom gateway returned wrong type or violated contract."""
26
+
27
+
28
+ class LuaScriptError(redis.exceptions.ResponseError, RedisMessageQueueError):
29
+ """A Lua script returned an unexpected error_reply."""
30
+
31
+ def __init__(
32
+ self,
33
+ *args: object,
34
+ queue: str | None = None,
35
+ message_id: str | None = None,
36
+ operation: str | None = None,
37
+ ) -> None:
38
+ RedisMessageQueueError.__init__(
39
+ self,
40
+ *args,
41
+ queue=queue,
42
+ message_id=message_id,
43
+ operation=operation,
44
+ )
45
+
46
+
47
+ class CleanupFailedError(RedisMessageQueueError):
48
+ """Cleanup after handler completion failed."""
49
+
50
+
51
+ class DrainFailedError(RedisMessageQueueError):
52
+ """Wraps a non-RMQ exception caught during drain pending-claim recovery.
53
+
54
+ drain() returns False as the bool result; this exception carries
55
+ F7 context (queue, operation="drain") into the drain/failure event
56
+ payload so users diagnosing drain incidents via on_event see the
57
+ same structured attrs as elsewhere.
58
+ """
59
+
60
+
61
+ class MalformedStoredMessageError(RedisMessageQueueError):
62
+ """Stored value is not a valid RMQ envelope for the configured decode mode."""
63
+
64
+
65
+ class QueueBackpressureError(RedisMessageQueueError):
66
+ """Publish rejected because the pending queue is at its configured limit."""
67
+
68
+ _REMEDIATION = (
69
+ "consider increasing `max_pending_length`, switching to "
70
+ "`pending_overload_policy='block'`, or adding consumer capacity."
71
+ )
72
+
73
+ def __init__(
74
+ self,
75
+ *args: object,
76
+ queue: str | None = None,
77
+ message_id: str | None = None,
78
+ operation: str | None = None,
79
+ ) -> None:
80
+ message = "Pending queue reached its configured limit" if not args else args[0]
81
+ if not isinstance(message, str) or len(args) > 1:
82
+ super().__init__(*args, queue=queue, message_id=message_id, operation=operation)
83
+ return
84
+ if self._REMEDIATION not in message:
85
+ message = f"{message}; {self._REMEDIATION}"
86
+ super().__init__(message, queue=queue, message_id=message_id, operation=operation)
87
+
88
+
89
+ class QueueDrainedError(RedisMessageQueueError):
90
+ """Raised when publish() is called after drain() or aclose()."""
91
+
92
+
93
+ class RetryBudgetExhaustedError(redis.exceptions.RedisError, RedisMessageQueueError):
94
+ """Tenacity retry budget exhausted; underlying redis-py exception is .__cause__."""
95
+
96
+ _REMEDIATION = (
97
+ "verify Redis connectivity and consider increasing `retry_budget_seconds` if transient failures are expected."
98
+ )
99
+
100
+ def __init__(
101
+ self,
102
+ *args: object,
103
+ queue: str | None = None,
104
+ message_id: str | None = None,
105
+ operation: str | None = None,
106
+ ) -> None:
107
+ message = "Redis retry budget exhausted" if not args else args[0]
108
+ if not isinstance(message, str) or len(args) > 1:
109
+ RedisMessageQueueError.__init__(
110
+ self,
111
+ *args,
112
+ queue=queue,
113
+ message_id=message_id,
114
+ operation=operation,
115
+ )
116
+ return
117
+ if self._REMEDIATION not in message:
118
+ message = f"{message}; {self._REMEDIATION}"
119
+ RedisMessageQueueError.__init__(
120
+ self,
121
+ message,
122
+ queue=queue,
123
+ message_id=message_id,
124
+ operation=operation,
125
+ )
126
+
127
+
128
+ def _set_exception_context(
129
+ exc: BaseException,
130
+ *,
131
+ queue: str | None = None,
132
+ message_id: str | None = None,
133
+ operation: str | None = None,
134
+ ) -> None:
135
+ if not isinstance(exc, RedisMessageQueueError):
136
+ return
137
+ if queue is not None:
138
+ exc.queue = queue
139
+ if message_id is not None:
140
+ exc.message_id = message_id
141
+ if operation is not None:
142
+ exc.operation = operation
143
+
144
+
145
+ def wrap_lua_response_error(exc: redis.exceptions.ResponseError) -> LuaScriptError | None:
146
+ message = str(exc)
147
+ if message.startswith("WRONGTYPE ") or message.startswith("OOM during publish;"):
148
+ return LuaScriptError(message)
149
+ return None
@@ -41,7 +41,9 @@ from redis_message_queue._event import EventOperation, EventOutcome
41
41
  from redis_message_queue._exceptions import (
42
42
  ConfigurationError,
43
43
  QueueBackpressureError,
44
+ RedisMessageQueueError,
44
45
  RetryBudgetExhaustedError,
46
+ _set_exception_context,
45
47
  wrap_lua_response_error,
46
48
  )
47
49
  from redis_message_queue._stored_message import (
@@ -49,6 +51,7 @@ from redis_message_queue._stored_message import (
49
51
  MessageData,
50
52
  decode_stored_message,
51
53
  encode_stored_message,
54
+ extract_stored_message_id,
52
55
  )
53
56
  from redis_message_queue.interrupt_handler._interface import (
54
57
  BaseGracefulInterruptHandler,
@@ -269,6 +272,7 @@ class RedisGateway(AbstractRedisGateway):
269
272
  self._recovering_claim_ids: dict[str, set[str]] = {}
270
273
  self._pending_claim_ids_lock = threading.Lock()
271
274
  self._drain_pending_claim_ids_lock = threading.Lock()
275
+ self._last_drain_error: BaseException | None = None
272
276
  self._event_queue_name: str | None = None
273
277
  self._event_emitter: Callable[..., None] | None = None
274
278
 
@@ -343,13 +347,17 @@ class RedisGateway(AbstractRedisGateway):
343
347
  return result
344
348
  if self._pending_overload_policy != "block":
345
349
  raise QueueBackpressureError(
346
- f"Pending queue {queue!r} reached max_pending_length={self._max_pending_length}"
350
+ f"Pending queue {queue!r} reached max_pending_length={self._max_pending_length}",
351
+ queue=queue,
352
+ operation="publish",
347
353
  )
348
354
  remaining = deadline - time.monotonic()
349
355
  if remaining <= 0:
350
356
  raise QueueBackpressureError(
351
357
  f"Pending queue {queue!r} stayed at max_pending_length={self._max_pending_length} "
352
- f"for {self._pending_overload_block_timeout_seconds} seconds"
358
+ f"for {self._pending_overload_block_timeout_seconds} seconds",
359
+ queue=queue,
360
+ operation="publish",
353
361
  )
354
362
  sleep_seconds = min(_jitter_pending_overload_backoff_seconds(backoff_seconds), remaining)
355
363
  time.sleep(sleep_seconds)
@@ -380,6 +388,7 @@ class RedisGateway(AbstractRedisGateway):
380
388
  "an empty key would create a bare-prefix Redis marker that silently suppresses unrelated messages"
381
389
  )
382
390
  stored_message = encode_stored_message(message)
391
+ message_id = extract_stored_message_id(stored_message)
383
392
  operation_id = uuid.uuid4().hex
384
393
  operation_result_key = self._publish_operation_result_key(dedup_key, operation_id)
385
394
 
@@ -404,6 +413,9 @@ class RedisGateway(AbstractRedisGateway):
404
413
 
405
414
  try:
406
415
  return _publish()
416
+ except RedisMessageQueueError as exc:
417
+ _set_exception_context(exc, queue=queue, message_id=message_id, operation="publish")
418
+ raise
407
419
  finally:
408
420
  self._delete_operation_result_key(operation_result_key)
409
421
 
@@ -423,20 +435,29 @@ class RedisGateway(AbstractRedisGateway):
423
435
  retry budget on top of the client.
424
436
  """
425
437
  stored_message = encode_stored_message(message)
438
+ message_id = extract_stored_message_id(stored_message)
426
439
  if self._max_pending_length is not None:
427
- self._run_pending_backpressure_operation(
428
- queue,
429
- lambda: self._eval(
430
- ADD_MESSAGE_LUA_SCRIPT,
431
- 1,
440
+ try:
441
+ self._run_pending_backpressure_operation(
432
442
  queue,
433
- stored_message,
434
- self._lua_max_pending_length(),
435
- self._pending_overload_policy,
436
- ),
437
- )
443
+ lambda: self._eval(
444
+ ADD_MESSAGE_LUA_SCRIPT,
445
+ 1,
446
+ queue,
447
+ stored_message,
448
+ self._lua_max_pending_length(),
449
+ self._pending_overload_policy,
450
+ ),
451
+ )
452
+ except RedisMessageQueueError as exc:
453
+ _set_exception_context(exc, queue=queue, message_id=message_id, operation="publish")
454
+ raise
438
455
  return
439
- self._redis_client.lpush(queue, stored_message)
456
+ try:
457
+ self._redis_client.lpush(queue, stored_message)
458
+ except RedisMessageQueueError as exc:
459
+ _set_exception_context(exc, queue=queue, message_id=message_id, operation="publish")
460
+ raise
440
461
 
441
462
  def move_message(
442
463
  self,
@@ -471,6 +492,14 @@ class RedisGateway(AbstractRedisGateway):
471
492
 
472
493
  try:
473
494
  return _move()
495
+ except RedisMessageQueueError as exc:
496
+ _set_exception_context(
497
+ exc,
498
+ queue=from_queue,
499
+ message_id=extract_stored_message_id(message),
500
+ operation="ack",
501
+ )
502
+ raise
474
503
  finally:
475
504
  self._delete_operation_result_key(operation_result_key)
476
505
 
@@ -501,6 +530,14 @@ class RedisGateway(AbstractRedisGateway):
501
530
 
502
531
  try:
503
532
  return _move_with_lease()
533
+ except RedisMessageQueueError as exc:
534
+ _set_exception_context(
535
+ exc,
536
+ queue=from_queue,
537
+ message_id=extract_stored_message_id(message),
538
+ operation="ack",
539
+ )
540
+ raise
504
541
  finally:
505
542
  self._delete_operation_result_key(operation_result_key)
506
543
 
@@ -526,6 +563,14 @@ class RedisGateway(AbstractRedisGateway):
526
563
 
527
564
  try:
528
565
  return _remove()
566
+ except RedisMessageQueueError as exc:
567
+ _set_exception_context(
568
+ exc,
569
+ queue=queue,
570
+ message_id=extract_stored_message_id(message),
571
+ operation="ack",
572
+ )
573
+ raise
529
574
  finally:
530
575
  self._delete_operation_result_key(operation_result_key)
531
576
 
@@ -554,6 +599,14 @@ class RedisGateway(AbstractRedisGateway):
554
599
 
555
600
  try:
556
601
  return _remove_with_lease()
602
+ except RedisMessageQueueError as exc:
603
+ _set_exception_context(
604
+ exc,
605
+ queue=queue,
606
+ message_id=extract_stored_message_id(message),
607
+ operation="ack",
608
+ )
609
+ raise
557
610
  finally:
558
611
  self._delete_operation_result_key(operation_result_key)
559
612
 
@@ -673,7 +726,9 @@ class RedisGateway(AbstractRedisGateway):
673
726
  error=retry_exc,
674
727
  )
675
728
  raise RetryBudgetExhaustedError(
676
- "Redis retry budget exhausted during message claim"
729
+ "Redis retry budget exhausted during message claim",
730
+ queue=from_queue,
731
+ operation="claim",
677
732
  ) from retry_exc
678
733
  except BaseException:
679
734
  pending_claim_id_to_share = claim_id
@@ -746,7 +801,9 @@ class RedisGateway(AbstractRedisGateway):
746
801
  error=last_retryable_exception,
747
802
  )
748
803
  raise RetryBudgetExhaustedError(
749
- "Redis retry budget exhausted during message claim"
804
+ "Redis retry budget exhausted during message claim",
805
+ queue=from_queue,
806
+ operation="claim",
750
807
  ) from last_retryable_exception
751
808
  return None
752
809
  time.sleep(min(_VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS, remaining))
@@ -1159,6 +1216,7 @@ class RedisGateway(AbstractRedisGateway):
1159
1216
  the deadline fired or transient Redis errors prevented full drain.
1160
1217
  """
1161
1218
  with self._drain_pending_claim_ids_lock:
1219
+ self._last_drain_error = None
1162
1220
  return self._drain_pending_claim_ids_unlocked(
1163
1221
  processing_queue,
1164
1222
  deadline_monotonic=deadline_monotonic,
@@ -1176,12 +1234,14 @@ class RedisGateway(AbstractRedisGateway):
1176
1234
  else:
1177
1235
  recover = self._recover_pending_non_visibility_timeout_claim
1178
1236
  skipped_transient: set[str] = set()
1237
+ last_error: BaseException | None = None
1179
1238
  while True:
1180
1239
  # ``>=`` (not ``>``) makes ``timeout=0`` deterministically take
1181
1240
  # the no-recovery fast path: the deadline equals the call-time
1182
1241
  # ``monotonic()``, so the first iteration falls through to the
1183
1242
  # state-only check below.
1184
1243
  if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
1244
+ last_error = TimeoutError("drain pending-claim recovery deadline expired")
1185
1245
  break
1186
1246
  with self._pending_claim_ids_lock:
1187
1247
  pending = self._pending_claim_ids.get(processing_queue)
@@ -1201,10 +1261,12 @@ class RedisGateway(AbstractRedisGateway):
1201
1261
  recover(processing_queue, claim_id, deadline_monotonic=deadline_monotonic)
1202
1262
  clear = True
1203
1263
  except _DrainDeadlineExceeded:
1264
+ last_error = TimeoutError("drain pending-claim recovery deadline expired")
1204
1265
  break
1205
1266
  except Exception as exc:
1206
1267
  if not is_redis_retryable_exception(exc):
1207
1268
  raise
1269
+ last_error = exc
1208
1270
  logger.warning(
1209
1271
  "Transient Redis error draining pending claim %s; will retry on next drain: %s",
1210
1272
  claim_id,
@@ -1214,4 +1276,7 @@ class RedisGateway(AbstractRedisGateway):
1214
1276
  finally:
1215
1277
  self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
1216
1278
  with self._pending_claim_ids_lock:
1217
- return not self._pending_claim_ids.get(processing_queue)
1279
+ pending = self._pending_claim_ids.get(processing_queue)
1280
+ if pending:
1281
+ self._last_drain_error = last_error
1282
+ return not pending
@@ -8,6 +8,7 @@ MessageData = str | bytes
8
8
 
9
9
  _STORED_MESSAGE_PREFIX = "\x1eRMQ1:"
10
10
  _STORED_MESSAGE_PREFIX_BYTES = _STORED_MESSAGE_PREFIX.encode("utf-8")
11
+ _NON_ENVELOPE_STRICT_ERROR = "value does not start with RMQ envelope prefix; expected an rmq-published message"
11
12
 
12
13
 
13
14
  @dataclass(frozen=True)
@@ -43,7 +44,7 @@ def encode_stored_message(message: str) -> str:
43
44
  return f"{_STORED_MESSAGE_PREFIX}{json.dumps(envelope, separators=(',', ':'))}"
44
45
 
45
46
 
46
- def decode_stored_message(message: MessageData) -> MessageData:
47
+ def decode_stored_message(message: MessageData, *, strict_envelope_decoding: bool = False) -> MessageData:
47
48
  """Strip the stored-message envelope and return the original payload.
48
49
 
49
50
  Designed to consume values produced by ``encode_stored_message`` only.
@@ -55,9 +56,11 @@ def decode_stored_message(message: MessageData) -> MessageData:
55
56
  custom gateways feeding raw input must wrap before decoding.
56
57
 
57
58
  Raises ``MalformedStoredMessageError`` when the value starts with the RMQ
58
- envelope prefix but is not a valid payload-bearing envelope.
59
+ envelope prefix but is not a valid payload-bearing envelope. When
60
+ ``strict_envelope_decoding=True``, also raises for values that do not start
61
+ with the RMQ envelope prefix.
59
62
  """
60
- envelope = _decode_envelope(message)
63
+ envelope = _decode_envelope(message, strict_envelope_decoding=strict_envelope_decoding)
61
64
  if envelope is None:
62
65
  return message
63
66
  _message_id, payload = envelope
@@ -66,22 +69,26 @@ def decode_stored_message(message: MessageData) -> MessageData:
66
69
  return payload
67
70
 
68
71
 
69
- def extract_stored_message_id(message: MessageData) -> str | None:
72
+ def extract_stored_message_id(message: MessageData, *, strict_envelope_decoding: bool = False) -> str | None:
70
73
  """Return the RMQ envelope id, or None for values that are not RMQ envelopes.
71
74
 
72
75
  Raises ``MalformedStoredMessageError`` when the value starts with the RMQ
73
- envelope prefix but is not a valid payload-bearing envelope.
76
+ envelope prefix but is not a valid payload-bearing envelope. When
77
+ ``strict_envelope_decoding=True``, also raises for values that do not start
78
+ with the RMQ envelope prefix.
74
79
  """
75
- envelope = _decode_envelope(message)
80
+ envelope = _decode_envelope(message, strict_envelope_decoding=strict_envelope_decoding)
76
81
  if envelope is None:
77
82
  return None
78
83
  message_id, _payload = envelope
79
84
  return message_id
80
85
 
81
86
 
82
- def _decode_envelope(message: MessageData) -> tuple[str, str] | None:
87
+ def _decode_envelope(message: MessageData, *, strict_envelope_decoding: bool = False) -> tuple[str, str] | None:
83
88
  if isinstance(message, bytes):
84
89
  if not message.startswith(_STORED_MESSAGE_PREFIX_BYTES):
90
+ if strict_envelope_decoding:
91
+ raise MalformedStoredMessageError(_NON_ENVELOPE_STRICT_ERROR)
85
92
  return None
86
93
  try:
87
94
  message = message.decode("utf-8")
@@ -90,6 +97,8 @@ def _decode_envelope(message: MessageData) -> tuple[str, str] | None:
90
97
  "Stored message starts with the RMQ envelope prefix but is not valid UTF-8"
91
98
  ) from exc
92
99
  elif not message.startswith(_STORED_MESSAGE_PREFIX):
100
+ if strict_envelope_decoding:
101
+ raise MalformedStoredMessageError(_NON_ENVELOPE_STRICT_ERROR)
93
102
  return None
94
103
 
95
104
  envelope_body = message[len(_STORED_MESSAGE_PREFIX) :]
@@ -2,6 +2,7 @@ from redis_message_queue._event import EventOperation, EventOutcome, QueueEvent
2
2
  from redis_message_queue._exceptions import (
3
3
  CleanupFailedError,
4
4
  ConfigurationError,
5
+ DrainFailedError,
5
6
  GatewayContractError,
6
7
  LuaScriptError,
7
8
  MalformedStoredMessageError,
@@ -14,7 +15,11 @@ from redis_message_queue._stored_message import ClaimedMessage, MessageData
14
15
  from redis_message_queue.asyncio._abstract_redis_gateway import AbstractRedisGateway
15
16
  from redis_message_queue.asyncio._redis_gateway import RedisGateway
16
17
  from redis_message_queue.asyncio.redis_message_queue import RedisMessageQueue
17
- from redis_message_queue.interrupt_handler import BaseGracefulInterruptHandler, GracefulInterruptHandler
18
+ from redis_message_queue.interrupt_handler import (
19
+ BaseGracefulInterruptHandler,
20
+ EventDrivenInterruptHandler,
21
+ GracefulInterruptHandler,
22
+ )
18
23
 
19
24
  __all__ = [
20
25
  "RedisMessageQueue",
@@ -22,6 +27,7 @@ __all__ = [
22
27
  "AbstractRedisGateway",
23
28
  "ClaimedMessage",
24
29
  "MessageData",
30
+ "EventDrivenInterruptHandler",
25
31
  "GracefulInterruptHandler",
26
32
  "BaseGracefulInterruptHandler",
27
33
  "QueueEvent",
@@ -29,6 +35,7 @@ __all__ = [
29
35
  "EventOutcome",
30
36
  "RedisMessageQueueError",
31
37
  "ConfigurationError",
38
+ "DrainFailedError",
32
39
  "GatewayContractError",
33
40
  "LuaScriptError",
34
41
  "MalformedStoredMessageError",