redis-message-queue 8.0.3__tar.gz → 8.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/PKG-INFO +32 -1
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/README.md +31 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/pyproject.toml +1 -1
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/__init__.py +4 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_event.py +6 -0
- redis_message_queue-8.1.0/redis_message_queue/_exceptions.py +149 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_redis_gateway.py +81 -16
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_stored_message.py +16 -7
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/__init__.py +8 -1
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/_redis_gateway.py +81 -16
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/redis_message_queue.py +208 -8
- redis_message_queue-8.1.0/redis_message_queue/interrupt_handler/__init__.py +9 -0
- redis_message_queue-8.1.0/redis_message_queue/interrupt_handler/_event_driven.py +24 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/redis_message_queue.py +223 -10
- redis_message_queue-8.0.3/redis_message_queue/_exceptions.py +0 -71
- redis_message_queue-8.0.3/redis_message_queue/interrupt_handler/__init__.py +0 -4
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/LICENSE +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_callable_utils.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_config.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_queue_key_manager.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_redis_cluster.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/_abstract_redis_gateway.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/interrupt_handler/_implementation.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/interrupt_handler/_interface.py +0 -0
- {redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/py.typed +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: redis-message-queue
|
|
3
|
-
Version: 8.0
|
|
3
|
+
Version: 8.1.0
|
|
4
4
|
Summary: Python message queuing with Redis and message deduplication
|
|
5
5
|
License: MIT
|
|
6
6
|
License-File: LICENSE
|
|
@@ -392,6 +392,32 @@ while not interrupt.is_interrupted():
|
|
|
392
392
|
> processes, or install one top-level signal owner that calls `queue.drain()`
|
|
393
393
|
> / `queue.aclose()` or sets an application stop event.
|
|
394
394
|
|
|
395
|
+
If another library owns SIGTERM/SIGINT in the same process, adapt its shutdown
|
|
396
|
+
signal to rmq with a user-owned event instead of installing rmq signal handlers:
|
|
397
|
+
|
|
398
|
+
```python
|
|
399
|
+
import threading
|
|
400
|
+
|
|
401
|
+
from redis_message_queue import EventDrivenInterruptHandler, RedisMessageQueue
|
|
402
|
+
|
|
403
|
+
stop_event = threading.Event()
|
|
404
|
+
interrupt = EventDrivenInterruptHandler(stop_event)
|
|
405
|
+
queue = RedisMessageQueue("q", client=client, interrupt=interrupt)
|
|
406
|
+
|
|
407
|
+
while not interrupt.is_interrupted():
|
|
408
|
+
with queue.process_message() as message:
|
|
409
|
+
if message is not None:
|
|
410
|
+
process(message)
|
|
411
|
+
|
|
412
|
+
# In the sibling library's shutdown hook:
|
|
413
|
+
stop_event.set()
|
|
414
|
+
queue.drain(timeout=25)
|
|
415
|
+
```
|
|
416
|
+
|
|
417
|
+
The caller MUST set `stop_event` before exiting. rmq observes
|
|
418
|
+
`is_interrupted()` and exits cooperatively; it does not call `sys.exit()` or
|
|
419
|
+
otherwise force process shutdown.
|
|
420
|
+
|
|
395
421
|
There are three distinct shutdown shapes; pick the one that matches your runtime:
|
|
396
422
|
|
|
397
423
|
| Shape | Trigger | In-flight handler | Pending claim IDs |
|
|
@@ -589,6 +615,11 @@ using `":queue:"` with a queue name that overlaps RQ keys. rmq has no fixed
|
|
|
589
615
|
library prefix; generated keys share the Redis DB namespace with every other
|
|
590
616
|
Redis user.
|
|
591
617
|
|
|
618
|
+
Set `strict_envelope_decoding=True` if this Redis is shared with sibling task
|
|
619
|
+
libraries (Celery, RQ, Dramatiq) to fail-fast on foreign payloads. With the
|
|
620
|
+
default `False`, non-rmq values that do not start with the rmq envelope prefix
|
|
621
|
+
remain backward-compatible raw messages and are yielded to the handler.
|
|
622
|
+
|
|
592
623
|
## Production notes
|
|
593
624
|
|
|
594
625
|
### Fork safety and pre-fork servers
|
|
@@ -366,6 +366,32 @@ while not interrupt.is_interrupted():
|
|
|
366
366
|
> processes, or install one top-level signal owner that calls `queue.drain()`
|
|
367
367
|
> / `queue.aclose()` or sets an application stop event.
|
|
368
368
|
|
|
369
|
+
If another library owns SIGTERM/SIGINT in the same process, adapt its shutdown
|
|
370
|
+
signal to rmq with a user-owned event instead of installing rmq signal handlers:
|
|
371
|
+
|
|
372
|
+
```python
|
|
373
|
+
import threading
|
|
374
|
+
|
|
375
|
+
from redis_message_queue import EventDrivenInterruptHandler, RedisMessageQueue
|
|
376
|
+
|
|
377
|
+
stop_event = threading.Event()
|
|
378
|
+
interrupt = EventDrivenInterruptHandler(stop_event)
|
|
379
|
+
queue = RedisMessageQueue("q", client=client, interrupt=interrupt)
|
|
380
|
+
|
|
381
|
+
while not interrupt.is_interrupted():
|
|
382
|
+
with queue.process_message() as message:
|
|
383
|
+
if message is not None:
|
|
384
|
+
process(message)
|
|
385
|
+
|
|
386
|
+
# In the sibling library's shutdown hook:
|
|
387
|
+
stop_event.set()
|
|
388
|
+
queue.drain(timeout=25)
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
The caller MUST set `stop_event` before exiting. rmq observes
|
|
392
|
+
`is_interrupted()` and exits cooperatively; it does not call `sys.exit()` or
|
|
393
|
+
otherwise force process shutdown.
|
|
394
|
+
|
|
369
395
|
There are three distinct shutdown shapes; pick the one that matches your runtime:
|
|
370
396
|
|
|
371
397
|
| Shape | Trigger | In-flight handler | Pending claim IDs |
|
|
@@ -563,6 +589,11 @@ using `":queue:"` with a queue name that overlaps RQ keys. rmq has no fixed
|
|
|
563
589
|
library prefix; generated keys share the Redis DB namespace with every other
|
|
564
590
|
Redis user.
|
|
565
591
|
|
|
592
|
+
Set `strict_envelope_decoding=True` if this Redis is shared with sibling task
|
|
593
|
+
libraries (Celery, RQ, Dramatiq) to fail-fast on foreign payloads. With the
|
|
594
|
+
default `False`, non-rmq values that do not start with the rmq envelope prefix
|
|
595
|
+
remain backward-compatible raw messages and are yielded to the handler.
|
|
596
|
+
|
|
566
597
|
## Production notes
|
|
567
598
|
|
|
568
599
|
### Fork safety and pre-fork servers
|
|
@@ -3,6 +3,7 @@ from redis_message_queue._event import EventOperation, EventOutcome, QueueEvent
|
|
|
3
3
|
from redis_message_queue._exceptions import (
|
|
4
4
|
CleanupFailedError,
|
|
5
5
|
ConfigurationError,
|
|
6
|
+
DrainFailedError,
|
|
6
7
|
GatewayContractError,
|
|
7
8
|
LuaScriptError,
|
|
8
9
|
MalformedStoredMessageError,
|
|
@@ -15,6 +16,7 @@ from redis_message_queue._redis_gateway import RedisGateway
|
|
|
15
16
|
from redis_message_queue._stored_message import ClaimedMessage, MessageData
|
|
16
17
|
from redis_message_queue.interrupt_handler import (
|
|
17
18
|
BaseGracefulInterruptHandler,
|
|
19
|
+
EventDrivenInterruptHandler,
|
|
18
20
|
GracefulInterruptHandler,
|
|
19
21
|
)
|
|
20
22
|
from redis_message_queue.redis_message_queue import RedisMessageQueue
|
|
@@ -25,6 +27,7 @@ __all__ = [
|
|
|
25
27
|
"AbstractRedisGateway",
|
|
26
28
|
"ClaimedMessage",
|
|
27
29
|
"MessageData",
|
|
30
|
+
"EventDrivenInterruptHandler",
|
|
28
31
|
"GracefulInterruptHandler",
|
|
29
32
|
"BaseGracefulInterruptHandler",
|
|
30
33
|
"QueueEvent",
|
|
@@ -32,6 +35,7 @@ __all__ = [
|
|
|
32
35
|
"EventOutcome",
|
|
33
36
|
"RedisMessageQueueError",
|
|
34
37
|
"ConfigurationError",
|
|
38
|
+
"DrainFailedError",
|
|
35
39
|
"GatewayContractError",
|
|
36
40
|
"LuaScriptError",
|
|
37
41
|
"MalformedStoredMessageError",
|
|
@@ -24,6 +24,7 @@ class EventOperation(StrEnum):
|
|
|
24
24
|
TRIM_FAILED = "trim_failed"
|
|
25
25
|
RETRY_ATTEMPT = "retry_attempt"
|
|
26
26
|
RETRY_EXHAUSTED = "retry_exhausted"
|
|
27
|
+
DRAIN = "drain"
|
|
27
28
|
|
|
28
29
|
|
|
29
30
|
class EventOutcome(StrEnum):
|
|
@@ -32,6 +33,7 @@ class EventOutcome(StrEnum):
|
|
|
32
33
|
SUCCESS = "success"
|
|
33
34
|
FAILURE = "failure"
|
|
34
35
|
SKIPPED = "skipped"
|
|
36
|
+
START = "start"
|
|
35
37
|
|
|
36
38
|
|
|
37
39
|
@dataclass(frozen=True)
|
|
@@ -68,3 +70,7 @@ class QueueEvent:
|
|
|
68
70
|
the actual exception object when one was raised; pass to OpenTelemetry
|
|
69
71
|
`span.record_exception(...)` for full trace attribution
|
|
70
72
|
"""
|
|
73
|
+
timeout_seconds: float | None = None
|
|
74
|
+
"""the caller-requested timeout for drain operations, when applicable"""
|
|
75
|
+
pending_claim_ids: int | None = None
|
|
76
|
+
"""number of unresolved pending claim ids for drain operations, when applicable"""
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
import redis.exceptions
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class RedisMessageQueueError(Exception):
|
|
5
|
+
"""Base class for redis-message-queue specific errors."""
|
|
6
|
+
|
|
7
|
+
def __init__(
|
|
8
|
+
self,
|
|
9
|
+
*args: object,
|
|
10
|
+
queue: str | None = None,
|
|
11
|
+
message_id: str | None = None,
|
|
12
|
+
operation: str | None = None,
|
|
13
|
+
) -> None:
|
|
14
|
+
super().__init__(*args)
|
|
15
|
+
self.queue = queue
|
|
16
|
+
self.message_id = message_id
|
|
17
|
+
self.operation = operation
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class ConfigurationError(RedisMessageQueueError, ValueError):
|
|
21
|
+
"""Bad parameter values or combinations."""
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class GatewayContractError(RedisMessageQueueError, TypeError):
|
|
25
|
+
"""Custom gateway returned wrong type or violated contract."""
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class LuaScriptError(redis.exceptions.ResponseError, RedisMessageQueueError):
|
|
29
|
+
"""A Lua script returned an unexpected error_reply."""
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
*args: object,
|
|
34
|
+
queue: str | None = None,
|
|
35
|
+
message_id: str | None = None,
|
|
36
|
+
operation: str | None = None,
|
|
37
|
+
) -> None:
|
|
38
|
+
RedisMessageQueueError.__init__(
|
|
39
|
+
self,
|
|
40
|
+
*args,
|
|
41
|
+
queue=queue,
|
|
42
|
+
message_id=message_id,
|
|
43
|
+
operation=operation,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class CleanupFailedError(RedisMessageQueueError):
|
|
48
|
+
"""Cleanup after handler completion failed."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class DrainFailedError(RedisMessageQueueError):
|
|
52
|
+
"""Wraps a non-RMQ exception caught during drain pending-claim recovery.
|
|
53
|
+
|
|
54
|
+
drain() returns False as the bool result; this exception carries
|
|
55
|
+
F7 context (queue, operation="drain") into the drain/failure event
|
|
56
|
+
payload so users diagnosing drain incidents via on_event see the
|
|
57
|
+
same structured attrs as elsewhere.
|
|
58
|
+
"""
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class MalformedStoredMessageError(RedisMessageQueueError):
|
|
62
|
+
"""Stored value is not a valid RMQ envelope for the configured decode mode."""
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class QueueBackpressureError(RedisMessageQueueError):
|
|
66
|
+
"""Publish rejected because the pending queue is at its configured limit."""
|
|
67
|
+
|
|
68
|
+
_REMEDIATION = (
|
|
69
|
+
"consider increasing `max_pending_length`, switching to "
|
|
70
|
+
"`pending_overload_policy='block'`, or adding consumer capacity."
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
def __init__(
|
|
74
|
+
self,
|
|
75
|
+
*args: object,
|
|
76
|
+
queue: str | None = None,
|
|
77
|
+
message_id: str | None = None,
|
|
78
|
+
operation: str | None = None,
|
|
79
|
+
) -> None:
|
|
80
|
+
message = "Pending queue reached its configured limit" if not args else args[0]
|
|
81
|
+
if not isinstance(message, str) or len(args) > 1:
|
|
82
|
+
super().__init__(*args, queue=queue, message_id=message_id, operation=operation)
|
|
83
|
+
return
|
|
84
|
+
if self._REMEDIATION not in message:
|
|
85
|
+
message = f"{message}; {self._REMEDIATION}"
|
|
86
|
+
super().__init__(message, queue=queue, message_id=message_id, operation=operation)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
class QueueDrainedError(RedisMessageQueueError):
|
|
90
|
+
"""Raised when publish() is called after drain() or aclose()."""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
class RetryBudgetExhaustedError(redis.exceptions.RedisError, RedisMessageQueueError):
|
|
94
|
+
"""Tenacity retry budget exhausted; underlying redis-py exception is .__cause__."""
|
|
95
|
+
|
|
96
|
+
_REMEDIATION = (
|
|
97
|
+
"verify Redis connectivity and consider increasing `retry_budget_seconds` if transient failures are expected."
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
def __init__(
|
|
101
|
+
self,
|
|
102
|
+
*args: object,
|
|
103
|
+
queue: str | None = None,
|
|
104
|
+
message_id: str | None = None,
|
|
105
|
+
operation: str | None = None,
|
|
106
|
+
) -> None:
|
|
107
|
+
message = "Redis retry budget exhausted" if not args else args[0]
|
|
108
|
+
if not isinstance(message, str) or len(args) > 1:
|
|
109
|
+
RedisMessageQueueError.__init__(
|
|
110
|
+
self,
|
|
111
|
+
*args,
|
|
112
|
+
queue=queue,
|
|
113
|
+
message_id=message_id,
|
|
114
|
+
operation=operation,
|
|
115
|
+
)
|
|
116
|
+
return
|
|
117
|
+
if self._REMEDIATION not in message:
|
|
118
|
+
message = f"{message}; {self._REMEDIATION}"
|
|
119
|
+
RedisMessageQueueError.__init__(
|
|
120
|
+
self,
|
|
121
|
+
message,
|
|
122
|
+
queue=queue,
|
|
123
|
+
message_id=message_id,
|
|
124
|
+
operation=operation,
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _set_exception_context(
|
|
129
|
+
exc: BaseException,
|
|
130
|
+
*,
|
|
131
|
+
queue: str | None = None,
|
|
132
|
+
message_id: str | None = None,
|
|
133
|
+
operation: str | None = None,
|
|
134
|
+
) -> None:
|
|
135
|
+
if not isinstance(exc, RedisMessageQueueError):
|
|
136
|
+
return
|
|
137
|
+
if queue is not None:
|
|
138
|
+
exc.queue = queue
|
|
139
|
+
if message_id is not None:
|
|
140
|
+
exc.message_id = message_id
|
|
141
|
+
if operation is not None:
|
|
142
|
+
exc.operation = operation
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def wrap_lua_response_error(exc: redis.exceptions.ResponseError) -> LuaScriptError | None:
|
|
146
|
+
message = str(exc)
|
|
147
|
+
if message.startswith("WRONGTYPE ") or message.startswith("OOM during publish;"):
|
|
148
|
+
return LuaScriptError(message)
|
|
149
|
+
return None
|
{redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_redis_gateway.py
RENAMED
|
@@ -41,7 +41,9 @@ from redis_message_queue._event import EventOperation, EventOutcome
|
|
|
41
41
|
from redis_message_queue._exceptions import (
|
|
42
42
|
ConfigurationError,
|
|
43
43
|
QueueBackpressureError,
|
|
44
|
+
RedisMessageQueueError,
|
|
44
45
|
RetryBudgetExhaustedError,
|
|
46
|
+
_set_exception_context,
|
|
45
47
|
wrap_lua_response_error,
|
|
46
48
|
)
|
|
47
49
|
from redis_message_queue._stored_message import (
|
|
@@ -49,6 +51,7 @@ from redis_message_queue._stored_message import (
|
|
|
49
51
|
MessageData,
|
|
50
52
|
decode_stored_message,
|
|
51
53
|
encode_stored_message,
|
|
54
|
+
extract_stored_message_id,
|
|
52
55
|
)
|
|
53
56
|
from redis_message_queue.interrupt_handler._interface import (
|
|
54
57
|
BaseGracefulInterruptHandler,
|
|
@@ -269,6 +272,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
269
272
|
self._recovering_claim_ids: dict[str, set[str]] = {}
|
|
270
273
|
self._pending_claim_ids_lock = threading.Lock()
|
|
271
274
|
self._drain_pending_claim_ids_lock = threading.Lock()
|
|
275
|
+
self._last_drain_error: BaseException | None = None
|
|
272
276
|
self._event_queue_name: str | None = None
|
|
273
277
|
self._event_emitter: Callable[..., None] | None = None
|
|
274
278
|
|
|
@@ -343,13 +347,17 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
343
347
|
return result
|
|
344
348
|
if self._pending_overload_policy != "block":
|
|
345
349
|
raise QueueBackpressureError(
|
|
346
|
-
f"Pending queue {queue!r} reached max_pending_length={self._max_pending_length}"
|
|
350
|
+
f"Pending queue {queue!r} reached max_pending_length={self._max_pending_length}",
|
|
351
|
+
queue=queue,
|
|
352
|
+
operation="publish",
|
|
347
353
|
)
|
|
348
354
|
remaining = deadline - time.monotonic()
|
|
349
355
|
if remaining <= 0:
|
|
350
356
|
raise QueueBackpressureError(
|
|
351
357
|
f"Pending queue {queue!r} stayed at max_pending_length={self._max_pending_length} "
|
|
352
|
-
f"for {self._pending_overload_block_timeout_seconds} seconds"
|
|
358
|
+
f"for {self._pending_overload_block_timeout_seconds} seconds",
|
|
359
|
+
queue=queue,
|
|
360
|
+
operation="publish",
|
|
353
361
|
)
|
|
354
362
|
sleep_seconds = min(_jitter_pending_overload_backoff_seconds(backoff_seconds), remaining)
|
|
355
363
|
time.sleep(sleep_seconds)
|
|
@@ -380,6 +388,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
380
388
|
"an empty key would create a bare-prefix Redis marker that silently suppresses unrelated messages"
|
|
381
389
|
)
|
|
382
390
|
stored_message = encode_stored_message(message)
|
|
391
|
+
message_id = extract_stored_message_id(stored_message)
|
|
383
392
|
operation_id = uuid.uuid4().hex
|
|
384
393
|
operation_result_key = self._publish_operation_result_key(dedup_key, operation_id)
|
|
385
394
|
|
|
@@ -404,6 +413,9 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
404
413
|
|
|
405
414
|
try:
|
|
406
415
|
return _publish()
|
|
416
|
+
except RedisMessageQueueError as exc:
|
|
417
|
+
_set_exception_context(exc, queue=queue, message_id=message_id, operation="publish")
|
|
418
|
+
raise
|
|
407
419
|
finally:
|
|
408
420
|
self._delete_operation_result_key(operation_result_key)
|
|
409
421
|
|
|
@@ -423,20 +435,29 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
423
435
|
retry budget on top of the client.
|
|
424
436
|
"""
|
|
425
437
|
stored_message = encode_stored_message(message)
|
|
438
|
+
message_id = extract_stored_message_id(stored_message)
|
|
426
439
|
if self._max_pending_length is not None:
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
lambda: self._eval(
|
|
430
|
-
ADD_MESSAGE_LUA_SCRIPT,
|
|
431
|
-
1,
|
|
440
|
+
try:
|
|
441
|
+
self._run_pending_backpressure_operation(
|
|
432
442
|
queue,
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
443
|
+
lambda: self._eval(
|
|
444
|
+
ADD_MESSAGE_LUA_SCRIPT,
|
|
445
|
+
1,
|
|
446
|
+
queue,
|
|
447
|
+
stored_message,
|
|
448
|
+
self._lua_max_pending_length(),
|
|
449
|
+
self._pending_overload_policy,
|
|
450
|
+
),
|
|
451
|
+
)
|
|
452
|
+
except RedisMessageQueueError as exc:
|
|
453
|
+
_set_exception_context(exc, queue=queue, message_id=message_id, operation="publish")
|
|
454
|
+
raise
|
|
438
455
|
return
|
|
439
|
-
|
|
456
|
+
try:
|
|
457
|
+
self._redis_client.lpush(queue, stored_message)
|
|
458
|
+
except RedisMessageQueueError as exc:
|
|
459
|
+
_set_exception_context(exc, queue=queue, message_id=message_id, operation="publish")
|
|
460
|
+
raise
|
|
440
461
|
|
|
441
462
|
def move_message(
|
|
442
463
|
self,
|
|
@@ -471,6 +492,14 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
471
492
|
|
|
472
493
|
try:
|
|
473
494
|
return _move()
|
|
495
|
+
except RedisMessageQueueError as exc:
|
|
496
|
+
_set_exception_context(
|
|
497
|
+
exc,
|
|
498
|
+
queue=from_queue,
|
|
499
|
+
message_id=extract_stored_message_id(message),
|
|
500
|
+
operation="ack",
|
|
501
|
+
)
|
|
502
|
+
raise
|
|
474
503
|
finally:
|
|
475
504
|
self._delete_operation_result_key(operation_result_key)
|
|
476
505
|
|
|
@@ -501,6 +530,14 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
501
530
|
|
|
502
531
|
try:
|
|
503
532
|
return _move_with_lease()
|
|
533
|
+
except RedisMessageQueueError as exc:
|
|
534
|
+
_set_exception_context(
|
|
535
|
+
exc,
|
|
536
|
+
queue=from_queue,
|
|
537
|
+
message_id=extract_stored_message_id(message),
|
|
538
|
+
operation="ack",
|
|
539
|
+
)
|
|
540
|
+
raise
|
|
504
541
|
finally:
|
|
505
542
|
self._delete_operation_result_key(operation_result_key)
|
|
506
543
|
|
|
@@ -526,6 +563,14 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
526
563
|
|
|
527
564
|
try:
|
|
528
565
|
return _remove()
|
|
566
|
+
except RedisMessageQueueError as exc:
|
|
567
|
+
_set_exception_context(
|
|
568
|
+
exc,
|
|
569
|
+
queue=queue,
|
|
570
|
+
message_id=extract_stored_message_id(message),
|
|
571
|
+
operation="ack",
|
|
572
|
+
)
|
|
573
|
+
raise
|
|
529
574
|
finally:
|
|
530
575
|
self._delete_operation_result_key(operation_result_key)
|
|
531
576
|
|
|
@@ -554,6 +599,14 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
554
599
|
|
|
555
600
|
try:
|
|
556
601
|
return _remove_with_lease()
|
|
602
|
+
except RedisMessageQueueError as exc:
|
|
603
|
+
_set_exception_context(
|
|
604
|
+
exc,
|
|
605
|
+
queue=queue,
|
|
606
|
+
message_id=extract_stored_message_id(message),
|
|
607
|
+
operation="ack",
|
|
608
|
+
)
|
|
609
|
+
raise
|
|
557
610
|
finally:
|
|
558
611
|
self._delete_operation_result_key(operation_result_key)
|
|
559
612
|
|
|
@@ -673,7 +726,9 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
673
726
|
error=retry_exc,
|
|
674
727
|
)
|
|
675
728
|
raise RetryBudgetExhaustedError(
|
|
676
|
-
"Redis retry budget exhausted during message claim"
|
|
729
|
+
"Redis retry budget exhausted during message claim",
|
|
730
|
+
queue=from_queue,
|
|
731
|
+
operation="claim",
|
|
677
732
|
) from retry_exc
|
|
678
733
|
except BaseException:
|
|
679
734
|
pending_claim_id_to_share = claim_id
|
|
@@ -746,7 +801,9 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
746
801
|
error=last_retryable_exception,
|
|
747
802
|
)
|
|
748
803
|
raise RetryBudgetExhaustedError(
|
|
749
|
-
"Redis retry budget exhausted during message claim"
|
|
804
|
+
"Redis retry budget exhausted during message claim",
|
|
805
|
+
queue=from_queue,
|
|
806
|
+
operation="claim",
|
|
750
807
|
) from last_retryable_exception
|
|
751
808
|
return None
|
|
752
809
|
time.sleep(min(_VISIBILITY_TIMEOUT_POLL_INTERVAL_SECONDS, remaining))
|
|
@@ -1159,6 +1216,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1159
1216
|
the deadline fired or transient Redis errors prevented full drain.
|
|
1160
1217
|
"""
|
|
1161
1218
|
with self._drain_pending_claim_ids_lock:
|
|
1219
|
+
self._last_drain_error = None
|
|
1162
1220
|
return self._drain_pending_claim_ids_unlocked(
|
|
1163
1221
|
processing_queue,
|
|
1164
1222
|
deadline_monotonic=deadline_monotonic,
|
|
@@ -1176,12 +1234,14 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1176
1234
|
else:
|
|
1177
1235
|
recover = self._recover_pending_non_visibility_timeout_claim
|
|
1178
1236
|
skipped_transient: set[str] = set()
|
|
1237
|
+
last_error: BaseException | None = None
|
|
1179
1238
|
while True:
|
|
1180
1239
|
# ``>=`` (not ``>``) makes ``timeout=0`` deterministically take
|
|
1181
1240
|
# the no-recovery fast path: the deadline equals the call-time
|
|
1182
1241
|
# ``monotonic()``, so the first iteration falls through to the
|
|
1183
1242
|
# state-only check below.
|
|
1184
1243
|
if deadline_monotonic is not None and time.monotonic() >= deadline_monotonic:
|
|
1244
|
+
last_error = TimeoutError("drain pending-claim recovery deadline expired")
|
|
1185
1245
|
break
|
|
1186
1246
|
with self._pending_claim_ids_lock:
|
|
1187
1247
|
pending = self._pending_claim_ids.get(processing_queue)
|
|
@@ -1201,10 +1261,12 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1201
1261
|
recover(processing_queue, claim_id, deadline_monotonic=deadline_monotonic)
|
|
1202
1262
|
clear = True
|
|
1203
1263
|
except _DrainDeadlineExceeded:
|
|
1264
|
+
last_error = TimeoutError("drain pending-claim recovery deadline expired")
|
|
1204
1265
|
break
|
|
1205
1266
|
except Exception as exc:
|
|
1206
1267
|
if not is_redis_retryable_exception(exc):
|
|
1207
1268
|
raise
|
|
1269
|
+
last_error = exc
|
|
1208
1270
|
logger.warning(
|
|
1209
1271
|
"Transient Redis error draining pending claim %s; will retry on next drain: %s",
|
|
1210
1272
|
claim_id,
|
|
@@ -1214,4 +1276,7 @@ class RedisGateway(AbstractRedisGateway):
|
|
|
1214
1276
|
finally:
|
|
1215
1277
|
self._finish_pending_claim_recovery(processing_queue, claim_id, clear=clear)
|
|
1216
1278
|
with self._pending_claim_ids_lock:
|
|
1217
|
-
|
|
1279
|
+
pending = self._pending_claim_ids.get(processing_queue)
|
|
1280
|
+
if pending:
|
|
1281
|
+
self._last_drain_error = last_error
|
|
1282
|
+
return not pending
|
{redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/_stored_message.py
RENAMED
|
@@ -8,6 +8,7 @@ MessageData = str | bytes
|
|
|
8
8
|
|
|
9
9
|
_STORED_MESSAGE_PREFIX = "\x1eRMQ1:"
|
|
10
10
|
_STORED_MESSAGE_PREFIX_BYTES = _STORED_MESSAGE_PREFIX.encode("utf-8")
|
|
11
|
+
_NON_ENVELOPE_STRICT_ERROR = "value does not start with RMQ envelope prefix; expected an rmq-published message"
|
|
11
12
|
|
|
12
13
|
|
|
13
14
|
@dataclass(frozen=True)
|
|
@@ -43,7 +44,7 @@ def encode_stored_message(message: str) -> str:
|
|
|
43
44
|
return f"{_STORED_MESSAGE_PREFIX}{json.dumps(envelope, separators=(',', ':'))}"
|
|
44
45
|
|
|
45
46
|
|
|
46
|
-
def decode_stored_message(message: MessageData) -> MessageData:
|
|
47
|
+
def decode_stored_message(message: MessageData, *, strict_envelope_decoding: bool = False) -> MessageData:
|
|
47
48
|
"""Strip the stored-message envelope and return the original payload.
|
|
48
49
|
|
|
49
50
|
Designed to consume values produced by ``encode_stored_message`` only.
|
|
@@ -55,9 +56,11 @@ def decode_stored_message(message: MessageData) -> MessageData:
|
|
|
55
56
|
custom gateways feeding raw input must wrap before decoding.
|
|
56
57
|
|
|
57
58
|
Raises ``MalformedStoredMessageError`` when the value starts with the RMQ
|
|
58
|
-
envelope prefix but is not a valid payload-bearing envelope.
|
|
59
|
+
envelope prefix but is not a valid payload-bearing envelope. When
|
|
60
|
+
``strict_envelope_decoding=True``, also raises for values that do not start
|
|
61
|
+
with the RMQ envelope prefix.
|
|
59
62
|
"""
|
|
60
|
-
envelope = _decode_envelope(message)
|
|
63
|
+
envelope = _decode_envelope(message, strict_envelope_decoding=strict_envelope_decoding)
|
|
61
64
|
if envelope is None:
|
|
62
65
|
return message
|
|
63
66
|
_message_id, payload = envelope
|
|
@@ -66,22 +69,26 @@ def decode_stored_message(message: MessageData) -> MessageData:
|
|
|
66
69
|
return payload
|
|
67
70
|
|
|
68
71
|
|
|
69
|
-
def extract_stored_message_id(message: MessageData) -> str | None:
|
|
72
|
+
def extract_stored_message_id(message: MessageData, *, strict_envelope_decoding: bool = False) -> str | None:
|
|
70
73
|
"""Return the RMQ envelope id, or None for values that are not RMQ envelopes.
|
|
71
74
|
|
|
72
75
|
Raises ``MalformedStoredMessageError`` when the value starts with the RMQ
|
|
73
|
-
envelope prefix but is not a valid payload-bearing envelope.
|
|
76
|
+
envelope prefix but is not a valid payload-bearing envelope. When
|
|
77
|
+
``strict_envelope_decoding=True``, also raises for values that do not start
|
|
78
|
+
with the RMQ envelope prefix.
|
|
74
79
|
"""
|
|
75
|
-
envelope = _decode_envelope(message)
|
|
80
|
+
envelope = _decode_envelope(message, strict_envelope_decoding=strict_envelope_decoding)
|
|
76
81
|
if envelope is None:
|
|
77
82
|
return None
|
|
78
83
|
message_id, _payload = envelope
|
|
79
84
|
return message_id
|
|
80
85
|
|
|
81
86
|
|
|
82
|
-
def _decode_envelope(message: MessageData) -> tuple[str, str] | None:
|
|
87
|
+
def _decode_envelope(message: MessageData, *, strict_envelope_decoding: bool = False) -> tuple[str, str] | None:
|
|
83
88
|
if isinstance(message, bytes):
|
|
84
89
|
if not message.startswith(_STORED_MESSAGE_PREFIX_BYTES):
|
|
90
|
+
if strict_envelope_decoding:
|
|
91
|
+
raise MalformedStoredMessageError(_NON_ENVELOPE_STRICT_ERROR)
|
|
85
92
|
return None
|
|
86
93
|
try:
|
|
87
94
|
message = message.decode("utf-8")
|
|
@@ -90,6 +97,8 @@ def _decode_envelope(message: MessageData) -> tuple[str, str] | None:
|
|
|
90
97
|
"Stored message starts with the RMQ envelope prefix but is not valid UTF-8"
|
|
91
98
|
) from exc
|
|
92
99
|
elif not message.startswith(_STORED_MESSAGE_PREFIX):
|
|
100
|
+
if strict_envelope_decoding:
|
|
101
|
+
raise MalformedStoredMessageError(_NON_ENVELOPE_STRICT_ERROR)
|
|
93
102
|
return None
|
|
94
103
|
|
|
95
104
|
envelope_body = message[len(_STORED_MESSAGE_PREFIX) :]
|
{redis_message_queue-8.0.3 → redis_message_queue-8.1.0}/redis_message_queue/asyncio/__init__.py
RENAMED
|
@@ -2,6 +2,7 @@ from redis_message_queue._event import EventOperation, EventOutcome, QueueEvent
|
|
|
2
2
|
from redis_message_queue._exceptions import (
|
|
3
3
|
CleanupFailedError,
|
|
4
4
|
ConfigurationError,
|
|
5
|
+
DrainFailedError,
|
|
5
6
|
GatewayContractError,
|
|
6
7
|
LuaScriptError,
|
|
7
8
|
MalformedStoredMessageError,
|
|
@@ -14,7 +15,11 @@ from redis_message_queue._stored_message import ClaimedMessage, MessageData
|
|
|
14
15
|
from redis_message_queue.asyncio._abstract_redis_gateway import AbstractRedisGateway
|
|
15
16
|
from redis_message_queue.asyncio._redis_gateway import RedisGateway
|
|
16
17
|
from redis_message_queue.asyncio.redis_message_queue import RedisMessageQueue
|
|
17
|
-
from redis_message_queue.interrupt_handler import
|
|
18
|
+
from redis_message_queue.interrupt_handler import (
|
|
19
|
+
BaseGracefulInterruptHandler,
|
|
20
|
+
EventDrivenInterruptHandler,
|
|
21
|
+
GracefulInterruptHandler,
|
|
22
|
+
)
|
|
18
23
|
|
|
19
24
|
__all__ = [
|
|
20
25
|
"RedisMessageQueue",
|
|
@@ -22,6 +27,7 @@ __all__ = [
|
|
|
22
27
|
"AbstractRedisGateway",
|
|
23
28
|
"ClaimedMessage",
|
|
24
29
|
"MessageData",
|
|
30
|
+
"EventDrivenInterruptHandler",
|
|
25
31
|
"GracefulInterruptHandler",
|
|
26
32
|
"BaseGracefulInterruptHandler",
|
|
27
33
|
"QueueEvent",
|
|
@@ -29,6 +35,7 @@ __all__ = [
|
|
|
29
35
|
"EventOutcome",
|
|
30
36
|
"RedisMessageQueueError",
|
|
31
37
|
"ConfigurationError",
|
|
38
|
+
"DrainFailedError",
|
|
32
39
|
"GatewayContractError",
|
|
33
40
|
"LuaScriptError",
|
|
34
41
|
"MalformedStoredMessageError",
|