babelqueue 1.8.0__tar.gz → 1.10.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {babelqueue-1.8.0 → babelqueue-1.10.0}/PKG-INFO +1 -1
- {babelqueue-1.8.0 → babelqueue-1.10.0}/pyproject.toml +1 -1
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/__init__.py +9 -2
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/app.py +14 -12
- babelqueue-1.10.0/src/babelqueue/redrive.py +182 -0
- babelqueue-1.10.0/src/babelqueue/replay.py +74 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/transport.py +28 -3
- babelqueue-1.10.0/tests/test_redrive.py +140 -0
- babelqueue-1.10.0/tests/test_replay.py +124 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/.github/FUNDING.yml +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/.github/workflows/ci.yml +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/.github/workflows/release.yml +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/.gitignore +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/CHANGELOG.md +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/LICENSE +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/README.md +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/artemis_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/asb_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/exceptions.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/idempotency.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/kafka_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/otel.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/pulsar_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/redis_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/schema.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/sqs_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/manifest.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_app.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_artemis_conformance.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_artemis_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_asb_conformance.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_asb_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_celery.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_codec.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_conformance.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_django.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_idempotency.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_kafka_conformance.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_kafka_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_otel.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_overhead.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_pulsar_conformance.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_pulsar_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_redis_transport.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_schema.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_schema_conformance.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_sqs_conformance.py +0 -0
- {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_sqs_transport.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.10.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.10.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -11,14 +11,15 @@ and dead-letter helpers. Framework adapters (Celery, Django, ...) build on this.
|
|
|
11
11
|
|
|
12
12
|
from __future__ import annotations
|
|
13
13
|
|
|
14
|
-
from . import dead_letter, idempotency
|
|
14
|
+
from . import dead_letter, idempotency, redrive, replay
|
|
15
15
|
from .app import BabelQueue
|
|
16
16
|
from .codec import SCHEMA_VERSION, SOURCE_LANG, EnvelopeCodec
|
|
17
17
|
from .contracts import HasTraceId, PolyglotMessage
|
|
18
18
|
from .idempotency import IdempotencyStore, InMemoryStore
|
|
19
19
|
from .exceptions import BabelQueueError, UnknownUrnError
|
|
20
|
+
from .replay import HEADER_REPLAY_BYPASS, bypass_external_effects, is_replay
|
|
20
21
|
from .routing import UnknownUrnStrategy
|
|
21
|
-
from .transport import InMemoryTransport, ReceivedMessage, Transport
|
|
22
|
+
from .transport import HeaderPublisher, InMemoryTransport, ReceivedMessage, Transport
|
|
22
23
|
|
|
23
24
|
__version__ = "1.6.0"
|
|
24
25
|
|
|
@@ -33,10 +34,16 @@ __all__ = [
|
|
|
33
34
|
"Transport",
|
|
34
35
|
"InMemoryTransport",
|
|
35
36
|
"ReceivedMessage",
|
|
37
|
+
"HeaderPublisher",
|
|
36
38
|
"BabelQueueError",
|
|
37
39
|
"UnknownUrnError",
|
|
38
40
|
"dead_letter",
|
|
39
41
|
"idempotency",
|
|
42
|
+
"redrive",
|
|
43
|
+
"replay",
|
|
44
|
+
"is_replay",
|
|
45
|
+
"bypass_external_effects",
|
|
46
|
+
"HEADER_REPLAY_BYPASS",
|
|
40
47
|
"IdempotencyStore",
|
|
41
48
|
"InMemoryStore",
|
|
42
49
|
"__version__",
|
|
@@ -25,6 +25,7 @@ from typing import Any, Callable, Dict, Mapping, Optional
|
|
|
25
25
|
from . import dead_letter
|
|
26
26
|
from .codec import EnvelopeCodec
|
|
27
27
|
from .exceptions import UnknownUrnError
|
|
28
|
+
from .replay import HEADER_REPLAY_BYPASS, _replay_scope
|
|
28
29
|
from .routing import UnknownUrnStrategy
|
|
29
30
|
from .transport import ReceivedMessage, Transport, make_transport
|
|
30
31
|
|
|
@@ -116,18 +117,19 @@ class BabelQueue:
|
|
|
116
117
|
|
|
117
118
|
def dispatch(self, received: ReceivedMessage) -> None:
|
|
118
119
|
"""Route one reserved message to its handler and acknowledge it."""
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
120
|
+
with _replay_scope(bool(received.headers.get(HEADER_REPLAY_BYPASS))):
|
|
121
|
+
envelope = EnvelopeCodec.decode(received.body)
|
|
122
|
+
urn = str(envelope.get("job") or envelope.get("urn") or "")
|
|
123
|
+
handler = self._handlers.get(urn) if urn else None
|
|
124
|
+
|
|
125
|
+
try:
|
|
126
|
+
if handler is None:
|
|
127
|
+
self._route_unknown(urn, received, envelope)
|
|
128
|
+
return
|
|
129
|
+
self._invoke(handler, envelope)
|
|
130
|
+
self.transport.ack(received)
|
|
131
|
+
except Exception as exc: # noqa: BLE001 - one bad message must not kill the loop
|
|
132
|
+
self._retry_or_dead_letter(received, envelope, exc)
|
|
131
133
|
|
|
132
134
|
# -- Internals ----------------------------------------------------------
|
|
133
135
|
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
"""Optional DLQ redrive tooling (ADR-0026): safe replay off the dead-letter queue.
|
|
2
|
+
|
|
3
|
+
The Python mirror of the Go ``Redrive``. It reads dead-lettered messages off a DLQ and
|
|
4
|
+
re-publishes each to its source queue (its ``dead_letter.original_queue``) or a chosen
|
|
5
|
+
``to_queue``, **reset for reprocessing**: the ``dead_letter`` block is removed and ``attempts``
|
|
6
|
+
reset to 0, while ``job``, ``trace_id``, ``data`` and ``meta`` are preserved verbatim. It is
|
|
7
|
+
the operator-side counterpart to the runtime's dead-letter routing — the contract leaves
|
|
8
|
+
redrive to tooling, and this is that tool.
|
|
9
|
+
|
|
10
|
+
from babelqueue import BabelQueue
|
|
11
|
+
from babelqueue.redrive import redrive
|
|
12
|
+
|
|
13
|
+
app = BabelQueue("redis://localhost:6379/0")
|
|
14
|
+
result = redrive(app.transport, "orders.dlq") # back to each source
|
|
15
|
+
result = redrive(app.transport, "orders.dlq", to_queue="sandbox") # safe sandbox replay
|
|
16
|
+
plan = redrive(app.transport, "orders.dlq", dry_run=True) # inspect, change nothing
|
|
17
|
+
|
|
18
|
+
Messages are drained from the DLQ first and then processed, so restored messages (skipped,
|
|
19
|
+
dry-run, or undecodable) are never re-encountered in the same run; a DLQ message is
|
|
20
|
+
acknowledged only after a successful re-publish, and an undecodable body is restored, not
|
|
21
|
+
dropped.
|
|
22
|
+
|
|
23
|
+
Replay safety is sandbox routing (``to_queue``) + ``dry_run``, plus the **Replay-Bypass** guard
|
|
24
|
+
(``bypass=True``): it stamps a ``bq-replay-bypass`` transport header that the runtime surfaces to
|
|
25
|
+
handlers via :func:`babelqueue.is_replay` / :func:`babelqueue.bypass_external_effects`, so a
|
|
26
|
+
replay can skip external side-effects that already fired (don't re-charge, don't re-email) — see
|
|
27
|
+
:mod:`babelqueue.replay` (ADR-0027). The header rides out of band, so the envelope stays frozen;
|
|
28
|
+
it propagates over a real broker only once that broker's transport implements the optional
|
|
29
|
+
:class:`~babelqueue.transport.HeaderPublisher` capability (the in-memory transport does today).
|
|
30
|
+
"""
|
|
31
|
+
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
from dataclasses import dataclass, field
|
|
35
|
+
from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
|
|
36
|
+
|
|
37
|
+
from .codec import EnvelopeCodec
|
|
38
|
+
from .replay import HEADER_REPLAY_BYPASS
|
|
39
|
+
from .transport import HeaderPublisher, ReceivedMessage, Transport
|
|
40
|
+
|
|
41
|
+
Envelope = Mapping[str, Any]
|
|
42
|
+
Select = Callable[[Envelope], bool]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
@dataclass
|
|
46
|
+
class RedriveItem:
|
|
47
|
+
"""What happened to one message during a redrive run."""
|
|
48
|
+
|
|
49
|
+
message_id: str
|
|
50
|
+
trace_id: str
|
|
51
|
+
urn: str
|
|
52
|
+
reason: str
|
|
53
|
+
from_queue: str
|
|
54
|
+
to: str # target queue (the plan, even on a dry run; "" when skipped/undecodable)
|
|
55
|
+
redriven: bool # True only when actually re-published to ``to``
|
|
56
|
+
bypassed: bool = False # True when the bq-replay-bypass header was stamped on the message
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass
|
|
60
|
+
class RedriveResult:
|
|
61
|
+
"""Summary of a redrive run."""
|
|
62
|
+
|
|
63
|
+
redriven: int = 0
|
|
64
|
+
skipped: int = 0
|
|
65
|
+
items: List[RedriveItem] = field(default_factory=list)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def redrive(
|
|
69
|
+
transport: Transport,
|
|
70
|
+
dlq: str,
|
|
71
|
+
*,
|
|
72
|
+
to_queue: Optional[str] = None,
|
|
73
|
+
max: int = 0,
|
|
74
|
+
dry_run: bool = False,
|
|
75
|
+
select: Optional[Select] = None,
|
|
76
|
+
bypass: bool = False,
|
|
77
|
+
timeout: float = 1.0,
|
|
78
|
+
) -> RedriveResult:
|
|
79
|
+
"""Move dead-lettered messages off ``dlq`` and replay them; see the module docstring."""
|
|
80
|
+
# Drain up to ``max`` messages (or all available) before processing any of them.
|
|
81
|
+
batch: List[Tuple[ReceivedMessage, Optional[Dict[str, Any]]]] = []
|
|
82
|
+
while max == 0 or len(batch) < max:
|
|
83
|
+
message = transport.pop(dlq, timeout)
|
|
84
|
+
if message is None:
|
|
85
|
+
break
|
|
86
|
+
batch.append((message, _decoded(message.body)))
|
|
87
|
+
|
|
88
|
+
result = RedriveResult()
|
|
89
|
+
for message, envelope in batch:
|
|
90
|
+
if envelope is None:
|
|
91
|
+
transport.publish(dlq, message.body) # restore the poison body; never drop it
|
|
92
|
+
transport.ack(message)
|
|
93
|
+
result.skipped += 1
|
|
94
|
+
result.items.append(RedriveItem("", "", "", "", dlq, "", False))
|
|
95
|
+
continue
|
|
96
|
+
|
|
97
|
+
meta_raw = envelope.get("meta")
|
|
98
|
+
meta: Mapping[str, Any] = meta_raw if isinstance(meta_raw, Mapping) else {}
|
|
99
|
+
dl_raw = envelope.get("dead_letter")
|
|
100
|
+
dead_letter: Mapping[str, Any] = dl_raw if isinstance(dl_raw, Mapping) else {}
|
|
101
|
+
item = RedriveItem(
|
|
102
|
+
message_id=str(meta.get("id", "")),
|
|
103
|
+
trace_id=str(envelope.get("trace_id", "")),
|
|
104
|
+
urn=EnvelopeCodec.urn(envelope),
|
|
105
|
+
reason=str(dead_letter.get("reason", "")),
|
|
106
|
+
from_queue=dlq,
|
|
107
|
+
to="",
|
|
108
|
+
redriven=False,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
if select is not None and not select(envelope):
|
|
112
|
+
transport.publish(dlq, message.body) # not selected: restore unchanged
|
|
113
|
+
transport.ack(message)
|
|
114
|
+
result.skipped += 1
|
|
115
|
+
result.items.append(item)
|
|
116
|
+
continue
|
|
117
|
+
|
|
118
|
+
target = to_queue or _source_queue_of(envelope)
|
|
119
|
+
item.to = target
|
|
120
|
+
|
|
121
|
+
if dry_run:
|
|
122
|
+
transport.publish(dlq, message.body) # report the plan; restore unchanged
|
|
123
|
+
transport.ack(message)
|
|
124
|
+
result.skipped += 1
|
|
125
|
+
result.items.append(item)
|
|
126
|
+
continue
|
|
127
|
+
|
|
128
|
+
reset = dict(envelope)
|
|
129
|
+
reset.pop("dead_letter", None)
|
|
130
|
+
reset["attempts"] = 0
|
|
131
|
+
try:
|
|
132
|
+
item.bypassed = _publish_redriven(
|
|
133
|
+
transport, target, EnvelopeCodec.encode(reset), bypass
|
|
134
|
+
)
|
|
135
|
+
except Exception:
|
|
136
|
+
transport.publish(dlq, message.body) # restore on a publish failure, then surface
|
|
137
|
+
transport.ack(message)
|
|
138
|
+
raise
|
|
139
|
+
transport.ack(message)
|
|
140
|
+
item.redriven = True
|
|
141
|
+
result.redriven += 1
|
|
142
|
+
result.items.append(item)
|
|
143
|
+
|
|
144
|
+
return result
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _decoded(body: str) -> Optional[Dict[str, Any]]:
|
|
148
|
+
"""Decode a DLQ body, or None when it is not a redrivable envelope.
|
|
149
|
+
|
|
150
|
+
``EnvelopeCodec.decode`` returns ``{}`` for malformed/non-object input; an object with no
|
|
151
|
+
string ``job`` is likewise not redrivable.
|
|
152
|
+
"""
|
|
153
|
+
envelope = EnvelopeCodec.decode(body)
|
|
154
|
+
if not envelope or not isinstance(envelope.get("job"), str):
|
|
155
|
+
return None
|
|
156
|
+
return envelope
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _publish_redriven(transport: Transport, queue: str, body: str, bypass: bool) -> bool:
|
|
160
|
+
"""Re-publish a reset message to ``queue``. When ``bypass`` is set and the transport is a
|
|
161
|
+
:class:`~babelqueue.transport.HeaderPublisher`, stamp the ``bq-replay-bypass`` header and
|
|
162
|
+
return True; otherwise publish plainly and return False."""
|
|
163
|
+
if bypass and isinstance(transport, HeaderPublisher):
|
|
164
|
+
transport.publish_with_headers(queue, body, {HEADER_REPLAY_BYPASS: "1"})
|
|
165
|
+
return True
|
|
166
|
+
transport.publish(queue, body)
|
|
167
|
+
return False
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _source_queue_of(envelope: Envelope) -> str:
|
|
171
|
+
"""Default redrive target: ``dead_letter.original_queue``, falling back to ``meta.queue``."""
|
|
172
|
+
dead_letter = envelope.get("dead_letter")
|
|
173
|
+
if isinstance(dead_letter, Mapping):
|
|
174
|
+
original = dead_letter.get("original_queue")
|
|
175
|
+
if isinstance(original, str) and original:
|
|
176
|
+
return original
|
|
177
|
+
meta = envelope.get("meta")
|
|
178
|
+
if isinstance(meta, Mapping):
|
|
179
|
+
queue = meta.get("queue")
|
|
180
|
+
if isinstance(queue, str):
|
|
181
|
+
return queue
|
|
182
|
+
return ""
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
"""Optional Replay-Bypass guard (ADR-0027): skip external side-effects on a deliberate replay.
|
|
2
|
+
|
|
3
|
+
The Python mirror of the Go ``replay.go``. It closes the loop left open by
|
|
4
|
+
:mod:`babelqueue.redrive`: a deliberate replay re-runs the handler, and its external
|
|
5
|
+
side-effects re-fire (a second charge, a duplicate email). :mod:`babelqueue.idempotency` stops
|
|
6
|
+
an *accidental* duplicate; this stops the *intended* reprocess from re-firing effects that
|
|
7
|
+
already happened.
|
|
8
|
+
|
|
9
|
+
``Redrive`` (with ``bypass=True``) stamps a ``bq-replay-bypass`` transport header on a redriven
|
|
10
|
+
message; the runtime surfaces it to the handler via a :class:`contextvars.ContextVar`. A handler
|
|
11
|
+
wraps its external, non-idempotent side in :func:`bypass_external_effects` so a replay re-runs the
|
|
12
|
+
idempotent core but skips effects that already fired::
|
|
13
|
+
|
|
14
|
+
from babelqueue import is_replay, bypass_external_effects
|
|
15
|
+
|
|
16
|
+
@app.handler("urn:babel:orders:created")
|
|
17
|
+
def on_order_created(data, meta):
|
|
18
|
+
save_order(data) # idempotent core — always runs
|
|
19
|
+
bypass_external_effects(lambda: send_email(data)) # external effect — skipped on replay
|
|
20
|
+
|
|
21
|
+
The marker rides **out of band** as a transport header, so the frozen envelope is untouched
|
|
22
|
+
(GR-1). It propagates over a real broker only once that broker's concrete transport implements
|
|
23
|
+
the optional :class:`~babelqueue.transport.HeaderPublisher` capability — a follow-up, like the
|
|
24
|
+
broker bindings; the in-memory transport supports it today, so the path is end-to-end testable.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import contextlib
|
|
30
|
+
import contextvars
|
|
31
|
+
from typing import Callable, Iterator, Optional, TypeVar
|
|
32
|
+
|
|
33
|
+
#: The out-of-band transport header :func:`~babelqueue.redrive.redrive` stamps on a replayed
|
|
34
|
+
#: message (with ``bypass=True``) and that the runtime surfaces as :func:`is_replay`.
|
|
35
|
+
HEADER_REPLAY_BYPASS = "bq-replay-bypass"
|
|
36
|
+
|
|
37
|
+
_replay_var: contextvars.ContextVar[bool] = contextvars.ContextVar(
|
|
38
|
+
"babelqueue_replay", default=False
|
|
39
|
+
)
|
|
40
|
+
|
|
41
|
+
T = TypeVar("T")
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def is_replay() -> bool:
|
|
45
|
+
"""Whether the message currently being handled was redriven with the replay-bypass marker.
|
|
46
|
+
|
|
47
|
+
True means this is a deliberate replay, so external side-effects that already happened should
|
|
48
|
+
be skipped. Reads the flag the runtime set on the context from the
|
|
49
|
+
:data:`HEADER_REPLAY_BYPASS` transport header.
|
|
50
|
+
"""
|
|
51
|
+
return _replay_var.get()
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def bypass_external_effects(fn: Callable[[], T]) -> Optional[T]:
|
|
55
|
+
"""Run ``fn`` unless the current message is a replay (see :func:`is_replay`), in which case
|
|
56
|
+
skip it and return ``None``.
|
|
57
|
+
|
|
58
|
+
Wrap the external, non-idempotent side of a handler — sending an email, charging a card,
|
|
59
|
+
calling a third party — so a replay re-runs the idempotent core but does not re-fire effects
|
|
60
|
+
that already happened.
|
|
61
|
+
"""
|
|
62
|
+
if is_replay():
|
|
63
|
+
return None
|
|
64
|
+
return fn()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@contextlib.contextmanager
|
|
68
|
+
def _replay_scope(active: bool) -> Iterator[None]:
|
|
69
|
+
"""Internal: mark the current context as a replay (or not) for the span of one dispatch."""
|
|
70
|
+
token = _replay_var.set(active)
|
|
71
|
+
try:
|
|
72
|
+
yield
|
|
73
|
+
finally:
|
|
74
|
+
_replay_var.reset(token)
|
|
@@ -8,8 +8,8 @@ from __future__ import annotations
|
|
|
8
8
|
|
|
9
9
|
from abc import ABC, abstractmethod
|
|
10
10
|
from collections import defaultdict, deque
|
|
11
|
-
from dataclasses import dataclass
|
|
12
|
-
from typing import Any, Deque, Dict, Optional
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from typing import Any, Deque, Dict, Optional, Protocol, runtime_checkable
|
|
13
13
|
|
|
14
14
|
from .exceptions import BabelQueueError
|
|
15
15
|
|
|
@@ -21,6 +21,9 @@ class ReceivedMessage:
|
|
|
21
21
|
body: str
|
|
22
22
|
queue: str
|
|
23
23
|
handle: Any = None
|
|
24
|
+
#: Out-of-band transport headers a :class:`HeaderPublisher` carried with the message
|
|
25
|
+
#: (e.g. the ``bq-replay-bypass`` marker). Empty for transports that don't surface them.
|
|
26
|
+
headers: Dict[str, str] = field(default_factory=dict)
|
|
24
27
|
|
|
25
28
|
|
|
26
29
|
class Transport(ABC):
|
|
@@ -42,20 +45,42 @@ class Transport(ABC):
|
|
|
42
45
|
"""Release any resources (override if needed)."""
|
|
43
46
|
|
|
44
47
|
|
|
48
|
+
@runtime_checkable
|
|
49
|
+
class HeaderPublisher(Protocol):
|
|
50
|
+
"""Optional :class:`Transport` capability: publish a body together with out-of-band
|
|
51
|
+
transport headers (e.g. the replay-bypass marker), for brokers that carry per-message
|
|
52
|
+
metadata. A transport that does not implement it simply does not propagate headers —
|
|
53
|
+
callers fall back to plain :meth:`Transport.publish` (ADR-0027)."""
|
|
54
|
+
|
|
55
|
+
def publish_with_headers(self, queue: str, body: str, headers: Dict[str, str]) -> None:
|
|
56
|
+
"""Append ``body`` to ``queue`` along with out-of-band ``headers``."""
|
|
57
|
+
...
|
|
58
|
+
|
|
59
|
+
|
|
45
60
|
class InMemoryTransport(Transport):
|
|
46
61
|
"""In-process transport for tests and broker-free local runs (``memory://``)."""
|
|
47
62
|
|
|
48
63
|
def __init__(self) -> None:
|
|
64
|
+
# Bodies and their out-of-band headers are kept in lockstep parallel deques, so the
|
|
65
|
+
# body storage layout stays a plain Deque[str].
|
|
49
66
|
self._queues: Dict[str, Deque[str]] = defaultdict(deque)
|
|
67
|
+
self._headers: Dict[str, Deque[Dict[str, str]]] = defaultdict(deque)
|
|
50
68
|
|
|
51
69
|
def publish(self, queue: str, body: str) -> None:
|
|
52
70
|
self._queues[queue].append(body)
|
|
71
|
+
self._headers[queue].append({})
|
|
72
|
+
|
|
73
|
+
def publish_with_headers(self, queue: str, body: str, headers: Dict[str, str]) -> None:
|
|
74
|
+
self._queues[queue].append(body)
|
|
75
|
+
self._headers[queue].append(dict(headers))
|
|
53
76
|
|
|
54
77
|
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
55
78
|
dq = self._queues.get(queue)
|
|
56
79
|
if not dq:
|
|
57
80
|
return None
|
|
58
|
-
return ReceivedMessage(
|
|
81
|
+
return ReceivedMessage(
|
|
82
|
+
body=dq.popleft(), queue=queue, headers=self._headers[queue].popleft()
|
|
83
|
+
)
|
|
59
84
|
|
|
60
85
|
def ack(self, message: ReceivedMessage) -> None:
|
|
61
86
|
# Already removed on pop; nothing to do.
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
"""Tests for the optional DLQ redrive tooling (ADR-0026)."""
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from babelqueue import dead_letter
|
|
6
|
+
from babelqueue.codec import EnvelopeCodec
|
|
7
|
+
from babelqueue.redrive import redrive
|
|
8
|
+
from babelqueue.transport import InMemoryTransport
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def _dead_letter(transport, dlq, urn, original_queue, data=None):
|
|
12
|
+
env = EnvelopeCodec.make(urn, data or {}, queue=original_queue)
|
|
13
|
+
annotated = dead_letter.annotate(env, "failed", original_queue, 3, error="boom")
|
|
14
|
+
transport.publish(dlq, EnvelopeCodec.encode(annotated))
|
|
15
|
+
return annotated
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _drain(transport, queue):
|
|
19
|
+
out = []
|
|
20
|
+
while True:
|
|
21
|
+
message = transport.pop(queue, 0)
|
|
22
|
+
if message is None:
|
|
23
|
+
break
|
|
24
|
+
out.append(EnvelopeCodec.decode(message.body))
|
|
25
|
+
transport.ack(message)
|
|
26
|
+
return out
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def test_redrive_to_source():
|
|
30
|
+
t = InMemoryTransport()
|
|
31
|
+
orig = _dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders", {"order_id": 1})
|
|
32
|
+
|
|
33
|
+
result = redrive(t, "orders.dlq")
|
|
34
|
+
|
|
35
|
+
assert result.redriven == 1 and result.skipped == 0
|
|
36
|
+
assert result.items[0].bypassed is False # bypass is off by default
|
|
37
|
+
got = _drain(t, "orders")
|
|
38
|
+
assert len(got) == 1
|
|
39
|
+
assert "dead_letter" not in got[0]
|
|
40
|
+
assert got[0]["attempts"] == 0
|
|
41
|
+
assert got[0]["trace_id"] == orig["trace_id"]
|
|
42
|
+
assert got[0]["data"] == {"order_id": 1}
|
|
43
|
+
assert EnvelopeCodec.urn(got[0]) == "urn:babel:orders:created"
|
|
44
|
+
assert _drain(t, "orders.dlq") == []
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def test_redrive_to_sandbox():
|
|
48
|
+
t = InMemoryTransport()
|
|
49
|
+
_dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
|
|
50
|
+
|
|
51
|
+
result = redrive(t, "orders.dlq", to_queue="sandbox")
|
|
52
|
+
|
|
53
|
+
assert result.redriven == 1
|
|
54
|
+
assert _drain(t, "orders") == []
|
|
55
|
+
assert len(_drain(t, "sandbox")) == 1
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def test_redrive_dry_run():
|
|
59
|
+
t = InMemoryTransport()
|
|
60
|
+
_dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
|
|
61
|
+
|
|
62
|
+
result = redrive(t, "orders.dlq", dry_run=True)
|
|
63
|
+
|
|
64
|
+
assert result.redriven == 0 and result.skipped == 1
|
|
65
|
+
assert result.items[0].to == "orders"
|
|
66
|
+
assert result.items[0].redriven is False
|
|
67
|
+
assert _drain(t, "orders") == []
|
|
68
|
+
dlq = _drain(t, "orders.dlq")
|
|
69
|
+
assert len(dlq) == 1 and "dead_letter" in dlq[0]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def test_redrive_select():
|
|
73
|
+
t = InMemoryTransport()
|
|
74
|
+
_dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
|
|
75
|
+
_dead_letter(t, "dlq", "urn:babel:emails:welcome", "emails")
|
|
76
|
+
|
|
77
|
+
result = redrive(t, "dlq", select=lambda e: EnvelopeCodec.urn(e) == "urn:babel:orders:created")
|
|
78
|
+
|
|
79
|
+
assert result.redriven == 1 and result.skipped == 1
|
|
80
|
+
assert len(_drain(t, "orders")) == 1
|
|
81
|
+
assert _drain(t, "emails") == []
|
|
82
|
+
assert len(_drain(t, "dlq")) == 1 # the unselected one is restored
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def test_redrive_max():
|
|
86
|
+
t = InMemoryTransport()
|
|
87
|
+
for _ in range(3):
|
|
88
|
+
_dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
|
|
89
|
+
|
|
90
|
+
result = redrive(t, "dlq", max=2)
|
|
91
|
+
|
|
92
|
+
assert result.redriven == 2
|
|
93
|
+
assert len(_drain(t, "dlq")) == 1 # Max respected
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def test_redrive_no_dead_letter_falls_back_to_meta_queue():
|
|
97
|
+
t = InMemoryTransport()
|
|
98
|
+
# a plain (never dead-lettered) envelope on the DLQ — redrive falls back to meta.queue
|
|
99
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {}, queue="orders")
|
|
100
|
+
t.publish("dlq", EnvelopeCodec.encode(env))
|
|
101
|
+
|
|
102
|
+
result = redrive(t, "dlq")
|
|
103
|
+
|
|
104
|
+
assert result.redriven == 1
|
|
105
|
+
assert len(_drain(t, "orders")) == 1
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
class _FailOnTarget(InMemoryTransport):
|
|
109
|
+
"""An in-memory transport that refuses to publish to one queue."""
|
|
110
|
+
|
|
111
|
+
def __init__(self, fail_queue):
|
|
112
|
+
super().__init__()
|
|
113
|
+
self._fail_queue = fail_queue
|
|
114
|
+
|
|
115
|
+
def publish(self, queue, body):
|
|
116
|
+
if queue == self._fail_queue:
|
|
117
|
+
raise RuntimeError("publish refused")
|
|
118
|
+
super().publish(queue, body)
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def test_redrive_publish_failure_restores():
|
|
122
|
+
t = _FailOnTarget("orders")
|
|
123
|
+
_dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
|
|
124
|
+
|
|
125
|
+
with pytest.raises(RuntimeError):
|
|
126
|
+
redrive(t, "dlq")
|
|
127
|
+
|
|
128
|
+
assert len(_drain(t, "dlq")) == 1 # restored to the DLQ, not lost
|
|
129
|
+
assert _drain(t, "orders") == [] # nothing reached the source queue
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def test_redrive_undecodable_restored():
|
|
133
|
+
t = InMemoryTransport()
|
|
134
|
+
t.publish("dlq", "not-json{{{")
|
|
135
|
+
|
|
136
|
+
result = redrive(t, "dlq")
|
|
137
|
+
|
|
138
|
+
assert result.redriven == 0 and result.skipped == 1
|
|
139
|
+
message = t.pop("dlq", 0)
|
|
140
|
+
assert message is not None and message.body == "not-json{{{"
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
"""Tests for the optional Replay-Bypass guard (ADR-0027)."""
|
|
2
|
+
|
|
3
|
+
from babelqueue import (
|
|
4
|
+
BabelQueue,
|
|
5
|
+
bypass_external_effects,
|
|
6
|
+
dead_letter,
|
|
7
|
+
is_replay,
|
|
8
|
+
)
|
|
9
|
+
from babelqueue.codec import EnvelopeCodec
|
|
10
|
+
from babelqueue.redrive import redrive
|
|
11
|
+
from babelqueue.replay import HEADER_REPLAY_BYPASS, _replay_scope
|
|
12
|
+
from babelqueue.transport import InMemoryTransport, Transport
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _dead_letter(transport, dlq, urn, original_queue, data=None):
|
|
16
|
+
env = EnvelopeCodec.make(urn, data or {}, queue=original_queue)
|
|
17
|
+
annotated = dead_letter.annotate(env, "failed", original_queue, 3, error="boom")
|
|
18
|
+
transport.publish(dlq, EnvelopeCodec.encode(annotated))
|
|
19
|
+
return annotated
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_is_replay_default_false():
|
|
23
|
+
assert is_replay() is False
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def test_bypass_runs_when_not_replay():
|
|
27
|
+
ran = []
|
|
28
|
+
|
|
29
|
+
def effect():
|
|
30
|
+
ran.append("x")
|
|
31
|
+
return "did-it"
|
|
32
|
+
|
|
33
|
+
assert bypass_external_effects(effect) == "did-it"
|
|
34
|
+
assert ran == ["x"]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def test_bypass_skips_on_replay_and_scope_resets():
|
|
38
|
+
ran = []
|
|
39
|
+
|
|
40
|
+
def effect():
|
|
41
|
+
ran.append("x")
|
|
42
|
+
raise AssertionError("must not run on a replay")
|
|
43
|
+
|
|
44
|
+
with _replay_scope(True):
|
|
45
|
+
assert is_replay() is True
|
|
46
|
+
assert bypass_external_effects(effect) is None
|
|
47
|
+
assert ran == []
|
|
48
|
+
assert is_replay() is False # the scope reset the contextvar
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def test_in_memory_transport_carries_headers():
|
|
52
|
+
t = InMemoryTransport()
|
|
53
|
+
t.publish_with_headers("q", "body", {HEADER_REPLAY_BYPASS: "1"})
|
|
54
|
+
msg = t.pop("q", 0)
|
|
55
|
+
assert msg is not None and msg.headers.get(HEADER_REPLAY_BYPASS) == "1"
|
|
56
|
+
# a plain publish carries no headers
|
|
57
|
+
t.publish("q", "plain")
|
|
58
|
+
msg2 = t.pop("q", 0)
|
|
59
|
+
assert msg2 is not None and msg2.headers == {}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_redrive_bypass_stamps_header_and_consume_skips_effects():
|
|
63
|
+
t = InMemoryTransport()
|
|
64
|
+
_dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders", {"order_id": 1})
|
|
65
|
+
|
|
66
|
+
result = redrive(t, "orders.dlq", bypass=True)
|
|
67
|
+
assert result.redriven == 1
|
|
68
|
+
assert result.items[0].bypassed is True
|
|
69
|
+
|
|
70
|
+
# the redriven message on the source queue carries the bypass header
|
|
71
|
+
msg = t.pop("orders", 0)
|
|
72
|
+
assert msg is not None and msg.headers.get(HEADER_REPLAY_BYPASS) == "1"
|
|
73
|
+
|
|
74
|
+
# consume it: the handler sees is_replay and BypassExternalEffects skips the side-effect
|
|
75
|
+
emailed = []
|
|
76
|
+
app = BabelQueue(transport=t)
|
|
77
|
+
|
|
78
|
+
@app.handler("urn:babel:orders:created")
|
|
79
|
+
def on_created(data, meta):
|
|
80
|
+
assert is_replay() is True
|
|
81
|
+
bypass_external_effects(lambda: emailed.append(data))
|
|
82
|
+
|
|
83
|
+
app.dispatch(msg)
|
|
84
|
+
assert emailed == [] # the external side-effect was skipped on the bypassed replay
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def test_normal_delivery_is_not_a_replay():
|
|
88
|
+
t = InMemoryTransport()
|
|
89
|
+
app = BabelQueue(transport=t)
|
|
90
|
+
fired = []
|
|
91
|
+
|
|
92
|
+
@app.handler("urn:babel:orders:created")
|
|
93
|
+
def on_created(data, meta):
|
|
94
|
+
assert is_replay() is False
|
|
95
|
+
bypass_external_effects(lambda: fired.append(data))
|
|
96
|
+
|
|
97
|
+
app.publish("urn:babel:orders:created", {"order_id": 9})
|
|
98
|
+
app.consume(max_messages=1, timeout=0)
|
|
99
|
+
assert fired == [{"order_id": 9}] # effect runs on a normal (non-replay) delivery
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
class _PlainTransport(Transport):
|
|
103
|
+
"""A Transport that is deliberately NOT a HeaderPublisher (no publish_with_headers)."""
|
|
104
|
+
|
|
105
|
+
def __init__(self):
|
|
106
|
+
self._inner = InMemoryTransport()
|
|
107
|
+
|
|
108
|
+
def publish(self, queue, body):
|
|
109
|
+
self._inner.publish(queue, body)
|
|
110
|
+
|
|
111
|
+
def pop(self, queue, timeout=1.0):
|
|
112
|
+
return self._inner.pop(queue, timeout)
|
|
113
|
+
|
|
114
|
+
def ack(self, message):
|
|
115
|
+
self._inner.ack(message)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def test_redrive_bypass_without_header_support_falls_back():
|
|
119
|
+
t = _PlainTransport()
|
|
120
|
+
_dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
|
|
121
|
+
|
|
122
|
+
result = redrive(t, "orders.dlq", bypass=True)
|
|
123
|
+
assert result.redriven == 1
|
|
124
|
+
assert result.items[0].bypassed is False # no-op without a HeaderPublisher
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/management/commands/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/schema/message-envelope.schema.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|