babelqueue 1.8.0__tar.gz → 1.10.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. {babelqueue-1.8.0 → babelqueue-1.10.0}/PKG-INFO +1 -1
  2. {babelqueue-1.8.0 → babelqueue-1.10.0}/pyproject.toml +1 -1
  3. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/__init__.py +9 -2
  4. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/app.py +14 -12
  5. babelqueue-1.10.0/src/babelqueue/redrive.py +182 -0
  6. babelqueue-1.10.0/src/babelqueue/replay.py +74 -0
  7. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/transport.py +28 -3
  8. babelqueue-1.10.0/tests/test_redrive.py +140 -0
  9. babelqueue-1.10.0/tests/test_replay.py +124 -0
  10. {babelqueue-1.8.0 → babelqueue-1.10.0}/.github/FUNDING.yml +0 -0
  11. {babelqueue-1.8.0 → babelqueue-1.10.0}/.github/workflows/ci.yml +0 -0
  12. {babelqueue-1.8.0 → babelqueue-1.10.0}/.github/workflows/release.yml +0 -0
  13. {babelqueue-1.8.0 → babelqueue-1.10.0}/.gitignore +0 -0
  14. {babelqueue-1.8.0 → babelqueue-1.10.0}/CHANGELOG.md +0 -0
  15. {babelqueue-1.8.0 → babelqueue-1.10.0}/LICENSE +0 -0
  16. {babelqueue-1.8.0 → babelqueue-1.10.0}/README.md +0 -0
  17. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/artemis_transport.py +0 -0
  18. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/asb_transport.py +0 -0
  19. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/celery.py +0 -0
  20. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/codec.py +0 -0
  21. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/contracts.py +0 -0
  22. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/dead_letter.py +0 -0
  23. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/__init__.py +0 -0
  24. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/apps.py +0 -0
  25. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/management/__init__.py +0 -0
  26. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
  27. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
  28. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/exceptions.py +0 -0
  29. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/idempotency.py +0 -0
  30. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/kafka_transport.py +0 -0
  31. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/otel.py +0 -0
  32. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/pika_transport.py +0 -0
  33. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/pulsar_transport.py +0 -0
  34. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/py.typed +0 -0
  35. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/redis_transport.py +0 -0
  36. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/routing.py +0 -0
  37. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/schema.py +0 -0
  38. {babelqueue-1.8.0 → babelqueue-1.10.0}/src/babelqueue/sqs_transport.py +0 -0
  39. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
  40. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
  41. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
  42. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/order-created.json +0 -0
  43. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
  44. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/fixtures/urn-alias.json +0 -0
  45. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/manifest.json +0 -0
  46. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
  47. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/fixtures/dead-lettered.json +0 -0
  48. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/fixtures/order-created.json +0 -0
  49. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_app.py +0 -0
  50. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_artemis_conformance.py +0 -0
  51. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_artemis_transport.py +0 -0
  52. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_asb_conformance.py +0 -0
  53. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_asb_transport.py +0 -0
  54. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_celery.py +0 -0
  55. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_codec.py +0 -0
  56. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_conformance.py +0 -0
  57. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_dead_letter.py +0 -0
  58. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_django.py +0 -0
  59. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_idempotency.py +0 -0
  60. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_kafka_conformance.py +0 -0
  61. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_kafka_transport.py +0 -0
  62. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_otel.py +0 -0
  63. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_overhead.py +0 -0
  64. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_pika_transport.py +0 -0
  65. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_pulsar_conformance.py +0 -0
  66. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_pulsar_transport.py +0 -0
  67. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_redis_transport.py +0 -0
  68. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_schema.py +0 -0
  69. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_schema_conformance.py +0 -0
  70. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_sqs_conformance.py +0 -0
  71. {babelqueue-1.8.0 → babelqueue-1.10.0}/tests/test_sqs_transport.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: babelqueue
3
- Version: 1.8.0
3
+ Version: 1.10.0
4
4
  Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
5
5
  Project-URL: Homepage, https://babelqueue.com
6
6
  Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
4
4
 
5
5
  [project]
6
6
  name = "babelqueue"
7
- version = "1.8.0"
7
+ version = "1.10.0"
8
8
  description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.9"
@@ -11,14 +11,15 @@ and dead-letter helpers. Framework adapters (Celery, Django, ...) build on this.
11
11
 
12
12
  from __future__ import annotations
13
13
 
14
- from . import dead_letter, idempotency
14
+ from . import dead_letter, idempotency, redrive, replay
15
15
  from .app import BabelQueue
16
16
  from .codec import SCHEMA_VERSION, SOURCE_LANG, EnvelopeCodec
17
17
  from .contracts import HasTraceId, PolyglotMessage
18
18
  from .idempotency import IdempotencyStore, InMemoryStore
19
19
  from .exceptions import BabelQueueError, UnknownUrnError
20
+ from .replay import HEADER_REPLAY_BYPASS, bypass_external_effects, is_replay
20
21
  from .routing import UnknownUrnStrategy
21
- from .transport import InMemoryTransport, ReceivedMessage, Transport
22
+ from .transport import HeaderPublisher, InMemoryTransport, ReceivedMessage, Transport
22
23
 
23
24
  __version__ = "1.6.0"
24
25
 
@@ -33,10 +34,16 @@ __all__ = [
33
34
  "Transport",
34
35
  "InMemoryTransport",
35
36
  "ReceivedMessage",
37
+ "HeaderPublisher",
36
38
  "BabelQueueError",
37
39
  "UnknownUrnError",
38
40
  "dead_letter",
39
41
  "idempotency",
42
+ "redrive",
43
+ "replay",
44
+ "is_replay",
45
+ "bypass_external_effects",
46
+ "HEADER_REPLAY_BYPASS",
40
47
  "IdempotencyStore",
41
48
  "InMemoryStore",
42
49
  "__version__",
@@ -25,6 +25,7 @@ from typing import Any, Callable, Dict, Mapping, Optional
25
25
  from . import dead_letter
26
26
  from .codec import EnvelopeCodec
27
27
  from .exceptions import UnknownUrnError
28
+ from .replay import HEADER_REPLAY_BYPASS, _replay_scope
28
29
  from .routing import UnknownUrnStrategy
29
30
  from .transport import ReceivedMessage, Transport, make_transport
30
31
 
@@ -116,18 +117,19 @@ class BabelQueue:
116
117
 
117
118
  def dispatch(self, received: ReceivedMessage) -> None:
118
119
  """Route one reserved message to its handler and acknowledge it."""
119
- envelope = EnvelopeCodec.decode(received.body)
120
- urn = str(envelope.get("job") or envelope.get("urn") or "")
121
- handler = self._handlers.get(urn) if urn else None
122
-
123
- try:
124
- if handler is None:
125
- self._route_unknown(urn, received, envelope)
126
- return
127
- self._invoke(handler, envelope)
128
- self.transport.ack(received)
129
- except Exception as exc: # noqa: BLE001 - one bad message must not kill the loop
130
- self._retry_or_dead_letter(received, envelope, exc)
120
+ with _replay_scope(bool(received.headers.get(HEADER_REPLAY_BYPASS))):
121
+ envelope = EnvelopeCodec.decode(received.body)
122
+ urn = str(envelope.get("job") or envelope.get("urn") or "")
123
+ handler = self._handlers.get(urn) if urn else None
124
+
125
+ try:
126
+ if handler is None:
127
+ self._route_unknown(urn, received, envelope)
128
+ return
129
+ self._invoke(handler, envelope)
130
+ self.transport.ack(received)
131
+ except Exception as exc: # noqa: BLE001 - one bad message must not kill the loop
132
+ self._retry_or_dead_letter(received, envelope, exc)
131
133
 
132
134
  # -- Internals ----------------------------------------------------------
133
135
 
@@ -0,0 +1,182 @@
1
+ """Optional DLQ redrive tooling (ADR-0026): safe replay off the dead-letter queue.
2
+
3
+ The Python mirror of the Go ``Redrive``. It reads dead-lettered messages off a DLQ and
4
+ re-publishes each to its source queue (its ``dead_letter.original_queue``) or a chosen
5
+ ``to_queue``, **reset for reprocessing**: the ``dead_letter`` block is removed and ``attempts``
6
+ reset to 0, while ``job``, ``trace_id``, ``data`` and ``meta`` are preserved verbatim. It is
7
+ the operator-side counterpart to the runtime's dead-letter routing — the contract leaves
8
+ redrive to tooling, and this is that tool.
9
+
10
+ from babelqueue import BabelQueue
11
+ from babelqueue.redrive import redrive
12
+
13
+ app = BabelQueue("redis://localhost:6379/0")
14
+ result = redrive(app.transport, "orders.dlq") # back to each source
15
+ result = redrive(app.transport, "orders.dlq", to_queue="sandbox") # safe sandbox replay
16
+ plan = redrive(app.transport, "orders.dlq", dry_run=True) # inspect, change nothing
17
+
18
+ Messages are drained from the DLQ first and then processed, so restored messages (skipped,
19
+ dry-run, or undecodable) are never re-encountered in the same run; a DLQ message is
20
+ acknowledged only after a successful re-publish, and an undecodable body is restored, not
21
+ dropped.
22
+
23
+ Replay safety is sandbox routing (``to_queue``) + ``dry_run``, plus the **Replay-Bypass** guard
24
+ (``bypass=True``): it stamps a ``bq-replay-bypass`` transport header that the runtime surfaces to
25
+ handlers via :func:`babelqueue.is_replay` / :func:`babelqueue.bypass_external_effects`, so a
26
+ replay can skip external side-effects that already fired (don't re-charge, don't re-email) — see
27
+ :mod:`babelqueue.replay` (ADR-0027). The header rides out of band, so the envelope stays frozen;
28
+ it propagates over a real broker only once that broker's transport implements the optional
29
+ :class:`~babelqueue.transport.HeaderPublisher` capability (the in-memory transport does today).
30
+ """
31
+
32
+ from __future__ import annotations
33
+
34
+ from dataclasses import dataclass, field
35
+ from typing import Any, Callable, Dict, List, Mapping, Optional, Tuple
36
+
37
+ from .codec import EnvelopeCodec
38
+ from .replay import HEADER_REPLAY_BYPASS
39
+ from .transport import HeaderPublisher, ReceivedMessage, Transport
40
+
41
+ Envelope = Mapping[str, Any]
42
+ Select = Callable[[Envelope], bool]
43
+
44
+
45
+ @dataclass
46
+ class RedriveItem:
47
+ """What happened to one message during a redrive run."""
48
+
49
+ message_id: str
50
+ trace_id: str
51
+ urn: str
52
+ reason: str
53
+ from_queue: str
54
+ to: str # target queue (the plan, even on a dry run; "" when skipped/undecodable)
55
+ redriven: bool # True only when actually re-published to ``to``
56
+ bypassed: bool = False # True when the bq-replay-bypass header was stamped on the message
57
+
58
+
59
+ @dataclass
60
+ class RedriveResult:
61
+ """Summary of a redrive run."""
62
+
63
+ redriven: int = 0
64
+ skipped: int = 0
65
+ items: List[RedriveItem] = field(default_factory=list)
66
+
67
+
68
+ def redrive(
69
+ transport: Transport,
70
+ dlq: str,
71
+ *,
72
+ to_queue: Optional[str] = None,
73
+ max: int = 0,
74
+ dry_run: bool = False,
75
+ select: Optional[Select] = None,
76
+ bypass: bool = False,
77
+ timeout: float = 1.0,
78
+ ) -> RedriveResult:
79
+ """Move dead-lettered messages off ``dlq`` and replay them; see the module docstring."""
80
+ # Drain up to ``max`` messages (or all available) before processing any of them.
81
+ batch: List[Tuple[ReceivedMessage, Optional[Dict[str, Any]]]] = []
82
+ while max == 0 or len(batch) < max:
83
+ message = transport.pop(dlq, timeout)
84
+ if message is None:
85
+ break
86
+ batch.append((message, _decoded(message.body)))
87
+
88
+ result = RedriveResult()
89
+ for message, envelope in batch:
90
+ if envelope is None:
91
+ transport.publish(dlq, message.body) # restore the poison body; never drop it
92
+ transport.ack(message)
93
+ result.skipped += 1
94
+ result.items.append(RedriveItem("", "", "", "", dlq, "", False))
95
+ continue
96
+
97
+ meta_raw = envelope.get("meta")
98
+ meta: Mapping[str, Any] = meta_raw if isinstance(meta_raw, Mapping) else {}
99
+ dl_raw = envelope.get("dead_letter")
100
+ dead_letter: Mapping[str, Any] = dl_raw if isinstance(dl_raw, Mapping) else {}
101
+ item = RedriveItem(
102
+ message_id=str(meta.get("id", "")),
103
+ trace_id=str(envelope.get("trace_id", "")),
104
+ urn=EnvelopeCodec.urn(envelope),
105
+ reason=str(dead_letter.get("reason", "")),
106
+ from_queue=dlq,
107
+ to="",
108
+ redriven=False,
109
+ )
110
+
111
+ if select is not None and not select(envelope):
112
+ transport.publish(dlq, message.body) # not selected: restore unchanged
113
+ transport.ack(message)
114
+ result.skipped += 1
115
+ result.items.append(item)
116
+ continue
117
+
118
+ target = to_queue or _source_queue_of(envelope)
119
+ item.to = target
120
+
121
+ if dry_run:
122
+ transport.publish(dlq, message.body) # report the plan; restore unchanged
123
+ transport.ack(message)
124
+ result.skipped += 1
125
+ result.items.append(item)
126
+ continue
127
+
128
+ reset = dict(envelope)
129
+ reset.pop("dead_letter", None)
130
+ reset["attempts"] = 0
131
+ try:
132
+ item.bypassed = _publish_redriven(
133
+ transport, target, EnvelopeCodec.encode(reset), bypass
134
+ )
135
+ except Exception:
136
+ transport.publish(dlq, message.body) # restore on a publish failure, then surface
137
+ transport.ack(message)
138
+ raise
139
+ transport.ack(message)
140
+ item.redriven = True
141
+ result.redriven += 1
142
+ result.items.append(item)
143
+
144
+ return result
145
+
146
+
147
+ def _decoded(body: str) -> Optional[Dict[str, Any]]:
148
+ """Decode a DLQ body, or None when it is not a redrivable envelope.
149
+
150
+ ``EnvelopeCodec.decode`` returns ``{}`` for malformed/non-object input; an object with no
151
+ string ``job`` is likewise not redrivable.
152
+ """
153
+ envelope = EnvelopeCodec.decode(body)
154
+ if not envelope or not isinstance(envelope.get("job"), str):
155
+ return None
156
+ return envelope
157
+
158
+
159
+ def _publish_redriven(transport: Transport, queue: str, body: str, bypass: bool) -> bool:
160
+ """Re-publish a reset message to ``queue``. When ``bypass`` is set and the transport is a
161
+ :class:`~babelqueue.transport.HeaderPublisher`, stamp the ``bq-replay-bypass`` header and
162
+ return True; otherwise publish plainly and return False."""
163
+ if bypass and isinstance(transport, HeaderPublisher):
164
+ transport.publish_with_headers(queue, body, {HEADER_REPLAY_BYPASS: "1"})
165
+ return True
166
+ transport.publish(queue, body)
167
+ return False
168
+
169
+
170
+ def _source_queue_of(envelope: Envelope) -> str:
171
+ """Default redrive target: ``dead_letter.original_queue``, falling back to ``meta.queue``."""
172
+ dead_letter = envelope.get("dead_letter")
173
+ if isinstance(dead_letter, Mapping):
174
+ original = dead_letter.get("original_queue")
175
+ if isinstance(original, str) and original:
176
+ return original
177
+ meta = envelope.get("meta")
178
+ if isinstance(meta, Mapping):
179
+ queue = meta.get("queue")
180
+ if isinstance(queue, str):
181
+ return queue
182
+ return ""
@@ -0,0 +1,74 @@
1
+ """Optional Replay-Bypass guard (ADR-0027): skip external side-effects on a deliberate replay.
2
+
3
+ The Python mirror of the Go ``replay.go``. It closes the loop left open by
4
+ :mod:`babelqueue.redrive`: a deliberate replay re-runs the handler, and its external
5
+ side-effects re-fire (a second charge, a duplicate email). :mod:`babelqueue.idempotency` stops
6
+ an *accidental* duplicate; this stops the *intended* reprocess from re-firing effects that
7
+ already happened.
8
+
9
+ ``Redrive`` (with ``bypass=True``) stamps a ``bq-replay-bypass`` transport header on a redriven
10
+ message; the runtime surfaces it to the handler via a :class:`contextvars.ContextVar`. A handler
11
+ wraps its external, non-idempotent side in :func:`bypass_external_effects` so a replay re-runs the
12
+ idempotent core but skips effects that already fired::
13
+
14
+ from babelqueue import is_replay, bypass_external_effects
15
+
16
+ @app.handler("urn:babel:orders:created")
17
+ def on_order_created(data, meta):
18
+ save_order(data) # idempotent core — always runs
19
+ bypass_external_effects(lambda: send_email(data)) # external effect — skipped on replay
20
+
21
+ The marker rides **out of band** as a transport header, so the frozen envelope is untouched
22
+ (GR-1). It propagates over a real broker only once that broker's concrete transport implements
23
+ the optional :class:`~babelqueue.transport.HeaderPublisher` capability — a follow-up, like the
24
+ broker bindings; the in-memory transport supports it today, so the path is end-to-end testable.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import contextlib
30
+ import contextvars
31
+ from typing import Callable, Iterator, Optional, TypeVar
32
+
33
+ #: The out-of-band transport header :func:`~babelqueue.redrive.redrive` stamps on a replayed
34
+ #: message (with ``bypass=True``) and that the runtime surfaces as :func:`is_replay`.
35
+ HEADER_REPLAY_BYPASS = "bq-replay-bypass"
36
+
37
+ _replay_var: contextvars.ContextVar[bool] = contextvars.ContextVar(
38
+ "babelqueue_replay", default=False
39
+ )
40
+
41
+ T = TypeVar("T")
42
+
43
+
44
+ def is_replay() -> bool:
45
+ """Whether the message currently being handled was redriven with the replay-bypass marker.
46
+
47
+ True means this is a deliberate replay, so external side-effects that already happened should
48
+ be skipped. Reads the flag the runtime set on the context from the
49
+ :data:`HEADER_REPLAY_BYPASS` transport header.
50
+ """
51
+ return _replay_var.get()
52
+
53
+
54
+ def bypass_external_effects(fn: Callable[[], T]) -> Optional[T]:
55
+ """Run ``fn`` unless the current message is a replay (see :func:`is_replay`), in which case
56
+ skip it and return ``None``.
57
+
58
+ Wrap the external, non-idempotent side of a handler — sending an email, charging a card,
59
+ calling a third party — so a replay re-runs the idempotent core but does not re-fire effects
60
+ that already happened.
61
+ """
62
+ if is_replay():
63
+ return None
64
+ return fn()
65
+
66
+
67
+ @contextlib.contextmanager
68
+ def _replay_scope(active: bool) -> Iterator[None]:
69
+ """Internal: mark the current context as a replay (or not) for the span of one dispatch."""
70
+ token = _replay_var.set(active)
71
+ try:
72
+ yield
73
+ finally:
74
+ _replay_var.reset(token)
@@ -8,8 +8,8 @@ from __future__ import annotations
8
8
 
9
9
  from abc import ABC, abstractmethod
10
10
  from collections import defaultdict, deque
11
- from dataclasses import dataclass
12
- from typing import Any, Deque, Dict, Optional
11
+ from dataclasses import dataclass, field
12
+ from typing import Any, Deque, Dict, Optional, Protocol, runtime_checkable
13
13
 
14
14
  from .exceptions import BabelQueueError
15
15
 
@@ -21,6 +21,9 @@ class ReceivedMessage:
21
21
  body: str
22
22
  queue: str
23
23
  handle: Any = None
24
+ #: Out-of-band transport headers a :class:`HeaderPublisher` carried with the message
25
+ #: (e.g. the ``bq-replay-bypass`` marker). Empty for transports that don't surface them.
26
+ headers: Dict[str, str] = field(default_factory=dict)
24
27
 
25
28
 
26
29
  class Transport(ABC):
@@ -42,20 +45,42 @@ class Transport(ABC):
42
45
  """Release any resources (override if needed)."""
43
46
 
44
47
 
48
+ @runtime_checkable
49
+ class HeaderPublisher(Protocol):
50
+ """Optional :class:`Transport` capability: publish a body together with out-of-band
51
+ transport headers (e.g. the replay-bypass marker), for brokers that carry per-message
52
+ metadata. A transport that does not implement it simply does not propagate headers —
53
+ callers fall back to plain :meth:`Transport.publish` (ADR-0027)."""
54
+
55
+ def publish_with_headers(self, queue: str, body: str, headers: Dict[str, str]) -> None:
56
+ """Append ``body`` to ``queue`` along with out-of-band ``headers``."""
57
+ ...
58
+
59
+
45
60
  class InMemoryTransport(Transport):
46
61
  """In-process transport for tests and broker-free local runs (``memory://``)."""
47
62
 
48
63
  def __init__(self) -> None:
64
+ # Bodies and their out-of-band headers are kept in lockstep parallel deques, so the
65
+ # body storage layout stays a plain Deque[str].
49
66
  self._queues: Dict[str, Deque[str]] = defaultdict(deque)
67
+ self._headers: Dict[str, Deque[Dict[str, str]]] = defaultdict(deque)
50
68
 
51
69
  def publish(self, queue: str, body: str) -> None:
52
70
  self._queues[queue].append(body)
71
+ self._headers[queue].append({})
72
+
73
+ def publish_with_headers(self, queue: str, body: str, headers: Dict[str, str]) -> None:
74
+ self._queues[queue].append(body)
75
+ self._headers[queue].append(dict(headers))
53
76
 
54
77
  def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
55
78
  dq = self._queues.get(queue)
56
79
  if not dq:
57
80
  return None
58
- return ReceivedMessage(body=dq.popleft(), queue=queue)
81
+ return ReceivedMessage(
82
+ body=dq.popleft(), queue=queue, headers=self._headers[queue].popleft()
83
+ )
59
84
 
60
85
  def ack(self, message: ReceivedMessage) -> None:
61
86
  # Already removed on pop; nothing to do.
@@ -0,0 +1,140 @@
1
+ """Tests for the optional DLQ redrive tooling (ADR-0026)."""
2
+
3
+ import pytest
4
+
5
+ from babelqueue import dead_letter
6
+ from babelqueue.codec import EnvelopeCodec
7
+ from babelqueue.redrive import redrive
8
+ from babelqueue.transport import InMemoryTransport
9
+
10
+
11
+ def _dead_letter(transport, dlq, urn, original_queue, data=None):
12
+ env = EnvelopeCodec.make(urn, data or {}, queue=original_queue)
13
+ annotated = dead_letter.annotate(env, "failed", original_queue, 3, error="boom")
14
+ transport.publish(dlq, EnvelopeCodec.encode(annotated))
15
+ return annotated
16
+
17
+
18
+ def _drain(transport, queue):
19
+ out = []
20
+ while True:
21
+ message = transport.pop(queue, 0)
22
+ if message is None:
23
+ break
24
+ out.append(EnvelopeCodec.decode(message.body))
25
+ transport.ack(message)
26
+ return out
27
+
28
+
29
+ def test_redrive_to_source():
30
+ t = InMemoryTransport()
31
+ orig = _dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders", {"order_id": 1})
32
+
33
+ result = redrive(t, "orders.dlq")
34
+
35
+ assert result.redriven == 1 and result.skipped == 0
36
+ assert result.items[0].bypassed is False # bypass is off by default
37
+ got = _drain(t, "orders")
38
+ assert len(got) == 1
39
+ assert "dead_letter" not in got[0]
40
+ assert got[0]["attempts"] == 0
41
+ assert got[0]["trace_id"] == orig["trace_id"]
42
+ assert got[0]["data"] == {"order_id": 1}
43
+ assert EnvelopeCodec.urn(got[0]) == "urn:babel:orders:created"
44
+ assert _drain(t, "orders.dlq") == []
45
+
46
+
47
+ def test_redrive_to_sandbox():
48
+ t = InMemoryTransport()
49
+ _dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
50
+
51
+ result = redrive(t, "orders.dlq", to_queue="sandbox")
52
+
53
+ assert result.redriven == 1
54
+ assert _drain(t, "orders") == []
55
+ assert len(_drain(t, "sandbox")) == 1
56
+
57
+
58
+ def test_redrive_dry_run():
59
+ t = InMemoryTransport()
60
+ _dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
61
+
62
+ result = redrive(t, "orders.dlq", dry_run=True)
63
+
64
+ assert result.redriven == 0 and result.skipped == 1
65
+ assert result.items[0].to == "orders"
66
+ assert result.items[0].redriven is False
67
+ assert _drain(t, "orders") == []
68
+ dlq = _drain(t, "orders.dlq")
69
+ assert len(dlq) == 1 and "dead_letter" in dlq[0]
70
+
71
+
72
+ def test_redrive_select():
73
+ t = InMemoryTransport()
74
+ _dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
75
+ _dead_letter(t, "dlq", "urn:babel:emails:welcome", "emails")
76
+
77
+ result = redrive(t, "dlq", select=lambda e: EnvelopeCodec.urn(e) == "urn:babel:orders:created")
78
+
79
+ assert result.redriven == 1 and result.skipped == 1
80
+ assert len(_drain(t, "orders")) == 1
81
+ assert _drain(t, "emails") == []
82
+ assert len(_drain(t, "dlq")) == 1 # the unselected one is restored
83
+
84
+
85
+ def test_redrive_max():
86
+ t = InMemoryTransport()
87
+ for _ in range(3):
88
+ _dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
89
+
90
+ result = redrive(t, "dlq", max=2)
91
+
92
+ assert result.redriven == 2
93
+ assert len(_drain(t, "dlq")) == 1 # Max respected
94
+
95
+
96
+ def test_redrive_no_dead_letter_falls_back_to_meta_queue():
97
+ t = InMemoryTransport()
98
+ # a plain (never dead-lettered) envelope on the DLQ — redrive falls back to meta.queue
99
+ env = EnvelopeCodec.make("urn:babel:orders:created", {}, queue="orders")
100
+ t.publish("dlq", EnvelopeCodec.encode(env))
101
+
102
+ result = redrive(t, "dlq")
103
+
104
+ assert result.redriven == 1
105
+ assert len(_drain(t, "orders")) == 1
106
+
107
+
108
+ class _FailOnTarget(InMemoryTransport):
109
+ """An in-memory transport that refuses to publish to one queue."""
110
+
111
+ def __init__(self, fail_queue):
112
+ super().__init__()
113
+ self._fail_queue = fail_queue
114
+
115
+ def publish(self, queue, body):
116
+ if queue == self._fail_queue:
117
+ raise RuntimeError("publish refused")
118
+ super().publish(queue, body)
119
+
120
+
121
+ def test_redrive_publish_failure_restores():
122
+ t = _FailOnTarget("orders")
123
+ _dead_letter(t, "dlq", "urn:babel:orders:created", "orders")
124
+
125
+ with pytest.raises(RuntimeError):
126
+ redrive(t, "dlq")
127
+
128
+ assert len(_drain(t, "dlq")) == 1 # restored to the DLQ, not lost
129
+ assert _drain(t, "orders") == [] # nothing reached the source queue
130
+
131
+
132
+ def test_redrive_undecodable_restored():
133
+ t = InMemoryTransport()
134
+ t.publish("dlq", "not-json{{{")
135
+
136
+ result = redrive(t, "dlq")
137
+
138
+ assert result.redriven == 0 and result.skipped == 1
139
+ message = t.pop("dlq", 0)
140
+ assert message is not None and message.body == "not-json{{{"
@@ -0,0 +1,124 @@
1
+ """Tests for the optional Replay-Bypass guard (ADR-0027)."""
2
+
3
+ from babelqueue import (
4
+ BabelQueue,
5
+ bypass_external_effects,
6
+ dead_letter,
7
+ is_replay,
8
+ )
9
+ from babelqueue.codec import EnvelopeCodec
10
+ from babelqueue.redrive import redrive
11
+ from babelqueue.replay import HEADER_REPLAY_BYPASS, _replay_scope
12
+ from babelqueue.transport import InMemoryTransport, Transport
13
+
14
+
15
+ def _dead_letter(transport, dlq, urn, original_queue, data=None):
16
+ env = EnvelopeCodec.make(urn, data or {}, queue=original_queue)
17
+ annotated = dead_letter.annotate(env, "failed", original_queue, 3, error="boom")
18
+ transport.publish(dlq, EnvelopeCodec.encode(annotated))
19
+ return annotated
20
+
21
+
22
+ def test_is_replay_default_false():
23
+ assert is_replay() is False
24
+
25
+
26
+ def test_bypass_runs_when_not_replay():
27
+ ran = []
28
+
29
+ def effect():
30
+ ran.append("x")
31
+ return "did-it"
32
+
33
+ assert bypass_external_effects(effect) == "did-it"
34
+ assert ran == ["x"]
35
+
36
+
37
+ def test_bypass_skips_on_replay_and_scope_resets():
38
+ ran = []
39
+
40
+ def effect():
41
+ ran.append("x")
42
+ raise AssertionError("must not run on a replay")
43
+
44
+ with _replay_scope(True):
45
+ assert is_replay() is True
46
+ assert bypass_external_effects(effect) is None
47
+ assert ran == []
48
+ assert is_replay() is False # the scope reset the contextvar
49
+
50
+
51
+ def test_in_memory_transport_carries_headers():
52
+ t = InMemoryTransport()
53
+ t.publish_with_headers("q", "body", {HEADER_REPLAY_BYPASS: "1"})
54
+ msg = t.pop("q", 0)
55
+ assert msg is not None and msg.headers.get(HEADER_REPLAY_BYPASS) == "1"
56
+ # a plain publish carries no headers
57
+ t.publish("q", "plain")
58
+ msg2 = t.pop("q", 0)
59
+ assert msg2 is not None and msg2.headers == {}
60
+
61
+
62
+ def test_redrive_bypass_stamps_header_and_consume_skips_effects():
63
+ t = InMemoryTransport()
64
+ _dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders", {"order_id": 1})
65
+
66
+ result = redrive(t, "orders.dlq", bypass=True)
67
+ assert result.redriven == 1
68
+ assert result.items[0].bypassed is True
69
+
70
+ # the redriven message on the source queue carries the bypass header
71
+ msg = t.pop("orders", 0)
72
+ assert msg is not None and msg.headers.get(HEADER_REPLAY_BYPASS) == "1"
73
+
74
+ # consume it: the handler sees is_replay and BypassExternalEffects skips the side-effect
75
+ emailed = []
76
+ app = BabelQueue(transport=t)
77
+
78
+ @app.handler("urn:babel:orders:created")
79
+ def on_created(data, meta):
80
+ assert is_replay() is True
81
+ bypass_external_effects(lambda: emailed.append(data))
82
+
83
+ app.dispatch(msg)
84
+ assert emailed == [] # the external side-effect was skipped on the bypassed replay
85
+
86
+
87
+ def test_normal_delivery_is_not_a_replay():
88
+ t = InMemoryTransport()
89
+ app = BabelQueue(transport=t)
90
+ fired = []
91
+
92
+ @app.handler("urn:babel:orders:created")
93
+ def on_created(data, meta):
94
+ assert is_replay() is False
95
+ bypass_external_effects(lambda: fired.append(data))
96
+
97
+ app.publish("urn:babel:orders:created", {"order_id": 9})
98
+ app.consume(max_messages=1, timeout=0)
99
+ assert fired == [{"order_id": 9}] # effect runs on a normal (non-replay) delivery
100
+
101
+
102
+ class _PlainTransport(Transport):
103
+ """A Transport that is deliberately NOT a HeaderPublisher (no publish_with_headers)."""
104
+
105
+ def __init__(self):
106
+ self._inner = InMemoryTransport()
107
+
108
+ def publish(self, queue, body):
109
+ self._inner.publish(queue, body)
110
+
111
+ def pop(self, queue, timeout=1.0):
112
+ return self._inner.pop(queue, timeout)
113
+
114
+ def ack(self, message):
115
+ self._inner.ack(message)
116
+
117
+
118
+ def test_redrive_bypass_without_header_support_falls_back():
119
+ t = _PlainTransport()
120
+ _dead_letter(t, "orders.dlq", "urn:babel:orders:created", "orders")
121
+
122
+ result = redrive(t, "orders.dlq", bypass=True)
123
+ assert result.redriven == 1
124
+ assert result.items[0].bypassed is False # no-op without a HeaderPublisher
File without changes
File without changes
File without changes
File without changes