babelqueue 1.2.0__tar.gz → 1.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {babelqueue-1.2.0 → babelqueue-1.3.0}/CHANGELOG.md +18 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/PKG-INFO +3 -1
- {babelqueue-1.2.0 → babelqueue-1.3.0}/pyproject.toml +2 -1
- babelqueue-1.3.0/src/babelqueue/pulsar_transport.py +202 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/transport.py +6 -1
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/manifest.json +26 -0
- babelqueue-1.3.0/tests/test_pulsar_conformance.py +48 -0
- babelqueue-1.3.0/tests/test_pulsar_transport.py +215 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/.github/FUNDING.yml +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/.github/workflows/ci.yml +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/.github/workflows/release.yml +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/.gitignore +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/LICENSE +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/README.md +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/__init__.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/app.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/asb_transport.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/exceptions.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/redis_transport.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/sqs_transport.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_app.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_asb_conformance.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_asb_transport.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_celery.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_codec.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_conformance.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_django.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_overhead.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_redis_transport.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_sqs_conformance.py +0 -0
- {babelqueue-1.2.0 → babelqueue-1.3.0}/tests/test_sqs_transport.py +0 -0
|
@@ -7,6 +7,24 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
The envelope wire format is versioned separately by `meta.schema_version`
|
|
8
8
|
(currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
|
|
9
9
|
|
|
10
|
+
## [1.3.0] - 2026-06-13
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **Apache Pulsar transport** (`babelqueue[pulsar]`, `pulsar-client`) — `PulsarTransport`,
|
|
14
|
+
selected by the `pulsar://` (or `pulsar+ssl://`) URL scheme (e.g.
|
|
15
|
+
`pulsar://localhost:6650`; or pass a built `client`). Implements [§5 of the broker-bindings
|
|
16
|
+
contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#apache-pulsar): the canonical
|
|
17
|
+
envelope is the message payload, projected onto native Pulsar message properties
|
|
18
|
+
(string→string) — `bq-job` = URN, `bq-trace-id` = `trace_id`, `bq-message-id` = `meta.id`,
|
|
19
|
+
plus `bq-schema-version`/`bq-source-lang`/`bq-attempts`; receive → `acknowledge`; `attempts`
|
|
20
|
+
reconciled to `max(bq-attempts, redelivery_count)` (Pulsar's redelivery count is 0-based, so
|
|
21
|
+
it maps directly with no −1, and the `max` never lowers a higher body count carried by a
|
|
22
|
+
republish-driven retry). Default `Shared` subscription named `babelqueue`; a client can be
|
|
23
|
+
injected for tests/DI. The projection + reconciliation + publish/pop/ack flow are unit-tested
|
|
24
|
+
with no broker and no `pulsar-client` (the pulsar import is lazy and publishing sends raw
|
|
25
|
+
bytes). The envelope is unchanged (`schema_version: 1`); Apache Pulsar is purely additive.
|
|
26
|
+
Ships as a MINOR.
|
|
27
|
+
|
|
10
28
|
## [1.2.0] - 2026-06-13
|
|
11
29
|
|
|
12
30
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.3.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -36,6 +36,8 @@ Requires-Dist: pytest>=7; extra == 'dev'
|
|
|
36
36
|
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
37
37
|
Provides-Extra: django
|
|
38
38
|
Requires-Dist: django>=4.2; extra == 'django'
|
|
39
|
+
Provides-Extra: pulsar
|
|
40
|
+
Requires-Dist: pulsar-client>=3.4; extra == 'pulsar'
|
|
39
41
|
Provides-Extra: redis
|
|
40
42
|
Requires-Dist: redis>=4; extra == 'redis'
|
|
41
43
|
Provides-Extra: sqs
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.3.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -33,6 +33,7 @@ redis = ["redis>=4"]
|
|
|
33
33
|
amqp = ["pika>=1.3"]
|
|
34
34
|
sqs = ["boto3>=1.26"]
|
|
35
35
|
azureservicebus = ["azure-servicebus>=7.11", "azure-identity>=1.15"]
|
|
36
|
+
pulsar = ["pulsar-client>=3.4"]
|
|
36
37
|
celery = ["celery>=5"]
|
|
37
38
|
django = ["django>=4.2"]
|
|
38
39
|
dev = ["pytest>=7", "pytest-cov>=4", "mypy>=1.8", "ruff>=0.5"]
|
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
"""Apache Pulsar transport. Requires the ``pulsar`` extra:
|
|
2
|
+
|
|
3
|
+
pip install "babelqueue[pulsar]"
|
|
4
|
+
|
|
5
|
+
Producing sends the canonical envelope as the message payload and projects the contract
|
|
6
|
+
envelope fields onto native Pulsar message properties (string->string): ``bq-job`` = URN,
|
|
7
|
+
``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
|
|
8
|
+
``bq-source-lang`` / ``bq-attempts`` — so a Java/.NET/... peer can route on ``bq-job``
|
|
9
|
+
without parsing the body. Consuming receives one message at a time
|
|
10
|
+
(``receive`` -> process -> ``acknowledge``); the authoritative attempt count is the
|
|
11
|
+
envelope's ``bq-attempts`` (the body), cross-checked against the broker's native
|
|
12
|
+
``redelivery_count`` (0-based) and reconciled onto the envelope as
|
|
13
|
+
``attempts = max(body, redelivery_count)`` — no -1, because Pulsar's redelivery count is
|
|
14
|
+
0-based (0 on first delivery). A message left un-acked is redelivered (at-least-once).
|
|
15
|
+
|
|
16
|
+
This implements §5 of the broker-bindings contract. The envelope is unchanged
|
|
17
|
+
(``schema_version`` stays 1); Apache Pulsar is purely additive.
|
|
18
|
+
|
|
19
|
+
URL form: ``pulsar://host:6650`` (or ``pulsar+ssl://``). The default subscription name is
|
|
20
|
+
``babelqueue`` over a ``Shared`` subscription. For a custom client/subscription, build the
|
|
21
|
+
transport directly and pass it via ``BabelQueue(transport=...)`` or
|
|
22
|
+
``PulsarTransport(client=...)``.
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
from typing import Any, Dict, Optional
|
|
28
|
+
|
|
29
|
+
from .codec import EnvelopeCodec
|
|
30
|
+
from .transport import ReceivedMessage, Transport
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class PulsarTransport(Transport):
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
url: str = "pulsar://localhost:6650",
|
|
37
|
+
*,
|
|
38
|
+
client: Any = None,
|
|
39
|
+
subscription: str = "babelqueue",
|
|
40
|
+
consumer_type: Any = None,
|
|
41
|
+
topic_prefix: str = "",
|
|
42
|
+
receive_timeout_millis: int = 1000,
|
|
43
|
+
**subscribe_options: Any,
|
|
44
|
+
) -> None:
|
|
45
|
+
self._url = url or "pulsar://localhost:6650"
|
|
46
|
+
self._subscription = subscription
|
|
47
|
+
self._consumer_type = consumer_type
|
|
48
|
+
self._topic_prefix = topic_prefix
|
|
49
|
+
self._receive_timeout_millis = receive_timeout_millis
|
|
50
|
+
self._subscribe_options = subscribe_options
|
|
51
|
+
self._producers: Dict[str, Any] = {}
|
|
52
|
+
self._consumers: Dict[str, Any] = {}
|
|
53
|
+
|
|
54
|
+
if client is not None:
|
|
55
|
+
self._client = client
|
|
56
|
+
return
|
|
57
|
+
self._client = self._build_client() # pragma: no cover - needs Pulsar / network
|
|
58
|
+
|
|
59
|
+
def _build_client(self) -> Any: # pragma: no cover - needs Pulsar / network
|
|
60
|
+
try:
|
|
61
|
+
import pulsar
|
|
62
|
+
except ImportError as exc:
|
|
63
|
+
raise ImportError(
|
|
64
|
+
"PulsarTransport requires the 'pulsar-client' package. Install with "
|
|
65
|
+
'pip install "babelqueue[pulsar]".'
|
|
66
|
+
) from exc
|
|
67
|
+
return pulsar.Client(self._url)
|
|
68
|
+
|
|
69
|
+
# -- helpers ------------------------------------------------------------
|
|
70
|
+
|
|
71
|
+
def _topic(self, queue: str) -> str:
|
|
72
|
+
return f"{self._topic_prefix}{queue}" if self._topic_prefix else queue
|
|
73
|
+
|
|
74
|
+
def _producer(self, queue: str) -> Any:
|
|
75
|
+
producer = self._producers.get(queue)
|
|
76
|
+
if producer is None:
|
|
77
|
+
producer = self._client.create_producer(self._topic(queue))
|
|
78
|
+
self._producers[queue] = producer
|
|
79
|
+
return producer
|
|
80
|
+
|
|
81
|
+
def _consumer(self, queue: str) -> Any:
|
|
82
|
+
consumer = self._consumers.get(queue)
|
|
83
|
+
if consumer is None:
|
|
84
|
+
consumer = self._client.subscribe(
|
|
85
|
+
self._topic(queue), self._subscription, **self._subscribe_kwargs()
|
|
86
|
+
)
|
|
87
|
+
self._consumers[queue] = consumer
|
|
88
|
+
return consumer
|
|
89
|
+
|
|
90
|
+
def _subscribe_kwargs(self) -> Dict[str, Any]:
|
|
91
|
+
kwargs = dict(self._subscribe_options)
|
|
92
|
+
consumer_type = self._consumer_type
|
|
93
|
+
if consumer_type is None:
|
|
94
|
+
consumer_type = self._default_shared_type()
|
|
95
|
+
if consumer_type is not None:
|
|
96
|
+
kwargs.setdefault("consumer_type", consumer_type)
|
|
97
|
+
return kwargs
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
def _default_shared_type() -> Any:
|
|
101
|
+
try:
|
|
102
|
+
import pulsar
|
|
103
|
+
except ImportError: # pragma: no cover - exercised only with pulsar absent
|
|
104
|
+
return None
|
|
105
|
+
return pulsar.ConsumerType.Shared # pragma: no cover - needs Pulsar
|
|
106
|
+
|
|
107
|
+
@staticmethod
|
|
108
|
+
def _projection(body: str) -> Dict[str, str]:
|
|
109
|
+
"""Native Pulsar message properties (string->string) — a redundant, routable view of
|
|
110
|
+
the body: bq-job/bq-trace-id/bq-message-id + bq-schema-version/lang/attempts. §5.2."""
|
|
111
|
+
env = EnvelopeCodec.decode(body)
|
|
112
|
+
if not env:
|
|
113
|
+
return {}
|
|
114
|
+
meta = env.get("meta") or {}
|
|
115
|
+
|
|
116
|
+
props: Dict[str, str] = {}
|
|
117
|
+
if env.get("job"):
|
|
118
|
+
props["bq-job"] = str(env["job"])
|
|
119
|
+
if env.get("trace_id"):
|
|
120
|
+
props["bq-trace-id"] = str(env["trace_id"])
|
|
121
|
+
if meta.get("id"):
|
|
122
|
+
props["bq-message-id"] = str(meta["id"])
|
|
123
|
+
if meta.get("schema_version") is not None:
|
|
124
|
+
props["bq-schema-version"] = str(meta["schema_version"])
|
|
125
|
+
if meta.get("lang"):
|
|
126
|
+
props["bq-source-lang"] = str(meta["lang"])
|
|
127
|
+
props["bq-attempts"] = str(int(env.get("attempts", 0) or 0))
|
|
128
|
+
return props
|
|
129
|
+
|
|
130
|
+
@staticmethod
|
|
131
|
+
def _reconcile(body: str, redelivery_count: Any) -> str:
|
|
132
|
+
"""Set attempts to max(current, redelivery_count). Pulsar's redelivery count is
|
|
133
|
+
0-based (0 on first delivery), so it maps directly to attempts with no -1. The
|
|
134
|
+
runtime retries by republishing with attempts+1 in the body (redelivery count back to
|
|
135
|
+
0), so a republished message must not have its higher body count lowered."""
|
|
136
|
+
try:
|
|
137
|
+
rc = int(redelivery_count)
|
|
138
|
+
except (ValueError, TypeError):
|
|
139
|
+
return body
|
|
140
|
+
if rc <= 0:
|
|
141
|
+
return body
|
|
142
|
+
env = EnvelopeCodec.decode(body)
|
|
143
|
+
if not env or rc <= int(env.get("attempts", 0) or 0):
|
|
144
|
+
return body
|
|
145
|
+
env["attempts"] = rc
|
|
146
|
+
return EnvelopeCodec.encode(env)
|
|
147
|
+
|
|
148
|
+
@staticmethod
|
|
149
|
+
def _redelivery_count(message: Any) -> int:
|
|
150
|
+
getter = getattr(message, "redelivery_count", None)
|
|
151
|
+
value = getter() if callable(getter) else getter
|
|
152
|
+
if value is None:
|
|
153
|
+
return 0
|
|
154
|
+
try:
|
|
155
|
+
return int(value)
|
|
156
|
+
except (TypeError, ValueError):
|
|
157
|
+
return 0
|
|
158
|
+
|
|
159
|
+
@staticmethod
|
|
160
|
+
def _payload(message: Any) -> str:
|
|
161
|
+
getter = getattr(message, "data", None)
|
|
162
|
+
data = getter() if callable(getter) else getter
|
|
163
|
+
if isinstance(data, bytes):
|
|
164
|
+
return data.decode("utf-8")
|
|
165
|
+
return str(data) if data is not None else ""
|
|
166
|
+
|
|
167
|
+
# -- Transport ----------------------------------------------------------
|
|
168
|
+
|
|
169
|
+
def publish(self, queue: str, body: str) -> None:
|
|
170
|
+
self._producer(queue).send(body.encode("utf-8"), properties=self._projection(body))
|
|
171
|
+
|
|
172
|
+
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
173
|
+
wait = self._receive_timeout_millis
|
|
174
|
+
if timeout and timeout > 0:
|
|
175
|
+
wait = int(timeout * 1000)
|
|
176
|
+
consumer = self._consumer(queue)
|
|
177
|
+
try:
|
|
178
|
+
message = consumer.receive(timeout_millis=wait)
|
|
179
|
+
except Exception as exc: # noqa: BLE001 - Pulsar raises pulsar.Timeout on no message
|
|
180
|
+
if type(exc).__name__ == "Timeout":
|
|
181
|
+
return None
|
|
182
|
+
raise
|
|
183
|
+
if message is None: # pragma: no cover - defensive (real client raises instead)
|
|
184
|
+
return None
|
|
185
|
+
body = self._reconcile(self._payload(message), self._redelivery_count(message))
|
|
186
|
+
return ReceivedMessage(body=body, queue=queue, handle=message)
|
|
187
|
+
|
|
188
|
+
def ack(self, message: ReceivedMessage) -> None:
|
|
189
|
+
if message.handle is None:
|
|
190
|
+
return
|
|
191
|
+
self._consumer(message.queue).acknowledge(message.handle)
|
|
192
|
+
|
|
193
|
+
def close(self) -> None: # pragma: no cover - resource cleanup
|
|
194
|
+
for resource in (*self._producers.values(), *self._consumers.values()):
|
|
195
|
+
try:
|
|
196
|
+
resource.close()
|
|
197
|
+
except Exception: # noqa: BLE001 - best-effort cleanup
|
|
198
|
+
pass
|
|
199
|
+
try:
|
|
200
|
+
self._client.close()
|
|
201
|
+
except Exception: # noqa: BLE001 - best-effort cleanup
|
|
202
|
+
pass
|
|
@@ -87,8 +87,13 @@ def make_transport(broker_url: str) -> Transport:
|
|
|
87
87
|
from .asb_transport import AsbTransport
|
|
88
88
|
|
|
89
89
|
return AsbTransport(broker_url)
|
|
90
|
+
if scheme in ("pulsar", "pulsar+ssl"):
|
|
91
|
+
from .pulsar_transport import PulsarTransport
|
|
92
|
+
|
|
93
|
+
return PulsarTransport(broker_url)
|
|
90
94
|
|
|
91
95
|
raise BabelQueueError(
|
|
92
96
|
f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://', "
|
|
93
|
-
"'amqp://', 'sqs://' or '
|
|
97
|
+
"'amqp://', 'sqs://', 'sb://' or 'pulsar://', or pass your own Transport via "
|
|
98
|
+
"BabelQueue(transport=...)."
|
|
94
99
|
)
|
|
@@ -122,5 +122,31 @@
|
|
|
122
122
|
{ "name": "zero-count-keeps-body", "body_attempts": 3, "delivery_count": 0, "expected_attempts": 3 }
|
|
123
123
|
]
|
|
124
124
|
}
|
|
125
|
+
},
|
|
126
|
+
"pulsar": {
|
|
127
|
+
"description": "Apache Pulsar binding conformance (broker-bindings.md §5). Every SDK that ships a Pulsar transport must satisfy these. The envelope body stays byte-identical (the 'cases' above); these lock the native projection + reconciliation the binding adds. Per-message values reuse fixtures/order-created.json so the expected projection is deterministic.",
|
|
128
|
+
"property_projection": {
|
|
129
|
+
"description": "On produce, the transport MUST project these native Pulsar message properties from the envelope, all string->string (Pulsar properties are string-typed, so numbers are stringified): bq-job = job (the URN), bq-trace-id = trace_id, bq-message-id = meta.id, bq-schema-version = str(meta.schema_version), bq-source-lang = meta.lang, bq-attempts = str(attempts). The payload is the byte-identical envelope; the native publishTime mirrors meta.created_at (broker-set, body authoritative). Applies to every Pulsar-producing SDK.",
|
|
130
|
+
"envelope_file": "fixtures/order-created.json",
|
|
131
|
+
"properties": {
|
|
132
|
+
"bq-job": "urn:babel:orders:created",
|
|
133
|
+
"bq-trace-id": "7b3f9c2a-e41d-4f88-9b2a-1c0d5e6f7a8b",
|
|
134
|
+
"bq-message-id": "f1e2d3c4-b5a6-4789-90ab-cdef01234567",
|
|
135
|
+
"bq-schema-version": "1",
|
|
136
|
+
"bq-source-lang": "php",
|
|
137
|
+
"bq-attempts": "0"
|
|
138
|
+
}
|
|
139
|
+
},
|
|
140
|
+
"attempts_reconciliation": {
|
|
141
|
+
"description": "On consume, attempts = max(body.attempts, RedeliveryCount): Pulsar's RedeliveryCount is 0-based (0 on first delivery) so it maps directly with NO -1, a runtime-incremented body count is never lowered, and RedeliveryCount 0 leaves the body's own count untouched (the runtime retries by republishing with attempts+1, which resets the broker's redelivery count to 0). The rule is identical across the native-consumer SDKs (.NET/Java/Node) and the Transport+App SDKs (Python/Go).",
|
|
142
|
+
"cases": [
|
|
143
|
+
{ "name": "first-delivery", "body_attempts": 0, "redelivery_count": 0, "expected_attempts": 0 },
|
|
144
|
+
{ "name": "third-delivery", "body_attempts": 0, "redelivery_count": 2, "expected_attempts": 2 },
|
|
145
|
+
{ "name": "native-exceeds-body", "body_attempts": 2, "redelivery_count": 5, "expected_attempts": 5 },
|
|
146
|
+
{ "name": "never-lower-runtime", "body_attempts": 5, "redelivery_count": 1, "expected_attempts": 5 },
|
|
147
|
+
{ "name": "first-delivery-keeps-body", "body_attempts": 4, "redelivery_count": 0, "expected_attempts": 4 },
|
|
148
|
+
{ "name": "native-equals-body", "body_attempts": 3, "redelivery_count": 3, "expected_attempts": 3 }
|
|
149
|
+
]
|
|
150
|
+
}
|
|
125
151
|
}
|
|
126
152
|
}
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
"""Apache Pulsar binding conformance: the vendored manifest's ``pulsar`` block locks the §5
|
|
2
|
+
property projection (bq-* string->string) and the ``attempts = max(body, redelivery_count)``
|
|
3
|
+
reconciliation (no -1, Pulsar's redelivery count is 0-based). No pulsar-client, no broker —
|
|
4
|
+
pure transport logic against golden values."""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import json
|
|
9
|
+
import os
|
|
10
|
+
import unittest
|
|
11
|
+
|
|
12
|
+
from babelqueue import EnvelopeCodec
|
|
13
|
+
from babelqueue.pulsar_transport import PulsarTransport
|
|
14
|
+
|
|
15
|
+
CONFORMANCE = os.path.join(os.path.dirname(__file__), "conformance")
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class PulsarConformanceTest(unittest.TestCase):
|
|
19
|
+
@classmethod
|
|
20
|
+
def setUpClass(cls) -> None:
|
|
21
|
+
with open(os.path.join(CONFORMANCE, "manifest.json"), encoding="utf-8") as fh:
|
|
22
|
+
cls.pulsar = json.load(fh)["pulsar"]
|
|
23
|
+
|
|
24
|
+
def test_property_projection(self) -> None:
|
|
25
|
+
projection = self.pulsar["property_projection"]
|
|
26
|
+
with open(os.path.join(CONFORMANCE, projection["envelope_file"]), encoding="utf-8") as fh:
|
|
27
|
+
body = fh.read()
|
|
28
|
+
|
|
29
|
+
got = PulsarTransport._projection(body)
|
|
30
|
+
self.assertEqual(got, projection["properties"])
|
|
31
|
+
|
|
32
|
+
def test_attempts_reconciliation(self) -> None:
|
|
33
|
+
for case in self.pulsar["attempts_reconciliation"]["cases"]:
|
|
34
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1})
|
|
35
|
+
env["attempts"] = case["body_attempts"]
|
|
36
|
+
body = EnvelopeCodec.encode(env)
|
|
37
|
+
|
|
38
|
+
out = PulsarTransport._reconcile(body, case["redelivery_count"])
|
|
39
|
+
|
|
40
|
+
self.assertEqual(
|
|
41
|
+
EnvelopeCodec.decode(out)["attempts"],
|
|
42
|
+
case["expected_attempts"],
|
|
43
|
+
case["name"],
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
if __name__ == "__main__": # pragma: no cover
|
|
48
|
+
unittest.main()
|
|
@@ -0,0 +1,215 @@
|
|
|
1
|
+
"""Apache Pulsar transport tests.
|
|
2
|
+
|
|
3
|
+
The property projection and the ``attempts = max(body, redelivery_count)`` reconciliation —
|
|
4
|
+
and the full publish / pop / ack flow — run with no broker and no ``pulsar-client`` (the
|
|
5
|
+
transport's pulsar import is lazy and the publish path sends raw bytes, so a duck-typed fake
|
|
6
|
+
client covers everything). Only ``_build_client`` needs the real package.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import unittest
|
|
12
|
+
|
|
13
|
+
from babelqueue import EnvelopeCodec
|
|
14
|
+
from babelqueue.pulsar_transport import PulsarTransport
|
|
15
|
+
from babelqueue.transport import ReceivedMessage, make_transport
|
|
16
|
+
|
|
17
|
+
try:
|
|
18
|
+
import pulsar # noqa: F401
|
|
19
|
+
|
|
20
|
+
HAVE_PULSAR = True
|
|
21
|
+
except ImportError:
|
|
22
|
+
HAVE_PULSAR = False
|
|
23
|
+
|
|
24
|
+
URN = "urn:babel:orders:created"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def body(attempts: int = 0) -> str:
|
|
28
|
+
env = EnvelopeCodec.decode(
|
|
29
|
+
EnvelopeCodec.encode(EnvelopeCodec.make(URN, {"order_id": 1042}, queue="orders"))
|
|
30
|
+
)
|
|
31
|
+
env["attempts"] = attempts
|
|
32
|
+
return EnvelopeCodec.encode(env)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Timeout(Exception):
|
|
36
|
+
"""Duck-typed stand-in for ``pulsar.Timeout`` (matched by class name)."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class FakeMessage:
|
|
40
|
+
"""Duck-typed pulsar Message: data()/redelivery_count() are callables."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, raw: str, redelivery_count: int = 0) -> None:
|
|
43
|
+
self._raw = raw
|
|
44
|
+
self._rc = redelivery_count
|
|
45
|
+
|
|
46
|
+
def data(self) -> bytes:
|
|
47
|
+
return self._raw.encode("utf-8")
|
|
48
|
+
|
|
49
|
+
def redelivery_count(self) -> int:
|
|
50
|
+
return self._rc
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class FakeProducer:
|
|
54
|
+
def __init__(self) -> None:
|
|
55
|
+
self.sent: list = []
|
|
56
|
+
|
|
57
|
+
def send(self, content, properties=None) -> None:
|
|
58
|
+
self.sent.append((content, properties))
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class FakeConsumer:
|
|
62
|
+
def __init__(self, messages=()) -> None:
|
|
63
|
+
self._messages = list(messages)
|
|
64
|
+
self.acked: list = []
|
|
65
|
+
self.subscribe_kwargs: dict = {}
|
|
66
|
+
|
|
67
|
+
def receive(self, timeout_millis=None):
|
|
68
|
+
if not self._messages:
|
|
69
|
+
raise Timeout()
|
|
70
|
+
return self._messages.pop(0)
|
|
71
|
+
|
|
72
|
+
def acknowledge(self, message) -> None:
|
|
73
|
+
self.acked.append(message)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class FakeClient:
|
|
77
|
+
def __init__(self, producer=None, consumer=None) -> None:
|
|
78
|
+
self.producer = producer if producer is not None else FakeProducer()
|
|
79
|
+
self.consumer = consumer if consumer is not None else FakeConsumer()
|
|
80
|
+
self.subscribed: list = []
|
|
81
|
+
|
|
82
|
+
def create_producer(self, topic):
|
|
83
|
+
return self.producer
|
|
84
|
+
|
|
85
|
+
def subscribe(self, topic, subscription, **kwargs):
|
|
86
|
+
self.subscribed.append((topic, subscription, kwargs))
|
|
87
|
+
self.consumer.subscribe_kwargs = kwargs
|
|
88
|
+
return self.consumer
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class PulsarProjectionTest(unittest.TestCase):
|
|
92
|
+
def test_projection_maps_contract_properties(self):
|
|
93
|
+
raw = body()
|
|
94
|
+
env = EnvelopeCodec.decode(raw)
|
|
95
|
+
props = PulsarTransport._projection(raw)
|
|
96
|
+
|
|
97
|
+
self.assertEqual(props["bq-job"], URN)
|
|
98
|
+
self.assertEqual(props["bq-trace-id"], env["trace_id"])
|
|
99
|
+
self.assertEqual(props["bq-message-id"], env["meta"]["id"])
|
|
100
|
+
self.assertEqual(props["bq-schema-version"], str(env["meta"]["schema_version"]))
|
|
101
|
+
self.assertEqual(props["bq-source-lang"], env["meta"]["lang"])
|
|
102
|
+
self.assertEqual(props["bq-attempts"], "0")
|
|
103
|
+
|
|
104
|
+
def test_projection_values_are_all_strings(self):
|
|
105
|
+
props = PulsarTransport._projection(body(3))
|
|
106
|
+
self.assertTrue(all(isinstance(v, str) for v in props.values()))
|
|
107
|
+
self.assertEqual(props["bq-attempts"], "3")
|
|
108
|
+
|
|
109
|
+
def test_projection_of_garbage_is_empty(self):
|
|
110
|
+
self.assertEqual(PulsarTransport._projection("not-json"), {})
|
|
111
|
+
|
|
112
|
+
def test_reconcile_attempts_is_redelivery_count(self):
|
|
113
|
+
out = PulsarTransport._reconcile(body(0), 2)
|
|
114
|
+
self.assertEqual(EnvelopeCodec.decode(out)["attempts"], 2)
|
|
115
|
+
|
|
116
|
+
def test_reconcile_first_delivery_is_zero(self):
|
|
117
|
+
out = PulsarTransport._reconcile(body(0), 0)
|
|
118
|
+
self.assertEqual(EnvelopeCodec.decode(out)["attempts"], 0)
|
|
119
|
+
|
|
120
|
+
def test_reconcile_ignores_garbage_count(self):
|
|
121
|
+
raw = body(5)
|
|
122
|
+
self.assertEqual(PulsarTransport._reconcile(raw, "not-a-number"), raw)
|
|
123
|
+
|
|
124
|
+
def test_reconcile_never_lowers_runtime_count(self):
|
|
125
|
+
# The runtime retries by republishing with attempts+1 (redelivery count back to 0),
|
|
126
|
+
# so a higher body count must win over a lower native one.
|
|
127
|
+
raw = body(5)
|
|
128
|
+
self.assertEqual(int(EnvelopeCodec.decode(PulsarTransport._reconcile(raw, 1))["attempts"]), 5)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class PulsarPublishTest(unittest.TestCase):
|
|
132
|
+
def test_publish_sends_payload_with_property_projection(self):
|
|
133
|
+
raw = body()
|
|
134
|
+
env = EnvelopeCodec.decode(raw)
|
|
135
|
+
producer = FakeProducer()
|
|
136
|
+
tr = PulsarTransport(client=FakeClient(producer=producer))
|
|
137
|
+
|
|
138
|
+
tr.publish("orders", raw)
|
|
139
|
+
|
|
140
|
+
self.assertEqual(len(producer.sent), 1)
|
|
141
|
+
content, props = producer.sent[0]
|
|
142
|
+
self.assertEqual(content, raw.encode("utf-8"))
|
|
143
|
+
self.assertEqual(props["bq-job"], URN)
|
|
144
|
+
self.assertEqual(props["bq-message-id"], env["meta"]["id"])
|
|
145
|
+
self.assertEqual(EnvelopeCodec.urn(EnvelopeCodec.decode(content.decode("utf-8"))), URN)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
class PulsarConsumeTest(unittest.TestCase):
|
|
149
|
+
def test_pop_reconciles_attempts_and_returns_handle(self):
|
|
150
|
+
message = FakeMessage(body(0), redelivery_count=2)
|
|
151
|
+
tr = PulsarTransport(client=FakeClient(consumer=FakeConsumer([message])))
|
|
152
|
+
|
|
153
|
+
msg = tr.pop("orders")
|
|
154
|
+
|
|
155
|
+
self.assertIsNotNone(msg)
|
|
156
|
+
self.assertEqual(msg.queue, "orders")
|
|
157
|
+
self.assertIs(msg.handle, message)
|
|
158
|
+
self.assertEqual(EnvelopeCodec.decode(msg.body)["attempts"], 2)
|
|
159
|
+
|
|
160
|
+
def test_pop_empty_returns_none_on_timeout(self):
|
|
161
|
+
tr = PulsarTransport(client=FakeClient(consumer=FakeConsumer([])))
|
|
162
|
+
self.assertIsNone(tr.pop("orders"))
|
|
163
|
+
|
|
164
|
+
def test_pop_passes_timeout_to_receive(self):
|
|
165
|
+
consumer = FakeConsumer([FakeMessage(body(), 0)])
|
|
166
|
+
captured = {}
|
|
167
|
+
original = consumer.receive
|
|
168
|
+
|
|
169
|
+
def spy(timeout_millis=None):
|
|
170
|
+
captured["timeout_millis"] = timeout_millis
|
|
171
|
+
return original(timeout_millis=timeout_millis)
|
|
172
|
+
|
|
173
|
+
consumer.receive = spy
|
|
174
|
+
tr = PulsarTransport(client=FakeClient(consumer=consumer))
|
|
175
|
+
tr.pop("orders", timeout=2.0)
|
|
176
|
+
self.assertEqual(captured["timeout_millis"], 2000)
|
|
177
|
+
|
|
178
|
+
def test_ack_acknowledges_the_reserved_message(self):
|
|
179
|
+
message = FakeMessage(body(), 0)
|
|
180
|
+
consumer = FakeConsumer([message])
|
|
181
|
+
tr = PulsarTransport(client=FakeClient(consumer=consumer))
|
|
182
|
+
|
|
183
|
+
tr.ack(tr.pop("orders"))
|
|
184
|
+
|
|
185
|
+
self.assertEqual(consumer.acked, [message])
|
|
186
|
+
|
|
187
|
+
def test_ack_noop_without_handle(self):
|
|
188
|
+
tr = PulsarTransport(client=FakeClient())
|
|
189
|
+
tr.ack(ReceivedMessage(body="", queue="orders", handle=None)) # no error
|
|
190
|
+
|
|
191
|
+
def test_explicit_consumer_type_is_forwarded_to_subscribe(self):
|
|
192
|
+
client = FakeClient()
|
|
193
|
+
tr = PulsarTransport(client=client, consumer_type="shared-sentinel")
|
|
194
|
+
tr.pop("orders")
|
|
195
|
+
self.assertEqual(client.subscribed[0][2].get("consumer_type"), "shared-sentinel")
|
|
196
|
+
|
|
197
|
+
def test_topic_prefix_is_applied(self):
|
|
198
|
+
client = FakeClient()
|
|
199
|
+
tr = PulsarTransport(client=client, topic_prefix="persistent://public/default/")
|
|
200
|
+
tr.publish("orders", body())
|
|
201
|
+
# The producer is cached per queue; subscribe/produce use the prefixed topic.
|
|
202
|
+
tr.pop("orders")
|
|
203
|
+
self.assertEqual(client.subscribed[0][0], "persistent://public/default/orders")
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
class PulsarSchemeTest(unittest.TestCase):
|
|
207
|
+
def test_make_transport_routes_pulsar_scheme(self):
|
|
208
|
+
if HAVE_PULSAR:
|
|
209
|
+
self.skipTest("pulsar-client installed — building a real client would need a broker")
|
|
210
|
+
with self.assertRaises(ImportError):
|
|
211
|
+
make_transport("pulsar://localhost:6650")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
if __name__ == "__main__":
|
|
215
|
+
unittest.main()
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.2.0 → babelqueue-1.3.0}/src/babelqueue/django/management/commands/babelqueue_worker.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.2.0 → babelqueue-1.3.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|