babelqueue 1.0.0__tar.gz → 1.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- babelqueue-1.2.0/.github/FUNDING.yml +2 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/.github/workflows/ci.yml +23 -3
- {babelqueue-1.0.0 → babelqueue-1.2.0}/CHANGELOG.md +34 -2
- {babelqueue-1.0.0 → babelqueue-1.2.0}/PKG-INFO +6 -1
- {babelqueue-1.0.0 → babelqueue-1.2.0}/pyproject.toml +3 -1
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/__init__.py +1 -1
- babelqueue-1.2.0/src/babelqueue/asb_transport.py +173 -0
- babelqueue-1.2.0/src/babelqueue/sqs_transport.py +209 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/transport.py +10 -2
- babelqueue-1.2.0/tests/conformance/manifest.json +126 -0
- babelqueue-1.2.0/tests/test_asb_conformance.py +54 -0
- babelqueue-1.2.0/tests/test_asb_transport.py +169 -0
- babelqueue-1.2.0/tests/test_sqs_conformance.py +53 -0
- babelqueue-1.2.0/tests/test_sqs_transport.py +299 -0
- babelqueue-1.0.0/tests/conformance/manifest.json +0 -71
- {babelqueue-1.0.0 → babelqueue-1.2.0}/.github/workflows/release.yml +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/.gitignore +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/LICENSE +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/README.md +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/app.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/exceptions.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/redis_transport.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_app.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_celery.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_codec.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_conformance.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_django.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_overhead.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.2.0}/tests/test_redis_transport.py +0 -0
|
@@ -44,14 +44,14 @@ jobs:
|
|
|
44
44
|
- name: Install (dev + all adapters for type context)
|
|
45
45
|
run: |
|
|
46
46
|
python -m pip install --upgrade pip
|
|
47
|
-
pip install -e ".[dev,celery,django,redis,amqp]"
|
|
47
|
+
pip install -e ".[dev,celery,django,redis,amqp,sqs,azureservicebus]"
|
|
48
48
|
- name: Ruff
|
|
49
49
|
run: ruff check src tests
|
|
50
50
|
- name: Mypy
|
|
51
51
|
run: mypy
|
|
52
52
|
|
|
53
53
|
integration:
|
|
54
|
-
name: Redis
|
|
54
|
+
name: Broker integration (Redis · RabbitMQ · SQS)
|
|
55
55
|
runs-on: ubuntu-latest
|
|
56
56
|
services:
|
|
57
57
|
redis:
|
|
@@ -72,6 +72,12 @@ jobs:
|
|
|
72
72
|
--health-interval 10s
|
|
73
73
|
--health-timeout 5s
|
|
74
74
|
--health-retries 15
|
|
75
|
+
# Free, SQS-compatible broker for the SqsTransport round-trip. The native
|
|
76
|
+
# image is shell-less, so readiness is polled by a TCP-probe step below.
|
|
77
|
+
elasticmq:
|
|
78
|
+
image: softwaremill/elasticmq-native:1.6.11
|
|
79
|
+
ports:
|
|
80
|
+
- 9324:9324
|
|
75
81
|
steps:
|
|
76
82
|
- uses: actions/checkout@v5
|
|
77
83
|
|
|
@@ -83,12 +89,26 @@ jobs:
|
|
|
83
89
|
- name: Install (all adapters — full coverage with brokers)
|
|
84
90
|
run: |
|
|
85
91
|
python -m pip install --upgrade pip
|
|
86
|
-
pip install -e ".[redis,amqp,celery,django,dev]"
|
|
92
|
+
pip install -e ".[redis,amqp,sqs,azureservicebus,celery,django,dev]"
|
|
93
|
+
|
|
94
|
+
- name: Wait for ElasticMQ
|
|
95
|
+
run: |
|
|
96
|
+
for i in $(seq 1 30); do
|
|
97
|
+
if (echo > /dev/tcp/localhost/9324) >/dev/null 2>&1; then
|
|
98
|
+
echo "ElasticMQ port 9324 open"; exit 0
|
|
99
|
+
fi
|
|
100
|
+
sleep 2
|
|
101
|
+
done
|
|
102
|
+
echo "ElasticMQ did not open port 9324"; exit 1
|
|
87
103
|
|
|
88
104
|
- name: Run full suite with coverage gate (>=90%)
|
|
89
105
|
env:
|
|
90
106
|
BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
|
|
91
107
|
BABELQUEUE_TEST_AMQP: amqp://guest:guest@localhost:5672/
|
|
108
|
+
SQS_ENDPOINT: http://localhost:9324
|
|
109
|
+
AWS_REGION: us-east-1
|
|
110
|
+
AWS_ACCESS_KEY_ID: test
|
|
111
|
+
AWS_SECRET_ACCESS_KEY: test
|
|
92
112
|
run: pytest --cov=babelqueue --cov-report=term-missing --cov-fail-under=90
|
|
93
113
|
|
|
94
114
|
conformance:
|
|
@@ -7,7 +7,37 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
The envelope wire format is versioned separately by `meta.schema_version`
|
|
8
8
|
(currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
|
|
9
9
|
|
|
10
|
-
## [
|
|
10
|
+
## [1.2.0] - 2026-06-13
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **Azure Service Bus transport** (`babelqueue[azureservicebus]`, `azure-servicebus`) —
|
|
14
|
+
`AsbTransport`, selected by the `sb://` URL scheme (e.g.
|
|
15
|
+
`sb://<namespace>.servicebus.windows.net`, Azure AD via `DefaultAzureCredential`; or pass
|
|
16
|
+
`connection_string=...` / a built `client`). Implements [§4 of the broker-bindings
|
|
17
|
+
contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#azure-service-bus): the
|
|
18
|
+
canonical envelope is the message body, projected onto native Service Bus fields
|
|
19
|
+
(`Subject` = URN, `CorrelationId` = `trace_id`, `MessageId` = `meta.id`, plus the `bq-`
|
|
20
|
+
application properties); PeekLock reserve → `complete_message` ack; `attempts` reconciled
|
|
21
|
+
to the broker-authoritative `DeliveryCount − 1`. A client can be injected for tests/DI. The
|
|
22
|
+
projection + reconciliation are unit-tested with no broker and no `azure-servicebus` (the
|
|
23
|
+
azure import is lazy); the publish flow is covered with a fake client. The envelope is
|
|
24
|
+
unchanged (`schema_version: 1`); Azure Service Bus is purely additive. Ships as a MINOR.
|
|
25
|
+
|
|
26
|
+
## [1.1.0] - 2026-06-12
|
|
27
|
+
|
|
28
|
+
### Added
|
|
29
|
+
- **Amazon SQS transport** (`babelqueue[sqs]`, `boto3`) — `SqsTransport`, selected by
|
|
30
|
+
the `sqs://` URL scheme (e.g. `sqs://us-east-1?endpoint=http://localhost:4566` for
|
|
31
|
+
LocalStack). Implements [§3 of the broker-bindings contract](https://babelqueue.com):
|
|
32
|
+
the canonical envelope is the `MessageBody`, projected onto native `MessageAttributes`
|
|
33
|
+
(`bq-job`/`bq-trace-id`/`bq-message-id`/`bq-schema-version`/`bq-source-lang`/`bq-created-at`);
|
|
34
|
+
visibility-timeout reserve → `delete_message` ack; `attempts` reconciled to
|
|
35
|
+
`ApproximateReceiveCount − 1` (never lowering a runtime-incremented count). Supports
|
|
36
|
+
FIFO (`MessageGroupId`/`MessageDeduplicationId` = `meta.id`), content-based dedup,
|
|
37
|
+
configurable wait/visibility, and a `queue_url_prefix` that skips `GetQueueUrl`. A
|
|
38
|
+
client can be injected for tests/DI. 16 unit tests run against a fake client (no boto3,
|
|
39
|
+
no broker); a LocalStack integration test round-trips the real `boto3` path. The
|
|
40
|
+
envelope is unchanged (`schema_version: 1`); SQS is purely additive. Ships as a MINOR.
|
|
11
41
|
|
|
12
42
|
## [1.0.0] - 2026-06-07
|
|
13
43
|
|
|
@@ -85,7 +115,9 @@ reference at [babelqueue.com](https://babelqueue.com).
|
|
|
85
115
|
- Pre-1.0: the public API may change before the `1.0.0` tag.
|
|
86
116
|
- The core has **zero runtime dependencies** (standard library only); Python `>=3.9`.
|
|
87
117
|
|
|
88
|
-
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.
|
|
118
|
+
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.2.0...HEAD
|
|
119
|
+
[1.2.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.1.0...v1.2.0
|
|
120
|
+
[1.1.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.0.0...v1.1.0
|
|
89
121
|
[1.0.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.5.0...v1.0.0
|
|
90
122
|
[0.5.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.4.0...v0.5.0
|
|
91
123
|
[0.4.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.3.0...v0.4.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.2.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -24,6 +24,9 @@ Classifier: Typing :: Typed
|
|
|
24
24
|
Requires-Python: >=3.9
|
|
25
25
|
Provides-Extra: amqp
|
|
26
26
|
Requires-Dist: pika>=1.3; extra == 'amqp'
|
|
27
|
+
Provides-Extra: azureservicebus
|
|
28
|
+
Requires-Dist: azure-identity>=1.15; extra == 'azureservicebus'
|
|
29
|
+
Requires-Dist: azure-servicebus>=7.11; extra == 'azureservicebus'
|
|
27
30
|
Provides-Extra: celery
|
|
28
31
|
Requires-Dist: celery>=5; extra == 'celery'
|
|
29
32
|
Provides-Extra: dev
|
|
@@ -35,6 +38,8 @@ Provides-Extra: django
|
|
|
35
38
|
Requires-Dist: django>=4.2; extra == 'django'
|
|
36
39
|
Provides-Extra: redis
|
|
37
40
|
Requires-Dist: redis>=4; extra == 'redis'
|
|
41
|
+
Provides-Extra: sqs
|
|
42
|
+
Requires-Dist: boto3>=1.26; extra == 'sqs'
|
|
38
43
|
Description-Content-Type: text/markdown
|
|
39
44
|
|
|
40
45
|
# BabelQueue for Python
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.2.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -31,6 +31,8 @@ dependencies = []
|
|
|
31
31
|
# Optional runtime drivers + framework adapters — standard, zero-heavy-dep.
|
|
32
32
|
redis = ["redis>=4"]
|
|
33
33
|
amqp = ["pika>=1.3"]
|
|
34
|
+
sqs = ["boto3>=1.26"]
|
|
35
|
+
azureservicebus = ["azure-servicebus>=7.11", "azure-identity>=1.15"]
|
|
34
36
|
celery = ["celery>=5"]
|
|
35
37
|
django = ["django>=4.2"]
|
|
36
38
|
dev = ["pytest>=7", "pytest-cov>=4", "mypy>=1.8", "ruff>=0.5"]
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
"""Azure Service Bus transport. Requires the ``azureservicebus`` extra:
|
|
2
|
+
|
|
3
|
+
pip install "babelqueue[azureservicebus]"
|
|
4
|
+
|
|
5
|
+
Producing sends the canonical envelope as the message body and projects the contract
|
|
6
|
+
envelope fields onto native Service Bus fields — ``Subject`` = URN, ``CorrelationId`` =
|
|
7
|
+
trace_id, ``MessageId`` = meta.id, plus the ``bq-`` application properties
|
|
8
|
+
(``bq-schema-version`` / ``bq-source-lang`` / ``bq-created-at``) — so a .NET/Java/... peer
|
|
9
|
+
can route on ``Subject`` and correlate on ``CorrelationId`` without parsing the body.
|
|
10
|
+
Consuming uses the PeekLock reservation model (``receive_messages`` -> process ->
|
|
11
|
+
``complete_message``); the authoritative attempt count is the broker's native
|
|
12
|
+
``DeliveryCount`` (1-based), reconciled onto the envelope as ``attempts = DeliveryCount - 1``.
|
|
13
|
+
A message left un-acked has its lock expire and is redelivered (at-least-once).
|
|
14
|
+
|
|
15
|
+
This implements §4 of the broker-bindings contract. The envelope is unchanged
|
|
16
|
+
(``schema_version`` stays 1); Azure Service Bus is purely additive.
|
|
17
|
+
|
|
18
|
+
URL form: ``sb://<namespace>.servicebus.windows.net`` (Azure AD via
|
|
19
|
+
``DefaultAzureCredential``). For connection-string auth or a custom client, build the
|
|
20
|
+
transport directly and pass it via ``BabelQueue(transport=...)`` or
|
|
21
|
+
``AsbTransport(connection_string=...)``.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
from typing import Any, Dict, Optional
|
|
27
|
+
from urllib.parse import urlsplit
|
|
28
|
+
|
|
29
|
+
from .codec import EnvelopeCodec
|
|
30
|
+
from .transport import ReceivedMessage, Transport
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class AsbTransport(Transport):
|
|
34
|
+
def __init__(
|
|
35
|
+
self,
|
|
36
|
+
url: str = "sb://",
|
|
37
|
+
*,
|
|
38
|
+
client: Any = None,
|
|
39
|
+
connection_string: Optional[str] = None,
|
|
40
|
+
credential: Any = None,
|
|
41
|
+
max_wait_time: Optional[float] = None,
|
|
42
|
+
) -> None:
|
|
43
|
+
parts = urlsplit(url) if url else urlsplit("sb://")
|
|
44
|
+
self._namespace = parts.hostname or None
|
|
45
|
+
self._connection_string = connection_string
|
|
46
|
+
self._credential = credential
|
|
47
|
+
self._max_wait_time = max_wait_time
|
|
48
|
+
self._senders: Dict[str, Any] = {}
|
|
49
|
+
self._receivers: Dict[str, Any] = {}
|
|
50
|
+
|
|
51
|
+
if client is not None:
|
|
52
|
+
self._client = client
|
|
53
|
+
return
|
|
54
|
+
self._client = self._build_client() # pragma: no cover - needs Azure / network
|
|
55
|
+
|
|
56
|
+
def _build_client(self) -> Any: # pragma: no cover - needs Azure / network
|
|
57
|
+
try:
|
|
58
|
+
from azure.servicebus import ServiceBusClient
|
|
59
|
+
except ImportError as exc:
|
|
60
|
+
raise ImportError(
|
|
61
|
+
"AsbTransport requires the 'azure-servicebus' package. Install with "
|
|
62
|
+
'pip install "babelqueue[azureservicebus]".'
|
|
63
|
+
) from exc
|
|
64
|
+
import os
|
|
65
|
+
|
|
66
|
+
cs = self._connection_string or os.environ.get("AZURE_SERVICEBUS_CONNECTION_STRING")
|
|
67
|
+
if cs:
|
|
68
|
+
return ServiceBusClient.from_connection_string(cs)
|
|
69
|
+
if self._namespace:
|
|
70
|
+
credential = self._credential
|
|
71
|
+
if credential is None:
|
|
72
|
+
from azure.identity import DefaultAzureCredential
|
|
73
|
+
|
|
74
|
+
credential = DefaultAzureCredential()
|
|
75
|
+
return ServiceBusClient(self._namespace, credential)
|
|
76
|
+
raise ValueError(
|
|
77
|
+
"AsbTransport needs a connection string, a namespace + credential, or an injected client."
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
# -- helpers ------------------------------------------------------------
|
|
81
|
+
|
|
82
|
+
def _sender(self, queue: str) -> Any:
|
|
83
|
+
sender = self._senders.get(queue)
|
|
84
|
+
if sender is None:
|
|
85
|
+
sender = self._client.get_queue_sender(queue)
|
|
86
|
+
self._senders[queue] = sender
|
|
87
|
+
return sender
|
|
88
|
+
|
|
89
|
+
def _receiver(self, queue: str) -> Any:
|
|
90
|
+
receiver = self._receivers.get(queue)
|
|
91
|
+
if receiver is None:
|
|
92
|
+
receiver = self._client.get_queue_receiver(queue)
|
|
93
|
+
self._receivers[queue] = receiver
|
|
94
|
+
return receiver
|
|
95
|
+
|
|
96
|
+
@staticmethod
|
|
97
|
+
def _projection(body: str) -> Dict[str, Any]:
|
|
98
|
+
"""Native ServiceBusMessage kwargs — Subject/CorrelationId/MessageId + the bq-
|
|
99
|
+
application properties (a redundant, routable view of the body). §4.2–§4.3."""
|
|
100
|
+
try:
|
|
101
|
+
env: Dict[str, Any] = EnvelopeCodec.decode(body)
|
|
102
|
+
except (ValueError, TypeError): # pragma: no cover - defensive
|
|
103
|
+
return {"content_type": "application/json"}
|
|
104
|
+
meta = env.get("meta") or {}
|
|
105
|
+
|
|
106
|
+
props: Dict[str, Any] = {}
|
|
107
|
+
if meta.get("schema_version") is not None:
|
|
108
|
+
props["bq-schema-version"] = meta["schema_version"]
|
|
109
|
+
if meta.get("lang"):
|
|
110
|
+
props["bq-source-lang"] = meta["lang"]
|
|
111
|
+
if meta.get("created_at") is not None:
|
|
112
|
+
props["bq-created-at"] = meta["created_at"]
|
|
113
|
+
|
|
114
|
+
kwargs: Dict[str, Any] = {"content_type": "application/json"}
|
|
115
|
+
if env.get("job"):
|
|
116
|
+
kwargs["subject"] = env["job"]
|
|
117
|
+
if env.get("trace_id"):
|
|
118
|
+
kwargs["correlation_id"] = env["trace_id"]
|
|
119
|
+
if meta.get("id"):
|
|
120
|
+
kwargs["message_id"] = meta["id"]
|
|
121
|
+
if props:
|
|
122
|
+
kwargs["application_properties"] = props
|
|
123
|
+
return kwargs
|
|
124
|
+
|
|
125
|
+
@staticmethod
|
|
126
|
+
def _reconcile(body: str, delivery_count: Any) -> str:
|
|
127
|
+
"""Set attempts to max(current, DeliveryCount - 1) — DeliveryCount (1-based) is the
|
|
128
|
+
broker's native redelivery floor, but the runtime retries by republishing with
|
|
129
|
+
attempts+1 in the body, so a republished message (DeliveryCount back to 1) must not
|
|
130
|
+
have its higher body count lowered. First delivery (DeliveryCount 1) reads 0."""
|
|
131
|
+
try:
|
|
132
|
+
dc = int(delivery_count)
|
|
133
|
+
except (ValueError, TypeError):
|
|
134
|
+
return body
|
|
135
|
+
if dc <= 1:
|
|
136
|
+
return body
|
|
137
|
+
native = dc - 1
|
|
138
|
+
env = EnvelopeCodec.decode(body)
|
|
139
|
+
if not env or native <= int(env.get("attempts", 0)):
|
|
140
|
+
return body
|
|
141
|
+
env["attempts"] = native
|
|
142
|
+
return EnvelopeCodec.encode(env)
|
|
143
|
+
|
|
144
|
+
# -- Transport ----------------------------------------------------------
|
|
145
|
+
|
|
146
|
+
def publish(self, queue: str, body: str) -> None:
|
|
147
|
+
from azure.servicebus import ServiceBusMessage
|
|
148
|
+
|
|
149
|
+
message = ServiceBusMessage(body, **self._projection(body))
|
|
150
|
+
self._sender(queue).send_messages(message)
|
|
151
|
+
|
|
152
|
+
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
153
|
+
wait = self._max_wait_time
|
|
154
|
+
if wait is None and timeout and timeout > 0:
|
|
155
|
+
wait = timeout
|
|
156
|
+
messages = self._receiver(queue).receive_messages(max_message_count=1, max_wait_time=wait)
|
|
157
|
+
if not messages:
|
|
158
|
+
return None
|
|
159
|
+
message = messages[0]
|
|
160
|
+
body = self._reconcile(str(message), getattr(message, "delivery_count", None))
|
|
161
|
+
return ReceivedMessage(body=body, queue=queue, handle=message)
|
|
162
|
+
|
|
163
|
+
def ack(self, message: ReceivedMessage) -> None:
|
|
164
|
+
if message.handle is None:
|
|
165
|
+
return
|
|
166
|
+
self._receiver(message.queue).complete_message(message.handle)
|
|
167
|
+
|
|
168
|
+
def close(self) -> None: # pragma: no cover - resource cleanup
|
|
169
|
+
for resource in (*self._senders.values(), *self._receivers.values(), self._client):
|
|
170
|
+
try:
|
|
171
|
+
resource.close()
|
|
172
|
+
except Exception: # noqa: BLE001 - best-effort cleanup
|
|
173
|
+
pass
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Amazon SQS transport. Requires the ``sqs`` extra:
|
|
2
|
+
|
|
3
|
+
pip install "babelqueue[sqs]"
|
|
4
|
+
|
|
5
|
+
Producing sends the canonical envelope as the message body and projects the
|
|
6
|
+
contract envelope fields onto native SQS ``MessageAttributes`` (``bq-job`` = URN,
|
|
7
|
+
``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
|
|
8
|
+
``bq-source-lang`` / ``bq-created-at``) — so a Go/PHP/... peer can route on ``bq-job``
|
|
9
|
+
and correlate on ``bq-trace-id`` without parsing the body. Consuming uses the
|
|
10
|
+
visibility-timeout reservation model (``receive_message`` -> process ->
|
|
11
|
+
``delete_message``); the authoritative attempt count is the broker's
|
|
12
|
+
``ApproximateReceiveCount``, reconciled onto the envelope as ``attempts = count - 1``.
|
|
13
|
+
|
|
14
|
+
This implements §3 of the broker-bindings contract. The envelope is unchanged
|
|
15
|
+
(``schema_version`` stays 1); SQS is purely additive.
|
|
16
|
+
|
|
17
|
+
URL form: ``sqs://[region][?endpoint=...&prefix=...&fifo=1&group_id=...&wait_time=20]``
|
|
18
|
+
(e.g. ``sqs://us-east-1?endpoint=http://localhost:4566`` for LocalStack). Credentials
|
|
19
|
+
come from the standard AWS default provider chain. For richer setups, build the
|
|
20
|
+
transport directly and pass it via ``BabelQueue(transport=...)``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Any, Dict, Optional
|
|
26
|
+
from urllib.parse import parse_qs, urlsplit
|
|
27
|
+
|
|
28
|
+
from .codec import EnvelopeCodec
|
|
29
|
+
from .transport import ReceivedMessage, Transport
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SqsTransport(Transport):
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
url: str = "sqs://",
|
|
36
|
+
*,
|
|
37
|
+
client: Any = None,
|
|
38
|
+
region: Optional[str] = None,
|
|
39
|
+
endpoint: Optional[str] = None,
|
|
40
|
+
queue_url_prefix: Optional[str] = None,
|
|
41
|
+
wait_time: Optional[int] = None,
|
|
42
|
+
visibility_timeout: Optional[int] = None,
|
|
43
|
+
fifo: bool = False,
|
|
44
|
+
message_group_id: Optional[str] = None,
|
|
45
|
+
content_dedup: bool = False,
|
|
46
|
+
) -> None:
|
|
47
|
+
parts = urlsplit(url) if url else urlsplit("sqs://")
|
|
48
|
+
q = parse_qs(parts.query)
|
|
49
|
+
|
|
50
|
+
self._region = region or (parts.hostname or None)
|
|
51
|
+
self._endpoint = endpoint or _q1(q, "endpoint")
|
|
52
|
+
self._queue_url_prefix = queue_url_prefix or _q1(q, "prefix")
|
|
53
|
+
self._wait_time = wait_time if wait_time is not None else _qint(q, "wait_time")
|
|
54
|
+
self._visibility_timeout = (
|
|
55
|
+
visibility_timeout if visibility_timeout is not None else _qint(q, "visibility_timeout")
|
|
56
|
+
)
|
|
57
|
+
self._fifo = fifo or _qbool(q, "fifo")
|
|
58
|
+
self._message_group_id = message_group_id or _q1(q, "group_id")
|
|
59
|
+
self._content_dedup = content_dedup or _qbool(q, "content_dedup")
|
|
60
|
+
self._urls: Dict[str, str] = {}
|
|
61
|
+
|
|
62
|
+
if client is not None:
|
|
63
|
+
self._sqs = client
|
|
64
|
+
return
|
|
65
|
+
try:
|
|
66
|
+
import boto3
|
|
67
|
+
except ImportError as exc: # pragma: no cover - import guard
|
|
68
|
+
raise ImportError(
|
|
69
|
+
"SqsTransport requires the 'boto3' package. Install with "
|
|
70
|
+
'pip install "babelqueue[sqs]".'
|
|
71
|
+
) from exc
|
|
72
|
+
kwargs: Dict[str, Any] = {}
|
|
73
|
+
if self._region:
|
|
74
|
+
kwargs["region_name"] = self._region
|
|
75
|
+
if self._endpoint:
|
|
76
|
+
kwargs["endpoint_url"] = self._endpoint
|
|
77
|
+
self._sqs = boto3.client("sqs", **kwargs) # pragma: no cover - needs AWS/LocalStack
|
|
78
|
+
|
|
79
|
+
# -- helpers ------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
def _resolve_url(self, name: str) -> str:
|
|
82
|
+
cached = self._urls.get(name)
|
|
83
|
+
if cached is not None:
|
|
84
|
+
return cached
|
|
85
|
+
if self._queue_url_prefix:
|
|
86
|
+
url = self._queue_url_prefix.rstrip("/") + "/" + name
|
|
87
|
+
else:
|
|
88
|
+
url = self._sqs.get_queue_url(QueueName=name)["QueueUrl"]
|
|
89
|
+
self._urls[name] = url
|
|
90
|
+
return url
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def _attributes(body: str) -> Dict[str, Dict[str, str]]:
|
|
94
|
+
"""Project the envelope's contract fields onto SQS MessageAttributes — a
|
|
95
|
+
redundant, routable view of the body (the body stays authoritative)."""
|
|
96
|
+
try:
|
|
97
|
+
env: Dict[str, Any] = EnvelopeCodec.decode(body)
|
|
98
|
+
except (ValueError, TypeError): # pragma: no cover - decode is defensive
|
|
99
|
+
return {}
|
|
100
|
+
meta = env.get("meta") or {}
|
|
101
|
+
|
|
102
|
+
def s(v: Any) -> Dict[str, str]:
|
|
103
|
+
return {"DataType": "String", "StringValue": str(v)}
|
|
104
|
+
|
|
105
|
+
def n(v: Any) -> Dict[str, str]:
|
|
106
|
+
return {"DataType": "Number", "StringValue": str(v)}
|
|
107
|
+
|
|
108
|
+
attrs: Dict[str, Dict[str, str]] = {}
|
|
109
|
+
if env.get("job"):
|
|
110
|
+
attrs["bq-job"] = s(env["job"])
|
|
111
|
+
if env.get("trace_id"):
|
|
112
|
+
attrs["bq-trace-id"] = s(env["trace_id"])
|
|
113
|
+
if meta.get("id"):
|
|
114
|
+
attrs["bq-message-id"] = s(meta["id"])
|
|
115
|
+
if meta.get("schema_version") is not None:
|
|
116
|
+
attrs["bq-schema-version"] = n(meta["schema_version"])
|
|
117
|
+
if meta.get("lang"):
|
|
118
|
+
attrs["bq-source-lang"] = s(meta["lang"])
|
|
119
|
+
if meta.get("created_at") is not None:
|
|
120
|
+
attrs["bq-created-at"] = n(meta["created_at"])
|
|
121
|
+
return attrs
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _reconcile(body: str, receive_count: Any) -> str:
|
|
125
|
+
"""Set attempts to max(current, ApproximateReceiveCount - 1): a first delivery
|
|
126
|
+
reads 0, a natively-redelivered message reflects its true count, and a
|
|
127
|
+
runtime-incremented counter is never lowered."""
|
|
128
|
+
try:
|
|
129
|
+
rc = int(receive_count)
|
|
130
|
+
except (ValueError, TypeError):
|
|
131
|
+
return body
|
|
132
|
+
if rc <= 1:
|
|
133
|
+
return body
|
|
134
|
+
env = EnvelopeCodec.decode(body)
|
|
135
|
+
if not env:
|
|
136
|
+
return body
|
|
137
|
+
native = rc - 1
|
|
138
|
+
if native <= int(env.get("attempts", 0)):
|
|
139
|
+
return body
|
|
140
|
+
env["attempts"] = native
|
|
141
|
+
return EnvelopeCodec.encode(env)
|
|
142
|
+
|
|
143
|
+
# -- Transport ----------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
def publish(self, queue: str, body: str) -> None:
|
|
146
|
+
params: Dict[str, Any] = {"QueueUrl": self._resolve_url(queue), "MessageBody": body}
|
|
147
|
+
attrs = self._attributes(body)
|
|
148
|
+
if attrs:
|
|
149
|
+
params["MessageAttributes"] = attrs
|
|
150
|
+
if self._fifo:
|
|
151
|
+
params["MessageGroupId"] = self._message_group_id or queue
|
|
152
|
+
if not self._content_dedup:
|
|
153
|
+
msg_id = (EnvelopeCodec.decode(body).get("meta") or {}).get("id")
|
|
154
|
+
if msg_id:
|
|
155
|
+
params["MessageDeduplicationId"] = msg_id
|
|
156
|
+
self._sqs.send_message(**params)
|
|
157
|
+
|
|
158
|
+
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
159
|
+
wait = int(timeout) if timeout and timeout > 0 else 0
|
|
160
|
+
if wait > 20:
|
|
161
|
+
wait = 20
|
|
162
|
+
if self._wait_time is not None and self._wait_time < wait:
|
|
163
|
+
wait = self._wait_time
|
|
164
|
+
params: Dict[str, Any] = {
|
|
165
|
+
"QueueUrl": self._resolve_url(queue),
|
|
166
|
+
"MaxNumberOfMessages": 1,
|
|
167
|
+
"WaitTimeSeconds": wait,
|
|
168
|
+
"MessageAttributeNames": ["All"],
|
|
169
|
+
"AttributeNames": ["ApproximateReceiveCount"],
|
|
170
|
+
}
|
|
171
|
+
if self._visibility_timeout is not None:
|
|
172
|
+
params["VisibilityTimeout"] = self._visibility_timeout
|
|
173
|
+
resp = self._sqs.receive_message(**params)
|
|
174
|
+
messages = resp.get("Messages") or []
|
|
175
|
+
if not messages:
|
|
176
|
+
return None
|
|
177
|
+
msg = messages[0]
|
|
178
|
+
body = msg.get("Body", "")
|
|
179
|
+
receive_count = (msg.get("Attributes") or {}).get("ApproximateReceiveCount")
|
|
180
|
+
if receive_count is not None:
|
|
181
|
+
body = self._reconcile(body, receive_count)
|
|
182
|
+
return ReceivedMessage(body=body, queue=queue, handle=msg.get("ReceiptHandle"))
|
|
183
|
+
|
|
184
|
+
def ack(self, message: ReceivedMessage) -> None:
|
|
185
|
+
if not message.handle:
|
|
186
|
+
return
|
|
187
|
+
self._sqs.delete_message(
|
|
188
|
+
QueueUrl=self._resolve_url(message.queue), ReceiptHandle=message.handle
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _q1(q: Dict[str, list], key: str) -> Optional[str]:
|
|
193
|
+
values = q.get(key)
|
|
194
|
+
return values[0] if values else None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _qint(q: Dict[str, list], key: str) -> Optional[int]:
|
|
198
|
+
v = _q1(q, key)
|
|
199
|
+
if v is None:
|
|
200
|
+
return None
|
|
201
|
+
try:
|
|
202
|
+
return int(v)
|
|
203
|
+
except ValueError: # pragma: no cover - defensive
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _qbool(q: Dict[str, list], key: str) -> bool:
|
|
208
|
+
v = _q1(q, key)
|
|
209
|
+
return v is not None and v.lower() in ("1", "true", "yes", "on")
|
|
@@ -79,8 +79,16 @@ def make_transport(broker_url: str) -> Transport:
|
|
|
79
79
|
from .pika_transport import PikaTransport
|
|
80
80
|
|
|
81
81
|
return PikaTransport(broker_url)
|
|
82
|
+
if scheme == "sqs":
|
|
83
|
+
from .sqs_transport import SqsTransport
|
|
84
|
+
|
|
85
|
+
return SqsTransport(broker_url)
|
|
86
|
+
if scheme in ("sb", "servicebus", "azureservicebus"):
|
|
87
|
+
from .asb_transport import AsbTransport
|
|
88
|
+
|
|
89
|
+
return AsbTransport(broker_url)
|
|
82
90
|
|
|
83
91
|
raise BabelQueueError(
|
|
84
|
-
f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://'
|
|
85
|
-
"'amqp://', or pass your own Transport via BabelQueue(transport=...)."
|
|
92
|
+
f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://', "
|
|
93
|
+
"'amqp://', 'sqs://' or 'sb://', or pass your own Transport via BabelQueue(transport=...)."
|
|
86
94
|
)
|