babelqueue 1.0.0__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- babelqueue-1.1.0/.github/FUNDING.yml +2 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/.github/workflows/ci.yml +23 -3
- {babelqueue-1.0.0 → babelqueue-1.1.0}/CHANGELOG.md +17 -2
- {babelqueue-1.0.0 → babelqueue-1.1.0}/PKG-INFO +3 -1
- {babelqueue-1.0.0 → babelqueue-1.1.0}/pyproject.toml +2 -1
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/__init__.py +1 -1
- babelqueue-1.1.0/src/babelqueue/sqs_transport.py +209 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/transport.py +6 -2
- babelqueue-1.1.0/tests/conformance/manifest.json +97 -0
- babelqueue-1.1.0/tests/test_sqs_conformance.py +53 -0
- babelqueue-1.1.0/tests/test_sqs_transport.py +299 -0
- babelqueue-1.0.0/tests/conformance/manifest.json +0 -71
- {babelqueue-1.0.0 → babelqueue-1.1.0}/.github/workflows/release.yml +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/.gitignore +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/LICENSE +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/README.md +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/app.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/exceptions.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/redis_transport.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_app.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_celery.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_codec.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_conformance.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_django.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_overhead.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-1.0.0 → babelqueue-1.1.0}/tests/test_redis_transport.py +0 -0
|
@@ -44,14 +44,14 @@ jobs:
|
|
|
44
44
|
- name: Install (dev + all adapters for type context)
|
|
45
45
|
run: |
|
|
46
46
|
python -m pip install --upgrade pip
|
|
47
|
-
pip install -e ".[dev,celery,django,redis,amqp]"
|
|
47
|
+
pip install -e ".[dev,celery,django,redis,amqp,sqs]"
|
|
48
48
|
- name: Ruff
|
|
49
49
|
run: ruff check src tests
|
|
50
50
|
- name: Mypy
|
|
51
51
|
run: mypy
|
|
52
52
|
|
|
53
53
|
integration:
|
|
54
|
-
name: Redis
|
|
54
|
+
name: Broker integration (Redis · RabbitMQ · SQS)
|
|
55
55
|
runs-on: ubuntu-latest
|
|
56
56
|
services:
|
|
57
57
|
redis:
|
|
@@ -72,6 +72,12 @@ jobs:
|
|
|
72
72
|
--health-interval 10s
|
|
73
73
|
--health-timeout 5s
|
|
74
74
|
--health-retries 15
|
|
75
|
+
# Free, SQS-compatible broker for the SqsTransport round-trip. The native
|
|
76
|
+
# image is shell-less, so readiness is polled by a TCP-probe step below.
|
|
77
|
+
elasticmq:
|
|
78
|
+
image: softwaremill/elasticmq-native:1.6.11
|
|
79
|
+
ports:
|
|
80
|
+
- 9324:9324
|
|
75
81
|
steps:
|
|
76
82
|
- uses: actions/checkout@v5
|
|
77
83
|
|
|
@@ -83,12 +89,26 @@ jobs:
|
|
|
83
89
|
- name: Install (all adapters — full coverage with brokers)
|
|
84
90
|
run: |
|
|
85
91
|
python -m pip install --upgrade pip
|
|
86
|
-
pip install -e ".[redis,amqp,celery,django,dev]"
|
|
92
|
+
pip install -e ".[redis,amqp,sqs,celery,django,dev]"
|
|
93
|
+
|
|
94
|
+
- name: Wait for ElasticMQ
|
|
95
|
+
run: |
|
|
96
|
+
for i in $(seq 1 30); do
|
|
97
|
+
if (echo > /dev/tcp/localhost/9324) >/dev/null 2>&1; then
|
|
98
|
+
echo "ElasticMQ port 9324 open"; exit 0
|
|
99
|
+
fi
|
|
100
|
+
sleep 2
|
|
101
|
+
done
|
|
102
|
+
echo "ElasticMQ did not open port 9324"; exit 1
|
|
87
103
|
|
|
88
104
|
- name: Run full suite with coverage gate (>=90%)
|
|
89
105
|
env:
|
|
90
106
|
BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
|
|
91
107
|
BABELQUEUE_TEST_AMQP: amqp://guest:guest@localhost:5672/
|
|
108
|
+
SQS_ENDPOINT: http://localhost:9324
|
|
109
|
+
AWS_REGION: us-east-1
|
|
110
|
+
AWS_ACCESS_KEY_ID: test
|
|
111
|
+
AWS_SECRET_ACCESS_KEY: test
|
|
92
112
|
run: pytest --cov=babelqueue --cov-report=term-missing --cov-fail-under=90
|
|
93
113
|
|
|
94
114
|
conformance:
|
|
@@ -7,7 +7,21 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
The envelope wire format is versioned separately by `meta.schema_version`
|
|
8
8
|
(currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
|
|
9
9
|
|
|
10
|
-
## [
|
|
10
|
+
## [1.1.0] - 2026-06-12
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **Amazon SQS transport** (`babelqueue[sqs]`, `boto3`) — `SqsTransport`, selected by
|
|
14
|
+
the `sqs://` URL scheme (e.g. `sqs://us-east-1?endpoint=http://localhost:4566` for
|
|
15
|
+
LocalStack). Implements [§3 of the broker-bindings contract](https://babelqueue.com):
|
|
16
|
+
the canonical envelope is the `MessageBody`, projected onto native `MessageAttributes`
|
|
17
|
+
(`bq-job`/`bq-trace-id`/`bq-message-id`/`bq-schema-version`/`bq-source-lang`/`bq-created-at`);
|
|
18
|
+
visibility-timeout reserve → `delete_message` ack; `attempts` reconciled to
|
|
19
|
+
`ApproximateReceiveCount − 1` (never lowering a runtime-incremented count). Supports
|
|
20
|
+
FIFO (`MessageGroupId`/`MessageDeduplicationId` = `meta.id`), content-based dedup,
|
|
21
|
+
configurable wait/visibility, and a `queue_url_prefix` that skips `GetQueueUrl`. A
|
|
22
|
+
client can be injected for tests/DI. 16 unit tests run against a fake client (no boto3,
|
|
23
|
+
no broker); a LocalStack integration test round-trips the real `boto3` path. The
|
|
24
|
+
envelope is unchanged (`schema_version: 1`); SQS is purely additive. Ships as a MINOR.
|
|
11
25
|
|
|
12
26
|
## [1.0.0] - 2026-06-07
|
|
13
27
|
|
|
@@ -85,7 +99,8 @@ reference at [babelqueue.com](https://babelqueue.com).
|
|
|
85
99
|
- Pre-1.0: the public API may change before the `1.0.0` tag.
|
|
86
100
|
- The core has **zero runtime dependencies** (standard library only); Python `>=3.9`.
|
|
87
101
|
|
|
88
|
-
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.
|
|
102
|
+
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.1.0...HEAD
|
|
103
|
+
[1.1.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.0.0...v1.1.0
|
|
89
104
|
[1.0.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.5.0...v1.0.0
|
|
90
105
|
[0.5.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.4.0...v0.5.0
|
|
91
106
|
[0.4.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.3.0...v0.4.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -35,6 +35,8 @@ Provides-Extra: django
|
|
|
35
35
|
Requires-Dist: django>=4.2; extra == 'django'
|
|
36
36
|
Provides-Extra: redis
|
|
37
37
|
Requires-Dist: redis>=4; extra == 'redis'
|
|
38
|
+
Provides-Extra: sqs
|
|
39
|
+
Requires-Dist: boto3>=1.26; extra == 'sqs'
|
|
38
40
|
Description-Content-Type: text/markdown
|
|
39
41
|
|
|
40
42
|
# BabelQueue for Python
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -31,6 +31,7 @@ dependencies = []
|
|
|
31
31
|
# Optional runtime drivers + framework adapters — standard, zero-heavy-dep.
|
|
32
32
|
redis = ["redis>=4"]
|
|
33
33
|
amqp = ["pika>=1.3"]
|
|
34
|
+
sqs = ["boto3>=1.26"]
|
|
34
35
|
celery = ["celery>=5"]
|
|
35
36
|
django = ["django>=4.2"]
|
|
36
37
|
dev = ["pytest>=7", "pytest-cov>=4", "mypy>=1.8", "ruff>=0.5"]
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Amazon SQS transport. Requires the ``sqs`` extra:
|
|
2
|
+
|
|
3
|
+
pip install "babelqueue[sqs]"
|
|
4
|
+
|
|
5
|
+
Producing sends the canonical envelope as the message body and projects the
|
|
6
|
+
contract envelope fields onto native SQS ``MessageAttributes`` (``bq-job`` = URN,
|
|
7
|
+
``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
|
|
8
|
+
``bq-source-lang`` / ``bq-created-at``) — so a Go/PHP/... peer can route on ``bq-job``
|
|
9
|
+
and correlate on ``bq-trace-id`` without parsing the body. Consuming uses the
|
|
10
|
+
visibility-timeout reservation model (``receive_message`` -> process ->
|
|
11
|
+
``delete_message``); the authoritative attempt count is the broker's
|
|
12
|
+
``ApproximateReceiveCount``, reconciled onto the envelope as ``attempts = count - 1``.
|
|
13
|
+
|
|
14
|
+
This implements §3 of the broker-bindings contract. The envelope is unchanged
|
|
15
|
+
(``schema_version`` stays 1); SQS is purely additive.
|
|
16
|
+
|
|
17
|
+
URL form: ``sqs://[region][?endpoint=...&prefix=...&fifo=1&group_id=...&wait_time=20]``
|
|
18
|
+
(e.g. ``sqs://us-east-1?endpoint=http://localhost:4566`` for LocalStack). Credentials
|
|
19
|
+
come from the standard AWS default provider chain. For richer setups, build the
|
|
20
|
+
transport directly and pass it via ``BabelQueue(transport=...)``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Any, Dict, Optional
|
|
26
|
+
from urllib.parse import parse_qs, urlsplit
|
|
27
|
+
|
|
28
|
+
from .codec import EnvelopeCodec
|
|
29
|
+
from .transport import ReceivedMessage, Transport
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SqsTransport(Transport):
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
url: str = "sqs://",
|
|
36
|
+
*,
|
|
37
|
+
client: Any = None,
|
|
38
|
+
region: Optional[str] = None,
|
|
39
|
+
endpoint: Optional[str] = None,
|
|
40
|
+
queue_url_prefix: Optional[str] = None,
|
|
41
|
+
wait_time: Optional[int] = None,
|
|
42
|
+
visibility_timeout: Optional[int] = None,
|
|
43
|
+
fifo: bool = False,
|
|
44
|
+
message_group_id: Optional[str] = None,
|
|
45
|
+
content_dedup: bool = False,
|
|
46
|
+
) -> None:
|
|
47
|
+
parts = urlsplit(url) if url else urlsplit("sqs://")
|
|
48
|
+
q = parse_qs(parts.query)
|
|
49
|
+
|
|
50
|
+
self._region = region or (parts.hostname or None)
|
|
51
|
+
self._endpoint = endpoint or _q1(q, "endpoint")
|
|
52
|
+
self._queue_url_prefix = queue_url_prefix or _q1(q, "prefix")
|
|
53
|
+
self._wait_time = wait_time if wait_time is not None else _qint(q, "wait_time")
|
|
54
|
+
self._visibility_timeout = (
|
|
55
|
+
visibility_timeout if visibility_timeout is not None else _qint(q, "visibility_timeout")
|
|
56
|
+
)
|
|
57
|
+
self._fifo = fifo or _qbool(q, "fifo")
|
|
58
|
+
self._message_group_id = message_group_id or _q1(q, "group_id")
|
|
59
|
+
self._content_dedup = content_dedup or _qbool(q, "content_dedup")
|
|
60
|
+
self._urls: Dict[str, str] = {}
|
|
61
|
+
|
|
62
|
+
if client is not None:
|
|
63
|
+
self._sqs = client
|
|
64
|
+
return
|
|
65
|
+
try:
|
|
66
|
+
import boto3
|
|
67
|
+
except ImportError as exc: # pragma: no cover - import guard
|
|
68
|
+
raise ImportError(
|
|
69
|
+
"SqsTransport requires the 'boto3' package. Install with "
|
|
70
|
+
'pip install "babelqueue[sqs]".'
|
|
71
|
+
) from exc
|
|
72
|
+
kwargs: Dict[str, Any] = {}
|
|
73
|
+
if self._region:
|
|
74
|
+
kwargs["region_name"] = self._region
|
|
75
|
+
if self._endpoint:
|
|
76
|
+
kwargs["endpoint_url"] = self._endpoint
|
|
77
|
+
self._sqs = boto3.client("sqs", **kwargs) # pragma: no cover - needs AWS/LocalStack
|
|
78
|
+
|
|
79
|
+
# -- helpers ------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
def _resolve_url(self, name: str) -> str:
|
|
82
|
+
cached = self._urls.get(name)
|
|
83
|
+
if cached is not None:
|
|
84
|
+
return cached
|
|
85
|
+
if self._queue_url_prefix:
|
|
86
|
+
url = self._queue_url_prefix.rstrip("/") + "/" + name
|
|
87
|
+
else:
|
|
88
|
+
url = self._sqs.get_queue_url(QueueName=name)["QueueUrl"]
|
|
89
|
+
self._urls[name] = url
|
|
90
|
+
return url
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def _attributes(body: str) -> Dict[str, Dict[str, str]]:
|
|
94
|
+
"""Project the envelope's contract fields onto SQS MessageAttributes — a
|
|
95
|
+
redundant, routable view of the body (the body stays authoritative)."""
|
|
96
|
+
try:
|
|
97
|
+
env: Dict[str, Any] = EnvelopeCodec.decode(body)
|
|
98
|
+
except (ValueError, TypeError): # pragma: no cover - decode is defensive
|
|
99
|
+
return {}
|
|
100
|
+
meta = env.get("meta") or {}
|
|
101
|
+
|
|
102
|
+
def s(v: Any) -> Dict[str, str]:
|
|
103
|
+
return {"DataType": "String", "StringValue": str(v)}
|
|
104
|
+
|
|
105
|
+
def n(v: Any) -> Dict[str, str]:
|
|
106
|
+
return {"DataType": "Number", "StringValue": str(v)}
|
|
107
|
+
|
|
108
|
+
attrs: Dict[str, Dict[str, str]] = {}
|
|
109
|
+
if env.get("job"):
|
|
110
|
+
attrs["bq-job"] = s(env["job"])
|
|
111
|
+
if env.get("trace_id"):
|
|
112
|
+
attrs["bq-trace-id"] = s(env["trace_id"])
|
|
113
|
+
if meta.get("id"):
|
|
114
|
+
attrs["bq-message-id"] = s(meta["id"])
|
|
115
|
+
if meta.get("schema_version") is not None:
|
|
116
|
+
attrs["bq-schema-version"] = n(meta["schema_version"])
|
|
117
|
+
if meta.get("lang"):
|
|
118
|
+
attrs["bq-source-lang"] = s(meta["lang"])
|
|
119
|
+
if meta.get("created_at") is not None:
|
|
120
|
+
attrs["bq-created-at"] = n(meta["created_at"])
|
|
121
|
+
return attrs
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _reconcile(body: str, receive_count: Any) -> str:
|
|
125
|
+
"""Set attempts to max(current, ApproximateReceiveCount - 1): a first delivery
|
|
126
|
+
reads 0, a natively-redelivered message reflects its true count, and a
|
|
127
|
+
runtime-incremented counter is never lowered."""
|
|
128
|
+
try:
|
|
129
|
+
rc = int(receive_count)
|
|
130
|
+
except (ValueError, TypeError):
|
|
131
|
+
return body
|
|
132
|
+
if rc <= 1:
|
|
133
|
+
return body
|
|
134
|
+
env = EnvelopeCodec.decode(body)
|
|
135
|
+
if not env:
|
|
136
|
+
return body
|
|
137
|
+
native = rc - 1
|
|
138
|
+
if native <= int(env.get("attempts", 0)):
|
|
139
|
+
return body
|
|
140
|
+
env["attempts"] = native
|
|
141
|
+
return EnvelopeCodec.encode(env)
|
|
142
|
+
|
|
143
|
+
# -- Transport ----------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
def publish(self, queue: str, body: str) -> None:
|
|
146
|
+
params: Dict[str, Any] = {"QueueUrl": self._resolve_url(queue), "MessageBody": body}
|
|
147
|
+
attrs = self._attributes(body)
|
|
148
|
+
if attrs:
|
|
149
|
+
params["MessageAttributes"] = attrs
|
|
150
|
+
if self._fifo:
|
|
151
|
+
params["MessageGroupId"] = self._message_group_id or queue
|
|
152
|
+
if not self._content_dedup:
|
|
153
|
+
msg_id = (EnvelopeCodec.decode(body).get("meta") or {}).get("id")
|
|
154
|
+
if msg_id:
|
|
155
|
+
params["MessageDeduplicationId"] = msg_id
|
|
156
|
+
self._sqs.send_message(**params)
|
|
157
|
+
|
|
158
|
+
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
159
|
+
wait = int(timeout) if timeout and timeout > 0 else 0
|
|
160
|
+
if wait > 20:
|
|
161
|
+
wait = 20
|
|
162
|
+
if self._wait_time is not None and self._wait_time < wait:
|
|
163
|
+
wait = self._wait_time
|
|
164
|
+
params: Dict[str, Any] = {
|
|
165
|
+
"QueueUrl": self._resolve_url(queue),
|
|
166
|
+
"MaxNumberOfMessages": 1,
|
|
167
|
+
"WaitTimeSeconds": wait,
|
|
168
|
+
"MessageAttributeNames": ["All"],
|
|
169
|
+
"AttributeNames": ["ApproximateReceiveCount"],
|
|
170
|
+
}
|
|
171
|
+
if self._visibility_timeout is not None:
|
|
172
|
+
params["VisibilityTimeout"] = self._visibility_timeout
|
|
173
|
+
resp = self._sqs.receive_message(**params)
|
|
174
|
+
messages = resp.get("Messages") or []
|
|
175
|
+
if not messages:
|
|
176
|
+
return None
|
|
177
|
+
msg = messages[0]
|
|
178
|
+
body = msg.get("Body", "")
|
|
179
|
+
receive_count = (msg.get("Attributes") or {}).get("ApproximateReceiveCount")
|
|
180
|
+
if receive_count is not None:
|
|
181
|
+
body = self._reconcile(body, receive_count)
|
|
182
|
+
return ReceivedMessage(body=body, queue=queue, handle=msg.get("ReceiptHandle"))
|
|
183
|
+
|
|
184
|
+
def ack(self, message: ReceivedMessage) -> None:
|
|
185
|
+
if not message.handle:
|
|
186
|
+
return
|
|
187
|
+
self._sqs.delete_message(
|
|
188
|
+
QueueUrl=self._resolve_url(message.queue), ReceiptHandle=message.handle
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _q1(q: Dict[str, list], key: str) -> Optional[str]:
|
|
193
|
+
values = q.get(key)
|
|
194
|
+
return values[0] if values else None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _qint(q: Dict[str, list], key: str) -> Optional[int]:
|
|
198
|
+
v = _q1(q, key)
|
|
199
|
+
if v is None:
|
|
200
|
+
return None
|
|
201
|
+
try:
|
|
202
|
+
return int(v)
|
|
203
|
+
except ValueError: # pragma: no cover - defensive
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _qbool(q: Dict[str, list], key: str) -> bool:
|
|
208
|
+
v = _q1(q, key)
|
|
209
|
+
return v is not None and v.lower() in ("1", "true", "yes", "on")
|
|
@@ -79,8 +79,12 @@ def make_transport(broker_url: str) -> Transport:
|
|
|
79
79
|
from .pika_transport import PikaTransport
|
|
80
80
|
|
|
81
81
|
return PikaTransport(broker_url)
|
|
82
|
+
if scheme == "sqs":
|
|
83
|
+
from .sqs_transport import SqsTransport
|
|
84
|
+
|
|
85
|
+
return SqsTransport(broker_url)
|
|
82
86
|
|
|
83
87
|
raise BabelQueueError(
|
|
84
|
-
f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://'
|
|
85
|
-
"'amqp://', or pass your own Transport via BabelQueue(transport=...)."
|
|
88
|
+
f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://', "
|
|
89
|
+
"'amqp://' or 'sqs://', or pass your own Transport via BabelQueue(transport=...)."
|
|
86
90
|
)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": 1,
|
|
3
|
+
"description": "Cross-SDK conformance cases. Every BabelQueue SDK core must satisfy these against the canonical wire envelope. Per-message fields (meta.id, trace_id, meta.created_at) are intrinsically unique and are NOT asserted by value.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{
|
|
6
|
+
"name": "order-created",
|
|
7
|
+
"file": "fixtures/order-created.json",
|
|
8
|
+
"valid": true,
|
|
9
|
+
"description": "A normal produced envelope.",
|
|
10
|
+
"expect": {
|
|
11
|
+
"urn": "urn:babel:orders:created",
|
|
12
|
+
"data": { "order_id": 1042 },
|
|
13
|
+
"attempts": 0,
|
|
14
|
+
"lang": "php",
|
|
15
|
+
"schema_version": 1
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"name": "urn-alias",
|
|
20
|
+
"file": "fixtures/urn-alias.json",
|
|
21
|
+
"valid": true,
|
|
22
|
+
"description": "Consumers MUST accept 'urn' as an inbound alias for 'job'.",
|
|
23
|
+
"expect": {
|
|
24
|
+
"urn": "urn:babel:orders:created",
|
|
25
|
+
"data": { "order_id": 1042 },
|
|
26
|
+
"attempts": 0,
|
|
27
|
+
"lang": "go",
|
|
28
|
+
"schema_version": 1
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"name": "dead-lettered",
|
|
33
|
+
"file": "fixtures/dead-lettered.json",
|
|
34
|
+
"valid": true,
|
|
35
|
+
"description": "A dead-lettered message: original preserved + additive dead_letter block.",
|
|
36
|
+
"expect": {
|
|
37
|
+
"urn": "urn:babel:orders:created",
|
|
38
|
+
"data": { "order_id": 1042 },
|
|
39
|
+
"attempts": 3,
|
|
40
|
+
"lang": "php",
|
|
41
|
+
"schema_version": 1,
|
|
42
|
+
"dead_letter": { "reason": "failed", "original_queue": "orders" }
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"name": "unicode-and-numbers",
|
|
47
|
+
"file": "fixtures/unicode-and-numbers.json",
|
|
48
|
+
"valid": true,
|
|
49
|
+
"description": "UTF-8 strings, integers, an exact float, boolean and null round-trip identically.",
|
|
50
|
+
"expect": {
|
|
51
|
+
"urn": "urn:babel:catalog:item.indexed",
|
|
52
|
+
"data": { "title": "Café — naïve ☕", "qty": 7, "price_cents": 1299, "ratio": 0.5, "active": true, "note": null },
|
|
53
|
+
"attempts": 2,
|
|
54
|
+
"lang": "python",
|
|
55
|
+
"schema_version": 1
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"name": "invalid-unknown-schema-version",
|
|
60
|
+
"file": "fixtures/invalid-unknown-schema-version.json",
|
|
61
|
+
"valid": false,
|
|
62
|
+
"reason": "meta.schema_version is not a version this SDK supports"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"name": "invalid-missing-urn",
|
|
66
|
+
"file": "fixtures/invalid-missing-urn.json",
|
|
67
|
+
"valid": false,
|
|
68
|
+
"reason": "no 'job' or 'urn' — the message has no identity"
|
|
69
|
+
}
|
|
70
|
+
],
|
|
71
|
+
"sqs": {
|
|
72
|
+
"description": "Amazon SQS binding conformance (broker-bindings.md §3). Every SDK that ships an SQS transport must satisfy these. The envelope body stays byte-identical (the 'cases' above); these lock the native projection + reconciliation the binding adds. Per-message values reuse fixtures/order-created.json so the expected attributes are deterministic.",
|
|
73
|
+
"attribute_projection": {
|
|
74
|
+
"description": "On produce, the transport MUST project these native MessageAttributes from the envelope (a redundant, routable view of the body; ids/strings are DataType String, counters are DataType Number). Applies to every SQS-producing SDK.",
|
|
75
|
+
"envelope_file": "fixtures/order-created.json",
|
|
76
|
+
"message_attributes": {
|
|
77
|
+
"bq-job": { "DataType": "String", "StringValue": "urn:babel:orders:created" },
|
|
78
|
+
"bq-trace-id": { "DataType": "String", "StringValue": "7b3f9c2a-e41d-4f88-9b2a-1c0d5e6f7a8b" },
|
|
79
|
+
"bq-message-id": { "DataType": "String", "StringValue": "f1e2d3c4-b5a6-4789-90ab-cdef01234567" },
|
|
80
|
+
"bq-schema-version": { "DataType": "Number", "StringValue": "1" },
|
|
81
|
+
"bq-source-lang": { "DataType": "String", "StringValue": "php" },
|
|
82
|
+
"bq-created-at": { "DataType": "Number", "StringValue": "1749132727000" }
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
"attempts_reconciliation": {
|
|
86
|
+
"description": "On consume, attempts = max(body.attempts, ApproximateReceiveCount - 1): a first delivery reads 0, an absent/garbage count is ignored, a runtime-incremented count is never lowered. Applies to SDKs that reconcile the envelope body on consume (the framework-less/runtime transports). A drop-in driver that surfaces the broker's native delivery count instead (e.g. Laravel's SqsJob.attempts() = ApproximateReceiveCount) is exempt — it documents that divergence.",
|
|
87
|
+
"cases": [
|
|
88
|
+
{ "name": "first-delivery", "body_attempts": 0, "approximate_receive_count": "1", "expected_attempts": 0 },
|
|
89
|
+
{ "name": "third-delivery", "body_attempts": 0, "approximate_receive_count": "3", "expected_attempts": 2 },
|
|
90
|
+
{ "name": "native-exceeds-body", "body_attempts": 2, "approximate_receive_count": "5", "expected_attempts": 4 },
|
|
91
|
+
{ "name": "never-lower-runtime", "body_attempts": 5, "approximate_receive_count": "1", "expected_attempts": 5 },
|
|
92
|
+
{ "name": "garbage-count-ignored", "body_attempts": 4, "approximate_receive_count": "not-a-number", "expected_attempts": 4 },
|
|
93
|
+
{ "name": "absent-count", "body_attempts": 3, "approximate_receive_count": null, "expected_attempts": 3 }
|
|
94
|
+
]
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Amazon SQS binding conformance: the vendored manifest's ``sqs`` block locks the
|
|
2
|
+
§3 MessageAttributes projection and the ``attempts = ApproximateReceiveCount - 1``
|
|
3
|
+
reconciliation. No boto3, no broker — pure transport logic against golden values."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import unittest
|
|
10
|
+
|
|
11
|
+
from babelqueue import EnvelopeCodec
|
|
12
|
+
from babelqueue.sqs_transport import SqsTransport
|
|
13
|
+
|
|
14
|
+
CONFORMANCE = os.path.join(os.path.dirname(__file__), "conformance")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class SqsConformanceTest(unittest.TestCase):
|
|
18
|
+
@classmethod
|
|
19
|
+
def setUpClass(cls) -> None:
|
|
20
|
+
with open(os.path.join(CONFORMANCE, "manifest.json"), encoding="utf-8") as fh:
|
|
21
|
+
cls.sqs = json.load(fh)["sqs"]
|
|
22
|
+
|
|
23
|
+
def test_attribute_projection(self) -> None:
|
|
24
|
+
projection = self.sqs["attribute_projection"]
|
|
25
|
+
with open(os.path.join(CONFORMANCE, projection["envelope_file"]), encoding="utf-8") as fh:
|
|
26
|
+
body = fh.read()
|
|
27
|
+
|
|
28
|
+
got = SqsTransport._attributes(body)
|
|
29
|
+
want = projection["message_attributes"]
|
|
30
|
+
|
|
31
|
+
self.assertEqual(set(got), set(want))
|
|
32
|
+
for key, expected in want.items():
|
|
33
|
+
self.assertEqual(got[key]["DataType"], expected["DataType"], key)
|
|
34
|
+
self.assertEqual(got[key]["StringValue"], expected["StringValue"], key)
|
|
35
|
+
|
|
36
|
+
def test_attempts_reconciliation(self) -> None:
|
|
37
|
+
for case in self.sqs["attempts_reconciliation"]["cases"]:
|
|
38
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1})
|
|
39
|
+
env["attempts"] = case["body_attempts"]
|
|
40
|
+
body = EnvelopeCodec.encode(env)
|
|
41
|
+
|
|
42
|
+
receive_count = case["approximate_receive_count"]
|
|
43
|
+
out = body if receive_count is None else SqsTransport._reconcile(body, receive_count)
|
|
44
|
+
|
|
45
|
+
self.assertEqual(
|
|
46
|
+
EnvelopeCodec.decode(out)["attempts"],
|
|
47
|
+
case["expected_attempts"],
|
|
48
|
+
case["name"],
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
if __name__ == "__main__": # pragma: no cover
|
|
53
|
+
unittest.main()
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""Tests for the Amazon SQS transport.
|
|
2
|
+
|
|
3
|
+
The unit tests inject a fake SQS client, so they run without ``boto3`` and without
|
|
4
|
+
a broker. A separate integration test round-trips against LocalStack and is skipped
|
|
5
|
+
unless ``boto3`` + a reachable endpoint are present (CI runs it).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import unittest
|
|
12
|
+
import uuid
|
|
13
|
+
|
|
14
|
+
from babelqueue import BabelQueue, EnvelopeCodec
|
|
15
|
+
from babelqueue.sqs_transport import SqsTransport
|
|
16
|
+
from babelqueue.transport import ReceivedMessage, make_transport
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
class FakeSQS:
|
|
20
|
+
"""In-memory SQS API — no boto3, no network."""
|
|
21
|
+
|
|
22
|
+
def __init__(self, err: Exception | None = None) -> None:
|
|
23
|
+
self.visible: dict[str, list[dict]] = {}
|
|
24
|
+
self.sent: list[dict] = []
|
|
25
|
+
self.deleted: list[str] = []
|
|
26
|
+
self.get_url_calls = 0
|
|
27
|
+
self.last_receive: dict | None = None
|
|
28
|
+
self.err = err
|
|
29
|
+
self._n = 0
|
|
30
|
+
|
|
31
|
+
def get_queue_url(self, QueueName): # noqa: N803 - AWS API casing
|
|
32
|
+
self.get_url_calls += 1
|
|
33
|
+
if self.err:
|
|
34
|
+
raise self.err
|
|
35
|
+
return {"QueueUrl": "http://fake/" + QueueName}
|
|
36
|
+
|
|
37
|
+
def send_message(self, **kw):
|
|
38
|
+
if self.err:
|
|
39
|
+
raise self.err
|
|
40
|
+
self.sent.append(kw)
|
|
41
|
+
self._n += 1
|
|
42
|
+
handle = f"rh-{self._n}"
|
|
43
|
+
self.visible.setdefault(kw["QueueUrl"], []).append(
|
|
44
|
+
{
|
|
45
|
+
"Body": kw["MessageBody"],
|
|
46
|
+
"MessageAttributes": kw.get("MessageAttributes"),
|
|
47
|
+
"ReceiptHandle": handle,
|
|
48
|
+
"Attributes": {"ApproximateReceiveCount": "1"},
|
|
49
|
+
}
|
|
50
|
+
)
|
|
51
|
+
return {"MessageId": handle}
|
|
52
|
+
|
|
53
|
+
def receive_message(self, **kw):
|
|
54
|
+
self.last_receive = kw
|
|
55
|
+
if self.err:
|
|
56
|
+
raise self.err
|
|
57
|
+
q = self.visible.get(kw["QueueUrl"], [])
|
|
58
|
+
if not q:
|
|
59
|
+
return {}
|
|
60
|
+
return {"Messages": [q.pop(0)]}
|
|
61
|
+
|
|
62
|
+
def delete_message(self, **kw):
|
|
63
|
+
if self.err:
|
|
64
|
+
raise self.err
|
|
65
|
+
self.deleted.append(kw["ReceiptHandle"])
|
|
66
|
+
return {}
|
|
67
|
+
|
|
68
|
+
def seed(self, url: str, body: str, receive_count: int) -> None:
|
|
69
|
+
self._n += 1
|
|
70
|
+
self.visible.setdefault(url, []).append(
|
|
71
|
+
{
|
|
72
|
+
"Body": body,
|
|
73
|
+
"ReceiptHandle": f"seed-{self._n}",
|
|
74
|
+
"Attributes": {"ApproximateReceiveCount": str(receive_count)},
|
|
75
|
+
}
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _attr(sent: dict, key: str) -> str:
|
|
80
|
+
return sent["MessageAttributes"][key]["StringValue"]
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class SqsTransportUnitTest(unittest.TestCase):
|
|
84
|
+
def _tr(self, **kw) -> tuple[SqsTransport, FakeSQS]:
|
|
85
|
+
fake = FakeSQS()
|
|
86
|
+
return SqsTransport("sqs://", client=fake, queue_url_prefix="http://fake", **kw), fake
|
|
87
|
+
|
|
88
|
+
def test_publish_projects_contract_attributes(self):
|
|
89
|
+
tr, fake = self._tr()
|
|
90
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {"order_id": 1042}, queue="orders")
|
|
91
|
+
body = EnvelopeCodec.encode(env)
|
|
92
|
+
tr.publish("orders", body)
|
|
93
|
+
|
|
94
|
+
self.assertEqual(len(fake.sent), 1)
|
|
95
|
+
sent = fake.sent[0]
|
|
96
|
+
self.assertEqual(sent["QueueUrl"], "http://fake/orders")
|
|
97
|
+
self.assertEqual(sent["MessageBody"], body) # byte-identical
|
|
98
|
+
self.assertEqual(_attr(sent, "bq-job"), env["job"])
|
|
99
|
+
self.assertEqual(_attr(sent, "bq-trace-id"), env["trace_id"])
|
|
100
|
+
self.assertEqual(_attr(sent, "bq-message-id"), env["meta"]["id"])
|
|
101
|
+
self.assertEqual(_attr(sent, "bq-schema-version"), "1")
|
|
102
|
+
self.assertEqual(_attr(sent, "bq-source-lang"), "python")
|
|
103
|
+
self.assertEqual(_attr(sent, "bq-created-at"), str(env["meta"]["created_at"]))
|
|
104
|
+
# Type discipline: ids String, counters Number.
|
|
105
|
+
self.assertEqual(sent["MessageAttributes"]["bq-job"]["DataType"], "String")
|
|
106
|
+
self.assertEqual(sent["MessageAttributes"]["bq-schema-version"]["DataType"], "Number")
|
|
107
|
+
|
|
108
|
+
def test_pop_reconciles_attempts_from_receive_count(self):
|
|
109
|
+
tr, fake = self._tr()
|
|
110
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1}, queue="orders")
|
|
111
|
+
fake.seed("http://fake/orders", EnvelopeCodec.encode(env), 3) # 3rd delivery -> attempts 2
|
|
112
|
+
msg = tr.pop("orders", timeout=0)
|
|
113
|
+
self.assertIsNotNone(msg)
|
|
114
|
+
self.assertEqual(EnvelopeCodec.decode(msg.body)["attempts"], 2)
|
|
115
|
+
|
|
116
|
+
def test_pop_does_not_lower_runtime_attempts(self):
|
|
117
|
+
tr, fake = self._tr()
|
|
118
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1})
|
|
119
|
+
env["attempts"] = 5
|
|
120
|
+
fake.seed("http://fake/default", EnvelopeCodec.encode(env), 1)
|
|
121
|
+
msg = tr.pop("default", timeout=0)
|
|
122
|
+
self.assertEqual(EnvelopeCodec.decode(msg.body)["attempts"], 5)
|
|
123
|
+
|
|
124
|
+
def test_pop_empty_returns_none(self):
|
|
125
|
+
tr, _ = self._tr()
|
|
126
|
+
self.assertIsNone(tr.pop("orders", timeout=0))
|
|
127
|
+
|
|
128
|
+
def test_ack_deletes_by_receipt_handle(self):
|
|
129
|
+
tr, fake = self._tr()
|
|
130
|
+
fake.seed("http://fake/orders", '{"job":"u","trace_id":"t","data":{},"meta":{"schema_version":1},"attempts":0}', 1)
|
|
131
|
+
msg = tr.pop("orders", timeout=0)
|
|
132
|
+
tr.ack(msg)
|
|
133
|
+
self.assertEqual(fake.deleted, [msg.handle])
|
|
134
|
+
|
|
135
|
+
def test_ack_noop_on_empty_handle(self):
|
|
136
|
+
tr, fake = self._tr()
|
|
137
|
+
tr.ack(ReceivedMessage(body="", queue="orders", handle=None))
|
|
138
|
+
self.assertEqual(fake.deleted, [])
|
|
139
|
+
|
|
140
|
+
def test_fifo_sets_group_and_dedup(self):
|
|
141
|
+
tr, fake = self._tr(fifo=True)
|
|
142
|
+
env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1}, queue="orders.fifo")
|
|
143
|
+
tr.publish("orders.fifo", EnvelopeCodec.encode(env))
|
|
144
|
+
sent = fake.sent[0]
|
|
145
|
+
self.assertEqual(sent["MessageGroupId"], "orders.fifo")
|
|
146
|
+
self.assertEqual(sent["MessageDeduplicationId"], env["meta"]["id"])
|
|
147
|
+
|
|
148
|
+
def test_content_dedup_omits_dedup_id(self):
|
|
149
|
+
tr, fake = self._tr(fifo=True, content_dedup=True, message_group_id="grp")
|
|
150
|
+
tr.publish("orders.fifo", '{"job":"u","trace_id":"t","data":{},"meta":{"id":"m1","schema_version":1},"attempts":0}')
|
|
151
|
+
sent = fake.sent[0]
|
|
152
|
+
self.assertEqual(sent["MessageGroupId"], "grp")
|
|
153
|
+
self.assertNotIn("MessageDeduplicationId", sent)
|
|
154
|
+
|
|
155
|
+
def test_resolve_via_get_queue_url_and_caches(self):
|
|
156
|
+
fake = FakeSQS()
|
|
157
|
+
tr = SqsTransport("sqs://", client=fake) # no prefix -> GetQueueUrl path
|
|
158
|
+
body = '{"job":"u","trace_id":"t","data":{},"meta":{"schema_version":1,"lang":"python"},"attempts":0}'
|
|
159
|
+
for _ in range(3):
|
|
160
|
+
tr.publish("orders", body)
|
|
161
|
+
self.assertEqual(fake.sent[0]["QueueUrl"], "http://fake/orders")
|
|
162
|
+
self.assertEqual(fake.get_url_calls, 1) # cached
|
|
163
|
+
|
|
164
|
+
def test_pop_applies_visibility_and_wait_options(self):
|
|
165
|
+
tr, fake = self._tr(visibility_timeout=45, wait_time=5)
|
|
166
|
+
tr.pop("orders", timeout=30)
|
|
167
|
+
self.assertEqual(fake.last_receive["VisibilityTimeout"], 45)
|
|
168
|
+
self.assertEqual(fake.last_receive["WaitTimeSeconds"], 5) # 30 -> clamp 20 -> cap 5
|
|
169
|
+
self.assertEqual(fake.last_receive["MessageAttributeNames"], ["All"])
|
|
170
|
+
self.assertEqual(fake.last_receive["AttributeNames"], ["ApproximateReceiveCount"])
|
|
171
|
+
|
|
172
|
+
def test_url_parses_region_endpoint_prefix_fifo(self):
|
|
173
|
+
fake = FakeSQS()
|
|
174
|
+
tr = SqsTransport(
|
|
175
|
+
"sqs://eu-west-1?endpoint=http://ls:4566&prefix=http://ls:4566/000000000000&fifo=1&group_id=g&wait_time=7",
|
|
176
|
+
client=fake,
|
|
177
|
+
)
|
|
178
|
+
self.assertEqual(tr._region, "eu-west-1")
|
|
179
|
+
self.assertEqual(tr._endpoint, "http://ls:4566")
|
|
180
|
+
self.assertTrue(tr._fifo)
|
|
181
|
+
self.assertEqual(tr._message_group_id, "g")
|
|
182
|
+
self.assertEqual(tr._wait_time, 7)
|
|
183
|
+
# prefix resolution avoids GetQueueUrl
|
|
184
|
+
tr.publish("orders.fifo", '{"job":"u","trace_id":"t","data":{},"meta":{"id":"m","schema_version":1}}')
|
|
185
|
+
self.assertEqual(fake.sent[0]["QueueUrl"], "http://ls:4566/000000000000/orders.fifo")
|
|
186
|
+
|
|
187
|
+
def test_reconcile_ignores_garbage_and_undecodable(self):
|
|
188
|
+
tr, fake = self._tr()
|
|
189
|
+
fake.seed("http://fake/orders", '{"job":"u","trace_id":"t","data":{},"meta":{"schema_version":1},"attempts":4}', 0)
|
|
190
|
+
fake.visible["http://fake/orders"][0]["Attributes"]["ApproximateReceiveCount"] = "not-a-number"
|
|
191
|
+
msg = tr.pop("orders", timeout=0)
|
|
192
|
+
self.assertEqual(EnvelopeCodec.decode(msg.body)["attempts"], 4)
|
|
193
|
+
|
|
194
|
+
fake.seed("http://fake/orders", "not-json", 3) # rc>1 but undecodable body
|
|
195
|
+
msg2 = tr.pop("orders", timeout=0)
|
|
196
|
+
self.assertEqual(msg2.body, "not-json")
|
|
197
|
+
|
|
198
|
+
def test_attributes_empty_for_undecodable_body(self):
|
|
199
|
+
self.assertEqual(SqsTransport._attributes("}{not json"), {})
|
|
200
|
+
|
|
201
|
+
def test_errors_propagate(self):
|
|
202
|
+
boom = RuntimeError("boom")
|
|
203
|
+
fail = SqsTransport("sqs://", client=FakeSQS(err=boom)) # no prefix -> GetQueueUrl errors
|
|
204
|
+
with self.assertRaises(RuntimeError):
|
|
205
|
+
fail.publish("orders", '{"job":"u","trace_id":"t","data":{},"meta":{"schema_version":1}}')
|
|
206
|
+
with self.assertRaises(RuntimeError):
|
|
207
|
+
fail.pop("orders", timeout=0)
|
|
208
|
+
with self.assertRaises(RuntimeError):
|
|
209
|
+
fail.ack(ReceivedMessage(body="", queue="orders", handle="h"))
|
|
210
|
+
|
|
211
|
+
def test_round_trip_through_app(self):
|
|
212
|
+
fake = FakeSQS()
|
|
213
|
+
tr = SqsTransport("sqs://", client=fake, queue_url_prefix="http://fake")
|
|
214
|
+
app = BabelQueue(transport=tr, queue="orders")
|
|
215
|
+
seen: dict = {}
|
|
216
|
+
|
|
217
|
+
@app.handler("urn:babel:orders:created")
|
|
218
|
+
def _on(data, meta):
|
|
219
|
+
seen.update({"data": data, "meta": meta})
|
|
220
|
+
|
|
221
|
+
msg_id = app.publish("urn:babel:orders:created", {"order_id": 7})
|
|
222
|
+
processed = app.consume("orders", max_messages=1, timeout=0)
|
|
223
|
+
self.assertEqual(processed, 1)
|
|
224
|
+
self.assertEqual(seen["data"]["order_id"], 7)
|
|
225
|
+
self.assertEqual(seen["meta"]["id"], msg_id)
|
|
226
|
+
self.assertEqual(len(fake.deleted), 1) # acked/deleted
|
|
227
|
+
|
|
228
|
+
def test_make_transport_routes_sqs_scheme(self):
|
|
229
|
+
# The scheme dispatches to SqsTransport; without boto3 it surfaces a clear
|
|
230
|
+
# install hint (covers the make_transport branch). With boto3 present this
|
|
231
|
+
# would construct a real client, so only assert the branch is reached.
|
|
232
|
+
try:
|
|
233
|
+
import boto3 # noqa: F401
|
|
234
|
+
self.skipTest("boto3 installed — real client would be built")
|
|
235
|
+
except ImportError:
|
|
236
|
+
with self.assertRaises(ImportError):
|
|
237
|
+
make_transport("sqs://")
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _sqs_available() -> bool:
|
|
241
|
+
try:
|
|
242
|
+
import boto3
|
|
243
|
+
except ImportError:
|
|
244
|
+
return False
|
|
245
|
+
endpoint = os.environ.get("SQS_ENDPOINT", "http://localhost:4566")
|
|
246
|
+
try:
|
|
247
|
+
client = boto3.client(
|
|
248
|
+
"sqs",
|
|
249
|
+
region_name=os.environ.get("AWS_REGION", "us-east-1"),
|
|
250
|
+
endpoint_url=endpoint,
|
|
251
|
+
aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID", "test"),
|
|
252
|
+
aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY", "test"),
|
|
253
|
+
)
|
|
254
|
+
client.list_queues()
|
|
255
|
+
return True
|
|
256
|
+
except Exception: # pragma: no cover - connection failure
|
|
257
|
+
return False
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
@unittest.skipUnless(_sqs_available(), "no reachable LocalStack SQS at SQS_ENDPOINT")
|
|
261
|
+
class SqsLocalStackIntegrationTest(unittest.TestCase):
|
|
262
|
+
def setUp(self) -> None:
|
|
263
|
+
import boto3
|
|
264
|
+
|
|
265
|
+
self.endpoint = os.environ.get("SQS_ENDPOINT", "http://localhost:4566")
|
|
266
|
+
self.region = os.environ.get("AWS_REGION", "us-east-1")
|
|
267
|
+
os.environ.setdefault("AWS_ACCESS_KEY_ID", "test")
|
|
268
|
+
os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "test")
|
|
269
|
+
self.raw = boto3.client("sqs", region_name=self.region, endpoint_url=self.endpoint)
|
|
270
|
+
self.queue = f"bq-it-{uuid.uuid4().hex}"
|
|
271
|
+
self.raw.create_queue(QueueName=self.queue)
|
|
272
|
+
|
|
273
|
+
def tearDown(self) -> None:
|
|
274
|
+
try:
|
|
275
|
+
url = self.raw.get_queue_url(QueueName=self.queue)["QueueUrl"]
|
|
276
|
+
self.raw.delete_queue(QueueUrl=url)
|
|
277
|
+
except Exception:
|
|
278
|
+
pass
|
|
279
|
+
|
|
280
|
+
def test_produce_consume_round_trip(self):
|
|
281
|
+
url = f"sqs://{self.region}?endpoint={self.endpoint}"
|
|
282
|
+
app = BabelQueue(url, queue=self.queue)
|
|
283
|
+
seen: dict = {}
|
|
284
|
+
app.register("urn:babel:orders:created", lambda data, meta: seen.update(data))
|
|
285
|
+
msg_id = app.publish("urn:babel:orders:created", {"order_id": 1042})
|
|
286
|
+
self.assertTrue(msg_id)
|
|
287
|
+
|
|
288
|
+
# publish carried the contract attributes; verify them off the raw queue too
|
|
289
|
+
processed = 0
|
|
290
|
+
for _ in range(30):
|
|
291
|
+
processed = app.consume(self.queue, max_messages=1, timeout=1)
|
|
292
|
+
if processed:
|
|
293
|
+
break
|
|
294
|
+
self.assertEqual(processed, 1)
|
|
295
|
+
self.assertEqual(seen.get("order_id"), 1042)
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
if __name__ == "__main__": # pragma: no cover
|
|
299
|
+
unittest.main()
|
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
{
|
|
2
|
-
"schema_version": 1,
|
|
3
|
-
"description": "Cross-SDK conformance cases. Every BabelQueue SDK core must satisfy these against the canonical wire envelope. Per-message fields (meta.id, trace_id, meta.created_at) are intrinsically unique and are NOT asserted by value.",
|
|
4
|
-
"cases": [
|
|
5
|
-
{
|
|
6
|
-
"name": "order-created",
|
|
7
|
-
"file": "fixtures/order-created.json",
|
|
8
|
-
"valid": true,
|
|
9
|
-
"description": "A normal produced envelope.",
|
|
10
|
-
"expect": {
|
|
11
|
-
"urn": "urn:babel:orders:created",
|
|
12
|
-
"data": { "order_id": 1042 },
|
|
13
|
-
"attempts": 0,
|
|
14
|
-
"lang": "php",
|
|
15
|
-
"schema_version": 1
|
|
16
|
-
}
|
|
17
|
-
},
|
|
18
|
-
{
|
|
19
|
-
"name": "urn-alias",
|
|
20
|
-
"file": "fixtures/urn-alias.json",
|
|
21
|
-
"valid": true,
|
|
22
|
-
"description": "Consumers MUST accept 'urn' as an inbound alias for 'job'.",
|
|
23
|
-
"expect": {
|
|
24
|
-
"urn": "urn:babel:orders:created",
|
|
25
|
-
"data": { "order_id": 1042 },
|
|
26
|
-
"attempts": 0,
|
|
27
|
-
"lang": "go",
|
|
28
|
-
"schema_version": 1
|
|
29
|
-
}
|
|
30
|
-
},
|
|
31
|
-
{
|
|
32
|
-
"name": "dead-lettered",
|
|
33
|
-
"file": "fixtures/dead-lettered.json",
|
|
34
|
-
"valid": true,
|
|
35
|
-
"description": "A dead-lettered message: original preserved + additive dead_letter block.",
|
|
36
|
-
"expect": {
|
|
37
|
-
"urn": "urn:babel:orders:created",
|
|
38
|
-
"data": { "order_id": 1042 },
|
|
39
|
-
"attempts": 3,
|
|
40
|
-
"lang": "php",
|
|
41
|
-
"schema_version": 1,
|
|
42
|
-
"dead_letter": { "reason": "failed", "original_queue": "orders" }
|
|
43
|
-
}
|
|
44
|
-
},
|
|
45
|
-
{
|
|
46
|
-
"name": "unicode-and-numbers",
|
|
47
|
-
"file": "fixtures/unicode-and-numbers.json",
|
|
48
|
-
"valid": true,
|
|
49
|
-
"description": "UTF-8 strings, integers, an exact float, boolean and null round-trip identically.",
|
|
50
|
-
"expect": {
|
|
51
|
-
"urn": "urn:babel:catalog:item.indexed",
|
|
52
|
-
"data": { "title": "Café — naïve ☕", "qty": 7, "price_cents": 1299, "ratio": 0.5, "active": true, "note": null },
|
|
53
|
-
"attempts": 2,
|
|
54
|
-
"lang": "python",
|
|
55
|
-
"schema_version": 1
|
|
56
|
-
}
|
|
57
|
-
},
|
|
58
|
-
{
|
|
59
|
-
"name": "invalid-unknown-schema-version",
|
|
60
|
-
"file": "fixtures/invalid-unknown-schema-version.json",
|
|
61
|
-
"valid": false,
|
|
62
|
-
"reason": "meta.schema_version is not a version this SDK supports"
|
|
63
|
-
},
|
|
64
|
-
{
|
|
65
|
-
"name": "invalid-missing-urn",
|
|
66
|
-
"file": "fixtures/invalid-missing-urn.json",
|
|
67
|
-
"valid": false,
|
|
68
|
-
"reason": "no 'job' or 'urn' — the message has no identity"
|
|
69
|
-
}
|
|
70
|
-
]
|
|
71
|
-
}
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.0.0 → babelqueue-1.1.0}/src/babelqueue/django/management/commands/babelqueue_worker.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{babelqueue-1.0.0 → babelqueue-1.1.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|