babelqueue 0.5.0__tar.gz → 1.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- babelqueue-1.1.0/.github/FUNDING.yml +2 -0
- babelqueue-1.1.0/.github/workflows/ci.yml +138 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/.github/workflows/release.yml +1 -1
- {babelqueue-0.5.0 → babelqueue-1.1.0}/.gitignore +3 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/CHANGELOG.md +35 -2
- {babelqueue-0.5.0 → babelqueue-1.1.0}/PKG-INFO +6 -1
- {babelqueue-0.5.0 → babelqueue-1.1.0}/pyproject.toml +21 -2
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/__init__.py +1 -1
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/redis_transport.py +7 -2
- babelqueue-1.1.0/src/babelqueue/sqs_transport.py +209 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/transport.py +7 -3
- babelqueue-1.1.0/tests/conformance/manifest.json +97 -0
- babelqueue-1.1.0/tests/test_overhead.py +44 -0
- babelqueue-1.1.0/tests/test_sqs_conformance.py +53 -0
- babelqueue-1.1.0/tests/test_sqs_transport.py +299 -0
- babelqueue-0.5.0/.github/workflows/ci.yml +0 -74
- babelqueue-0.5.0/tests/conformance/manifest.json +0 -71
- {babelqueue-0.5.0 → babelqueue-1.1.0}/LICENSE +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/README.md +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/app.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/exceptions.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_app.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_celery.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_codec.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_conformance.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_django.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-0.5.0 → babelqueue-1.1.0}/tests/test_redis_transport.py +0 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
name: CI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
permissions:
|
|
9
|
+
contents: read
|
|
10
|
+
|
|
11
|
+
jobs:
|
|
12
|
+
test:
|
|
13
|
+
name: Python ${{ matrix.python }}
|
|
14
|
+
runs-on: ubuntu-latest
|
|
15
|
+
strategy:
|
|
16
|
+
fail-fast: false
|
|
17
|
+
matrix:
|
|
18
|
+
python: ['3.9', '3.10', '3.11', '3.12', '3.13']
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v5
|
|
21
|
+
|
|
22
|
+
- name: Setup Python
|
|
23
|
+
uses: actions/setup-python@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: ${{ matrix.python }}
|
|
26
|
+
|
|
27
|
+
- name: Install (with Celery + Django adapters)
|
|
28
|
+
run: |
|
|
29
|
+
python -m pip install --upgrade pip
|
|
30
|
+
pip install -e ".[dev,celery,django]"
|
|
31
|
+
|
|
32
|
+
- name: Run tests
|
|
33
|
+
run: pytest
|
|
34
|
+
|
|
35
|
+
lint:
|
|
36
|
+
name: Static analysis (ruff + mypy)
|
|
37
|
+
runs-on: ubuntu-latest
|
|
38
|
+
steps:
|
|
39
|
+
- uses: actions/checkout@v5
|
|
40
|
+
- name: Setup Python
|
|
41
|
+
uses: actions/setup-python@v5
|
|
42
|
+
with:
|
|
43
|
+
python-version: '3.12'
|
|
44
|
+
- name: Install (dev + all adapters for type context)
|
|
45
|
+
run: |
|
|
46
|
+
python -m pip install --upgrade pip
|
|
47
|
+
pip install -e ".[dev,celery,django,redis,amqp,sqs]"
|
|
48
|
+
- name: Ruff
|
|
49
|
+
run: ruff check src tests
|
|
50
|
+
- name: Mypy
|
|
51
|
+
run: mypy
|
|
52
|
+
|
|
53
|
+
integration:
|
|
54
|
+
name: Broker integration (Redis · RabbitMQ · SQS)
|
|
55
|
+
runs-on: ubuntu-latest
|
|
56
|
+
services:
|
|
57
|
+
redis:
|
|
58
|
+
image: redis:7
|
|
59
|
+
ports:
|
|
60
|
+
- 6379:6379
|
|
61
|
+
options: >-
|
|
62
|
+
--health-cmd "redis-cli ping"
|
|
63
|
+
--health-interval 5s
|
|
64
|
+
--health-timeout 3s
|
|
65
|
+
--health-retries 10
|
|
66
|
+
rabbitmq:
|
|
67
|
+
image: rabbitmq:3
|
|
68
|
+
ports:
|
|
69
|
+
- 5672:5672
|
|
70
|
+
options: >-
|
|
71
|
+
--health-cmd "rabbitmq-diagnostics -q ping"
|
|
72
|
+
--health-interval 10s
|
|
73
|
+
--health-timeout 5s
|
|
74
|
+
--health-retries 15
|
|
75
|
+
# Free, SQS-compatible broker for the SqsTransport round-trip. The native
|
|
76
|
+
# image is shell-less, so readiness is polled by a TCP-probe step below.
|
|
77
|
+
elasticmq:
|
|
78
|
+
image: softwaremill/elasticmq-native:1.6.11
|
|
79
|
+
ports:
|
|
80
|
+
- 9324:9324
|
|
81
|
+
steps:
|
|
82
|
+
- uses: actions/checkout@v5
|
|
83
|
+
|
|
84
|
+
- name: Setup Python
|
|
85
|
+
uses: actions/setup-python@v5
|
|
86
|
+
with:
|
|
87
|
+
python-version: '3.12'
|
|
88
|
+
|
|
89
|
+
- name: Install (all adapters — full coverage with brokers)
|
|
90
|
+
run: |
|
|
91
|
+
python -m pip install --upgrade pip
|
|
92
|
+
pip install -e ".[redis,amqp,sqs,celery,django,dev]"
|
|
93
|
+
|
|
94
|
+
- name: Wait for ElasticMQ
|
|
95
|
+
run: |
|
|
96
|
+
for i in $(seq 1 30); do
|
|
97
|
+
if (echo > /dev/tcp/localhost/9324) >/dev/null 2>&1; then
|
|
98
|
+
echo "ElasticMQ port 9324 open"; exit 0
|
|
99
|
+
fi
|
|
100
|
+
sleep 2
|
|
101
|
+
done
|
|
102
|
+
echo "ElasticMQ did not open port 9324"; exit 1
|
|
103
|
+
|
|
104
|
+
- name: Run full suite with coverage gate (>=90%)
|
|
105
|
+
env:
|
|
106
|
+
BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
|
|
107
|
+
BABELQUEUE_TEST_AMQP: amqp://guest:guest@localhost:5672/
|
|
108
|
+
SQS_ENDPOINT: http://localhost:9324
|
|
109
|
+
AWS_REGION: us-east-1
|
|
110
|
+
AWS_ACCESS_KEY_ID: test
|
|
111
|
+
AWS_SECRET_ACCESS_KEY: test
|
|
112
|
+
run: pytest --cov=babelqueue --cov-report=term-missing --cov-fail-under=90
|
|
113
|
+
|
|
114
|
+
conformance:
|
|
115
|
+
name: Conformance suite in sync
|
|
116
|
+
runs-on: ubuntu-latest
|
|
117
|
+
steps:
|
|
118
|
+
- uses: actions/checkout@v5
|
|
119
|
+
- name: Verify vendored conformance matches the canonical suite
|
|
120
|
+
run: |
|
|
121
|
+
git clone --depth 1 https://github.com/BabelQueue/conformance.git "$RUNNER_TEMP/conformance"
|
|
122
|
+
diff -ru "$RUNNER_TEMP/conformance/manifest.json" "tests/conformance/manifest.json"
|
|
123
|
+
diff -ru "$RUNNER_TEMP/conformance/fixtures" "tests/conformance/fixtures"
|
|
124
|
+
diff -ru "$RUNNER_TEMP/conformance/schema" "tests/conformance/schema"
|
|
125
|
+
echo "Vendored conformance is in sync with the canonical suite."
|
|
126
|
+
|
|
127
|
+
ci-green:
|
|
128
|
+
name: CI green
|
|
129
|
+
runs-on: ubuntu-latest
|
|
130
|
+
needs: [test, lint, integration, conformance]
|
|
131
|
+
if: ${{ always() }}
|
|
132
|
+
steps:
|
|
133
|
+
- name: Fail if any required job did not pass
|
|
134
|
+
run: |
|
|
135
|
+
if ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}; then
|
|
136
|
+
echo "A required job failed or was cancelled."
|
|
137
|
+
exit 1
|
|
138
|
+
fi
|
|
@@ -7,7 +7,38 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
|
|
7
7
|
The envelope wire format is versioned separately by `meta.schema_version`
|
|
8
8
|
(currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
|
|
9
9
|
|
|
10
|
-
## [
|
|
10
|
+
## [1.1.0] - 2026-06-12
|
|
11
|
+
|
|
12
|
+
### Added
|
|
13
|
+
- **Amazon SQS transport** (`babelqueue[sqs]`, `boto3`) — `SqsTransport`, selected by
|
|
14
|
+
the `sqs://` URL scheme (e.g. `sqs://us-east-1?endpoint=http://localhost:4566` for
|
|
15
|
+
LocalStack). Implements [§3 of the broker-bindings contract](https://babelqueue.com):
|
|
16
|
+
the canonical envelope is the `MessageBody`, projected onto native `MessageAttributes`
|
|
17
|
+
(`bq-job`/`bq-trace-id`/`bq-message-id`/`bq-schema-version`/`bq-source-lang`/`bq-created-at`);
|
|
18
|
+
visibility-timeout reserve → `delete_message` ack; `attempts` reconciled to
|
|
19
|
+
`ApproximateReceiveCount − 1` (never lowering a runtime-incremented count). Supports
|
|
20
|
+
FIFO (`MessageGroupId`/`MessageDeduplicationId` = `meta.id`), content-based dedup,
|
|
21
|
+
configurable wait/visibility, and a `queue_url_prefix` that skips `GetQueueUrl`. A
|
|
22
|
+
client can be injected for tests/DI. 16 unit tests run against a fake client (no boto3,
|
|
23
|
+
no broker); a LocalStack integration test round-trips the real `boto3` path. The
|
|
24
|
+
envelope is unchanged (`schema_version: 1`); SQS is purely additive. Ships as a MINOR.
|
|
25
|
+
|
|
26
|
+
## [1.0.0] - 2026-06-07
|
|
27
|
+
|
|
28
|
+
**1.0.0 — the public API is now SemVer-stable**: breaking changes require a MAJOR,
|
|
29
|
+
following the deprecation policy. The wire envelope is unchanged
|
|
30
|
+
(`schema_version: 1`); the core + Celery/Django adapters ship together. Full
|
|
31
|
+
reference at [babelqueue.com](https://babelqueue.com).
|
|
32
|
+
|
|
33
|
+
### Internal
|
|
34
|
+
- CI adds **ruff** + **mypy** static analysis and a **>=90% coverage gate**
|
|
35
|
+
(`pytest --cov --cov-fail-under=90`, run in the broker-backed job so the Redis /
|
|
36
|
+
RabbitMQ transports count). Type-safety fix in `redis_transport` (str-narrow the
|
|
37
|
+
BLMOVE reply) surfaced by mypy — no behaviour change.
|
|
38
|
+
- **GR-8 latency benchmark** (`tests/test_overhead.py`) — asserts the envelope
|
|
39
|
+
encode/decode path adds **≤2%** over plain-JSON serialization vs a conservative
|
|
40
|
+
2ms broker round-trip (the pure-Python codec is slower than the compiled SDKs —
|
|
41
|
+
~16µs marginal on CPython 3.9/CI — so the reference is higher to stay robust).
|
|
11
42
|
|
|
12
43
|
## [0.5.0] - 2026-06-06
|
|
13
44
|
|
|
@@ -68,7 +99,9 @@ The envelope wire format is versioned separately by `meta.schema_version`
|
|
|
68
99
|
- Pre-1.0: the public API may change before the `1.0.0` tag.
|
|
69
100
|
- The core has **zero runtime dependencies** (standard library only); Python `>=3.9`.
|
|
70
101
|
|
|
71
|
-
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/
|
|
102
|
+
[Unreleased]: https://github.com/BabelQueue/babelqueue-python/compare/v1.1.0...HEAD
|
|
103
|
+
[1.1.0]: https://github.com/BabelQueue/babelqueue-python/compare/v1.0.0...v1.1.0
|
|
104
|
+
[1.0.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.5.0...v1.0.0
|
|
72
105
|
[0.5.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.4.0...v0.5.0
|
|
73
106
|
[0.4.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.3.0...v0.4.0
|
|
74
107
|
[0.3.0]: https://github.com/BabelQueue/babelqueue-python/compare/v0.2.0...v0.3.0
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version:
|
|
3
|
+
Version: 1.1.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -27,11 +27,16 @@ Requires-Dist: pika>=1.3; extra == 'amqp'
|
|
|
27
27
|
Provides-Extra: celery
|
|
28
28
|
Requires-Dist: celery>=5; extra == 'celery'
|
|
29
29
|
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: mypy>=1.8; extra == 'dev'
|
|
31
|
+
Requires-Dist: pytest-cov>=4; extra == 'dev'
|
|
30
32
|
Requires-Dist: pytest>=7; extra == 'dev'
|
|
33
|
+
Requires-Dist: ruff>=0.5; extra == 'dev'
|
|
31
34
|
Provides-Extra: django
|
|
32
35
|
Requires-Dist: django>=4.2; extra == 'django'
|
|
33
36
|
Provides-Extra: redis
|
|
34
37
|
Requires-Dist: redis>=4; extra == 'redis'
|
|
38
|
+
Provides-Extra: sqs
|
|
39
|
+
Requires-Dist: boto3>=1.26; extra == 'sqs'
|
|
35
40
|
Description-Content-Type: text/markdown
|
|
36
41
|
|
|
37
42
|
# BabelQueue for Python
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "
|
|
7
|
+
version = "1.1.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -31,9 +31,10 @@ dependencies = []
|
|
|
31
31
|
# Optional runtime drivers + framework adapters — standard, zero-heavy-dep.
|
|
32
32
|
redis = ["redis>=4"]
|
|
33
33
|
amqp = ["pika>=1.3"]
|
|
34
|
+
sqs = ["boto3>=1.26"]
|
|
34
35
|
celery = ["celery>=5"]
|
|
35
36
|
django = ["django>=4.2"]
|
|
36
|
-
dev = ["pytest>=7"]
|
|
37
|
+
dev = ["pytest>=7", "pytest-cov>=4", "mypy>=1.8", "ruff>=0.5"]
|
|
37
38
|
|
|
38
39
|
[project.urls]
|
|
39
40
|
Homepage = "https://babelqueue.com"
|
|
@@ -43,3 +44,21 @@ Changelog = "https://github.com/BabelQueue/babelqueue-python/blob/main/CHANGELOG
|
|
|
43
44
|
|
|
44
45
|
[tool.hatch.build.targets.wheel]
|
|
45
46
|
packages = ["src/babelqueue"]
|
|
47
|
+
|
|
48
|
+
[tool.ruff]
|
|
49
|
+
target-version = "py39"
|
|
50
|
+
line-length = 100
|
|
51
|
+
|
|
52
|
+
[tool.mypy]
|
|
53
|
+
# Lowest target mypy accepts; the package itself still supports Python 3.9 at
|
|
54
|
+
# runtime (requires-python >=3.9) and uses only 3.9-compatible typing.
|
|
55
|
+
python_version = "3.10"
|
|
56
|
+
files = ["src/babelqueue"]
|
|
57
|
+
ignore_missing_imports = true
|
|
58
|
+
|
|
59
|
+
[tool.coverage.run]
|
|
60
|
+
source = ["babelqueue"]
|
|
61
|
+
|
|
62
|
+
[tool.coverage.report]
|
|
63
|
+
show_missing = true
|
|
64
|
+
exclude_lines = ["pragma: no cover", "raise NotImplementedError", "if TYPE_CHECKING:"]
|
|
@@ -36,10 +36,15 @@ class RedisTransport(Transport):
|
|
|
36
36
|
self._redis.rpush(queue, body)
|
|
37
37
|
|
|
38
38
|
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
39
|
-
|
|
39
|
+
# redis-py types the BLMOVE timeout as int, but Redis accepts a float
|
|
40
|
+
# (sub-second) timeout; passing it through is correct at runtime.
|
|
41
|
+
body = self._redis.blmove(queue, self._processing(queue), timeout, "LEFT", "RIGHT") # type: ignore[arg-type]
|
|
40
42
|
if body is None:
|
|
41
43
|
return None
|
|
42
|
-
|
|
44
|
+
# decode_responses=True yields str; the guard satisfies the type checker
|
|
45
|
+
# (and is a harmless safety net otherwise).
|
|
46
|
+
text = body if isinstance(body, str) else body.decode()
|
|
47
|
+
return ReceivedMessage(body=text, queue=queue, handle=text)
|
|
43
48
|
|
|
44
49
|
def ack(self, message: ReceivedMessage) -> None:
|
|
45
50
|
self._redis.lrem(self._processing(message.queue), 1, message.handle)
|
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""Amazon SQS transport. Requires the ``sqs`` extra:
|
|
2
|
+
|
|
3
|
+
pip install "babelqueue[sqs]"
|
|
4
|
+
|
|
5
|
+
Producing sends the canonical envelope as the message body and projects the
|
|
6
|
+
contract envelope fields onto native SQS ``MessageAttributes`` (``bq-job`` = URN,
|
|
7
|
+
``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
|
|
8
|
+
``bq-source-lang`` / ``bq-created-at``) — so a Go/PHP/... peer can route on ``bq-job``
|
|
9
|
+
and correlate on ``bq-trace-id`` without parsing the body. Consuming uses the
|
|
10
|
+
visibility-timeout reservation model (``receive_message`` -> process ->
|
|
11
|
+
``delete_message``); the authoritative attempt count is the broker's
|
|
12
|
+
``ApproximateReceiveCount``, reconciled onto the envelope as ``attempts = count - 1``.
|
|
13
|
+
|
|
14
|
+
This implements §3 of the broker-bindings contract. The envelope is unchanged
|
|
15
|
+
(``schema_version`` stays 1); SQS is purely additive.
|
|
16
|
+
|
|
17
|
+
URL form: ``sqs://[region][?endpoint=...&prefix=...&fifo=1&group_id=...&wait_time=20]``
|
|
18
|
+
(e.g. ``sqs://us-east-1?endpoint=http://localhost:4566`` for LocalStack). Credentials
|
|
19
|
+
come from the standard AWS default provider chain. For richer setups, build the
|
|
20
|
+
transport directly and pass it via ``BabelQueue(transport=...)``.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
from typing import Any, Dict, Optional
|
|
26
|
+
from urllib.parse import parse_qs, urlsplit
|
|
27
|
+
|
|
28
|
+
from .codec import EnvelopeCodec
|
|
29
|
+
from .transport import ReceivedMessage, Transport
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class SqsTransport(Transport):
|
|
33
|
+
def __init__(
|
|
34
|
+
self,
|
|
35
|
+
url: str = "sqs://",
|
|
36
|
+
*,
|
|
37
|
+
client: Any = None,
|
|
38
|
+
region: Optional[str] = None,
|
|
39
|
+
endpoint: Optional[str] = None,
|
|
40
|
+
queue_url_prefix: Optional[str] = None,
|
|
41
|
+
wait_time: Optional[int] = None,
|
|
42
|
+
visibility_timeout: Optional[int] = None,
|
|
43
|
+
fifo: bool = False,
|
|
44
|
+
message_group_id: Optional[str] = None,
|
|
45
|
+
content_dedup: bool = False,
|
|
46
|
+
) -> None:
|
|
47
|
+
parts = urlsplit(url) if url else urlsplit("sqs://")
|
|
48
|
+
q = parse_qs(parts.query)
|
|
49
|
+
|
|
50
|
+
self._region = region or (parts.hostname or None)
|
|
51
|
+
self._endpoint = endpoint or _q1(q, "endpoint")
|
|
52
|
+
self._queue_url_prefix = queue_url_prefix or _q1(q, "prefix")
|
|
53
|
+
self._wait_time = wait_time if wait_time is not None else _qint(q, "wait_time")
|
|
54
|
+
self._visibility_timeout = (
|
|
55
|
+
visibility_timeout if visibility_timeout is not None else _qint(q, "visibility_timeout")
|
|
56
|
+
)
|
|
57
|
+
self._fifo = fifo or _qbool(q, "fifo")
|
|
58
|
+
self._message_group_id = message_group_id or _q1(q, "group_id")
|
|
59
|
+
self._content_dedup = content_dedup or _qbool(q, "content_dedup")
|
|
60
|
+
self._urls: Dict[str, str] = {}
|
|
61
|
+
|
|
62
|
+
if client is not None:
|
|
63
|
+
self._sqs = client
|
|
64
|
+
return
|
|
65
|
+
try:
|
|
66
|
+
import boto3
|
|
67
|
+
except ImportError as exc: # pragma: no cover - import guard
|
|
68
|
+
raise ImportError(
|
|
69
|
+
"SqsTransport requires the 'boto3' package. Install with "
|
|
70
|
+
'pip install "babelqueue[sqs]".'
|
|
71
|
+
) from exc
|
|
72
|
+
kwargs: Dict[str, Any] = {}
|
|
73
|
+
if self._region:
|
|
74
|
+
kwargs["region_name"] = self._region
|
|
75
|
+
if self._endpoint:
|
|
76
|
+
kwargs["endpoint_url"] = self._endpoint
|
|
77
|
+
self._sqs = boto3.client("sqs", **kwargs) # pragma: no cover - needs AWS/LocalStack
|
|
78
|
+
|
|
79
|
+
# -- helpers ------------------------------------------------------------
|
|
80
|
+
|
|
81
|
+
def _resolve_url(self, name: str) -> str:
|
|
82
|
+
cached = self._urls.get(name)
|
|
83
|
+
if cached is not None:
|
|
84
|
+
return cached
|
|
85
|
+
if self._queue_url_prefix:
|
|
86
|
+
url = self._queue_url_prefix.rstrip("/") + "/" + name
|
|
87
|
+
else:
|
|
88
|
+
url = self._sqs.get_queue_url(QueueName=name)["QueueUrl"]
|
|
89
|
+
self._urls[name] = url
|
|
90
|
+
return url
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def _attributes(body: str) -> Dict[str, Dict[str, str]]:
|
|
94
|
+
"""Project the envelope's contract fields onto SQS MessageAttributes — a
|
|
95
|
+
redundant, routable view of the body (the body stays authoritative)."""
|
|
96
|
+
try:
|
|
97
|
+
env: Dict[str, Any] = EnvelopeCodec.decode(body)
|
|
98
|
+
except (ValueError, TypeError): # pragma: no cover - decode is defensive
|
|
99
|
+
return {}
|
|
100
|
+
meta = env.get("meta") or {}
|
|
101
|
+
|
|
102
|
+
def s(v: Any) -> Dict[str, str]:
|
|
103
|
+
return {"DataType": "String", "StringValue": str(v)}
|
|
104
|
+
|
|
105
|
+
def n(v: Any) -> Dict[str, str]:
|
|
106
|
+
return {"DataType": "Number", "StringValue": str(v)}
|
|
107
|
+
|
|
108
|
+
attrs: Dict[str, Dict[str, str]] = {}
|
|
109
|
+
if env.get("job"):
|
|
110
|
+
attrs["bq-job"] = s(env["job"])
|
|
111
|
+
if env.get("trace_id"):
|
|
112
|
+
attrs["bq-trace-id"] = s(env["trace_id"])
|
|
113
|
+
if meta.get("id"):
|
|
114
|
+
attrs["bq-message-id"] = s(meta["id"])
|
|
115
|
+
if meta.get("schema_version") is not None:
|
|
116
|
+
attrs["bq-schema-version"] = n(meta["schema_version"])
|
|
117
|
+
if meta.get("lang"):
|
|
118
|
+
attrs["bq-source-lang"] = s(meta["lang"])
|
|
119
|
+
if meta.get("created_at") is not None:
|
|
120
|
+
attrs["bq-created-at"] = n(meta["created_at"])
|
|
121
|
+
return attrs
|
|
122
|
+
|
|
123
|
+
@staticmethod
|
|
124
|
+
def _reconcile(body: str, receive_count: Any) -> str:
|
|
125
|
+
"""Set attempts to max(current, ApproximateReceiveCount - 1): a first delivery
|
|
126
|
+
reads 0, a natively-redelivered message reflects its true count, and a
|
|
127
|
+
runtime-incremented counter is never lowered."""
|
|
128
|
+
try:
|
|
129
|
+
rc = int(receive_count)
|
|
130
|
+
except (ValueError, TypeError):
|
|
131
|
+
return body
|
|
132
|
+
if rc <= 1:
|
|
133
|
+
return body
|
|
134
|
+
env = EnvelopeCodec.decode(body)
|
|
135
|
+
if not env:
|
|
136
|
+
return body
|
|
137
|
+
native = rc - 1
|
|
138
|
+
if native <= int(env.get("attempts", 0)):
|
|
139
|
+
return body
|
|
140
|
+
env["attempts"] = native
|
|
141
|
+
return EnvelopeCodec.encode(env)
|
|
142
|
+
|
|
143
|
+
# -- Transport ----------------------------------------------------------
|
|
144
|
+
|
|
145
|
+
def publish(self, queue: str, body: str) -> None:
|
|
146
|
+
params: Dict[str, Any] = {"QueueUrl": self._resolve_url(queue), "MessageBody": body}
|
|
147
|
+
attrs = self._attributes(body)
|
|
148
|
+
if attrs:
|
|
149
|
+
params["MessageAttributes"] = attrs
|
|
150
|
+
if self._fifo:
|
|
151
|
+
params["MessageGroupId"] = self._message_group_id or queue
|
|
152
|
+
if not self._content_dedup:
|
|
153
|
+
msg_id = (EnvelopeCodec.decode(body).get("meta") or {}).get("id")
|
|
154
|
+
if msg_id:
|
|
155
|
+
params["MessageDeduplicationId"] = msg_id
|
|
156
|
+
self._sqs.send_message(**params)
|
|
157
|
+
|
|
158
|
+
def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
|
|
159
|
+
wait = int(timeout) if timeout and timeout > 0 else 0
|
|
160
|
+
if wait > 20:
|
|
161
|
+
wait = 20
|
|
162
|
+
if self._wait_time is not None and self._wait_time < wait:
|
|
163
|
+
wait = self._wait_time
|
|
164
|
+
params: Dict[str, Any] = {
|
|
165
|
+
"QueueUrl": self._resolve_url(queue),
|
|
166
|
+
"MaxNumberOfMessages": 1,
|
|
167
|
+
"WaitTimeSeconds": wait,
|
|
168
|
+
"MessageAttributeNames": ["All"],
|
|
169
|
+
"AttributeNames": ["ApproximateReceiveCount"],
|
|
170
|
+
}
|
|
171
|
+
if self._visibility_timeout is not None:
|
|
172
|
+
params["VisibilityTimeout"] = self._visibility_timeout
|
|
173
|
+
resp = self._sqs.receive_message(**params)
|
|
174
|
+
messages = resp.get("Messages") or []
|
|
175
|
+
if not messages:
|
|
176
|
+
return None
|
|
177
|
+
msg = messages[0]
|
|
178
|
+
body = msg.get("Body", "")
|
|
179
|
+
receive_count = (msg.get("Attributes") or {}).get("ApproximateReceiveCount")
|
|
180
|
+
if receive_count is not None:
|
|
181
|
+
body = self._reconcile(body, receive_count)
|
|
182
|
+
return ReceivedMessage(body=body, queue=queue, handle=msg.get("ReceiptHandle"))
|
|
183
|
+
|
|
184
|
+
def ack(self, message: ReceivedMessage) -> None:
|
|
185
|
+
if not message.handle:
|
|
186
|
+
return
|
|
187
|
+
self._sqs.delete_message(
|
|
188
|
+
QueueUrl=self._resolve_url(message.queue), ReceiptHandle=message.handle
|
|
189
|
+
)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _q1(q: Dict[str, list], key: str) -> Optional[str]:
|
|
193
|
+
values = q.get(key)
|
|
194
|
+
return values[0] if values else None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _qint(q: Dict[str, list], key: str) -> Optional[int]:
|
|
198
|
+
v = _q1(q, key)
|
|
199
|
+
if v is None:
|
|
200
|
+
return None
|
|
201
|
+
try:
|
|
202
|
+
return int(v)
|
|
203
|
+
except ValueError: # pragma: no cover - defensive
|
|
204
|
+
return None
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def _qbool(q: Dict[str, list], key: str) -> bool:
|
|
208
|
+
v = _q1(q, key)
|
|
209
|
+
return v is not None and v.lower() in ("1", "true", "yes", "on")
|
|
@@ -8,7 +8,7 @@ from __future__ import annotations
|
|
|
8
8
|
|
|
9
9
|
from abc import ABC, abstractmethod
|
|
10
10
|
from collections import defaultdict, deque
|
|
11
|
-
from dataclasses import dataclass
|
|
11
|
+
from dataclasses import dataclass
|
|
12
12
|
from typing import Any, Deque, Dict, Optional
|
|
13
13
|
|
|
14
14
|
from .exceptions import BabelQueueError
|
|
@@ -79,8 +79,12 @@ def make_transport(broker_url: str) -> Transport:
|
|
|
79
79
|
from .pika_transport import PikaTransport
|
|
80
80
|
|
|
81
81
|
return PikaTransport(broker_url)
|
|
82
|
+
if scheme == "sqs":
|
|
83
|
+
from .sqs_transport import SqsTransport
|
|
84
|
+
|
|
85
|
+
return SqsTransport(broker_url)
|
|
82
86
|
|
|
83
87
|
raise BabelQueueError(
|
|
84
|
-
f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://'
|
|
85
|
-
"'amqp://', or pass your own Transport via BabelQueue(transport=...)."
|
|
88
|
+
f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://', "
|
|
89
|
+
"'amqp://' or 'sqs://', or pass your own Transport via BabelQueue(transport=...)."
|
|
86
90
|
)
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema_version": 1,
|
|
3
|
+
"description": "Cross-SDK conformance cases. Every BabelQueue SDK core must satisfy these against the canonical wire envelope. Per-message fields (meta.id, trace_id, meta.created_at) are intrinsically unique and are NOT asserted by value.",
|
|
4
|
+
"cases": [
|
|
5
|
+
{
|
|
6
|
+
"name": "order-created",
|
|
7
|
+
"file": "fixtures/order-created.json",
|
|
8
|
+
"valid": true,
|
|
9
|
+
"description": "A normal produced envelope.",
|
|
10
|
+
"expect": {
|
|
11
|
+
"urn": "urn:babel:orders:created",
|
|
12
|
+
"data": { "order_id": 1042 },
|
|
13
|
+
"attempts": 0,
|
|
14
|
+
"lang": "php",
|
|
15
|
+
"schema_version": 1
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
"name": "urn-alias",
|
|
20
|
+
"file": "fixtures/urn-alias.json",
|
|
21
|
+
"valid": true,
|
|
22
|
+
"description": "Consumers MUST accept 'urn' as an inbound alias for 'job'.",
|
|
23
|
+
"expect": {
|
|
24
|
+
"urn": "urn:babel:orders:created",
|
|
25
|
+
"data": { "order_id": 1042 },
|
|
26
|
+
"attempts": 0,
|
|
27
|
+
"lang": "go",
|
|
28
|
+
"schema_version": 1
|
|
29
|
+
}
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"name": "dead-lettered",
|
|
33
|
+
"file": "fixtures/dead-lettered.json",
|
|
34
|
+
"valid": true,
|
|
35
|
+
"description": "A dead-lettered message: original preserved + additive dead_letter block.",
|
|
36
|
+
"expect": {
|
|
37
|
+
"urn": "urn:babel:orders:created",
|
|
38
|
+
"data": { "order_id": 1042 },
|
|
39
|
+
"attempts": 3,
|
|
40
|
+
"lang": "php",
|
|
41
|
+
"schema_version": 1,
|
|
42
|
+
"dead_letter": { "reason": "failed", "original_queue": "orders" }
|
|
43
|
+
}
|
|
44
|
+
},
|
|
45
|
+
{
|
|
46
|
+
"name": "unicode-and-numbers",
|
|
47
|
+
"file": "fixtures/unicode-and-numbers.json",
|
|
48
|
+
"valid": true,
|
|
49
|
+
"description": "UTF-8 strings, integers, an exact float, boolean and null round-trip identically.",
|
|
50
|
+
"expect": {
|
|
51
|
+
"urn": "urn:babel:catalog:item.indexed",
|
|
52
|
+
"data": { "title": "Café — naïve ☕", "qty": 7, "price_cents": 1299, "ratio": 0.5, "active": true, "note": null },
|
|
53
|
+
"attempts": 2,
|
|
54
|
+
"lang": "python",
|
|
55
|
+
"schema_version": 1
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
"name": "invalid-unknown-schema-version",
|
|
60
|
+
"file": "fixtures/invalid-unknown-schema-version.json",
|
|
61
|
+
"valid": false,
|
|
62
|
+
"reason": "meta.schema_version is not a version this SDK supports"
|
|
63
|
+
},
|
|
64
|
+
{
|
|
65
|
+
"name": "invalid-missing-urn",
|
|
66
|
+
"file": "fixtures/invalid-missing-urn.json",
|
|
67
|
+
"valid": false,
|
|
68
|
+
"reason": "no 'job' or 'urn' — the message has no identity"
|
|
69
|
+
}
|
|
70
|
+
],
|
|
71
|
+
"sqs": {
|
|
72
|
+
"description": "Amazon SQS binding conformance (broker-bindings.md §3). Every SDK that ships an SQS transport must satisfy these. The envelope body stays byte-identical (the 'cases' above); these lock the native projection + reconciliation the binding adds. Per-message values reuse fixtures/order-created.json so the expected attributes are deterministic.",
|
|
73
|
+
"attribute_projection": {
|
|
74
|
+
"description": "On produce, the transport MUST project these native MessageAttributes from the envelope (a redundant, routable view of the body; ids/strings are DataType String, counters are DataType Number). Applies to every SQS-producing SDK.",
|
|
75
|
+
"envelope_file": "fixtures/order-created.json",
|
|
76
|
+
"message_attributes": {
|
|
77
|
+
"bq-job": { "DataType": "String", "StringValue": "urn:babel:orders:created" },
|
|
78
|
+
"bq-trace-id": { "DataType": "String", "StringValue": "7b3f9c2a-e41d-4f88-9b2a-1c0d5e6f7a8b" },
|
|
79
|
+
"bq-message-id": { "DataType": "String", "StringValue": "f1e2d3c4-b5a6-4789-90ab-cdef01234567" },
|
|
80
|
+
"bq-schema-version": { "DataType": "Number", "StringValue": "1" },
|
|
81
|
+
"bq-source-lang": { "DataType": "String", "StringValue": "php" },
|
|
82
|
+
"bq-created-at": { "DataType": "Number", "StringValue": "1749132727000" }
|
|
83
|
+
}
|
|
84
|
+
},
|
|
85
|
+
"attempts_reconciliation": {
|
|
86
|
+
"description": "On consume, attempts = max(body.attempts, ApproximateReceiveCount - 1): a first delivery reads 0, an absent/garbage count is ignored, a runtime-incremented count is never lowered. Applies to SDKs that reconcile the envelope body on consume (the framework-less/runtime transports). A drop-in driver that surfaces the broker's native delivery count instead (e.g. Laravel's SqsJob.attempts() = ApproximateReceiveCount) is exempt — it documents that divergence.",
|
|
87
|
+
"cases": [
|
|
88
|
+
{ "name": "first-delivery", "body_attempts": 0, "approximate_receive_count": "1", "expected_attempts": 0 },
|
|
89
|
+
{ "name": "third-delivery", "body_attempts": 0, "approximate_receive_count": "3", "expected_attempts": 2 },
|
|
90
|
+
{ "name": "native-exceeds-body", "body_attempts": 2, "approximate_receive_count": "5", "expected_attempts": 4 },
|
|
91
|
+
{ "name": "never-lower-runtime", "body_attempts": 5, "approximate_receive_count": "1", "expected_attempts": 5 },
|
|
92
|
+
{ "name": "garbage-count-ignored", "body_attempts": 4, "approximate_receive_count": "not-a-number", "expected_attempts": 4 },
|
|
93
|
+
{ "name": "absent-count", "body_attempts": 3, "approximate_receive_count": null, "expected_attempts": 3 }
|
|
94
|
+
]
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
}
|