babelqueue 1.15.0__tar.gz → 1.16.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {babelqueue-1.15.0 → babelqueue-1.16.0}/.github/workflows/ci.yml +8 -8
- {babelqueue-1.15.0 → babelqueue-1.16.0}/.github/workflows/release.yml +3 -3
- {babelqueue-1.15.0 → babelqueue-1.16.0}/CHANGELOG.md +34 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/PKG-INFO +38 -1
- {babelqueue-1.15.0 → babelqueue-1.16.0}/README.md +37 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/pyproject.toml +1 -1
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/__init__.py +3 -1
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/app.py +58 -12
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/artemis_transport.py +63 -2
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/asb_transport.py +73 -3
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/kafka_transport.py +92 -3
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/pulsar_transport.py +5 -5
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/sqs_transport.py +61 -5
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/transport.py +28 -0
- babelqueue-1.16.0/tests/conformance/CONFORMANCE_VERSION +1 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/manifest.json +129 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_app.py +101 -20
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_artemis_conformance.py +27 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_artemis_transport.py +132 -2
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_asb_conformance.py +27 -0
- babelqueue-1.16.0/tests/test_asb_transport.py +327 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_kafka_conformance.py +27 -0
- babelqueue-1.16.0/tests/test_kafka_transport.py +438 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_pulsar_conformance.py +27 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_pulsar_transport.py +8 -3
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_sqs_conformance.py +27 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_sqs_transport.py +161 -0
- babelqueue-1.15.0/tests/conformance/CONFORMANCE_VERSION +0 -1
- babelqueue-1.15.0/tests/test_asb_transport.py +0 -191
- babelqueue-1.15.0/tests/test_kafka_transport.py +0 -194
- {babelqueue-1.15.0 → babelqueue-1.16.0}/.github/FUNDING.yml +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/.github/dependabot.yml +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/.gitignore +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/LICENSE +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/celery.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/codec.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/contracts.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/dead_letter.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/django/__init__.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/django/apps.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/django/management/__init__.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/django/management/commands/__init__.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/django/management/commands/babelqueue_worker.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/exceptions.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/gdpr.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/headers.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/idempotency.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/otel.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/outbox.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/pika_transport.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/py.typed +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/redis_transport.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/redrive.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/replay.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/routing.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/src/babelqueue/schema.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/data-array-rejected.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/empty-data-roundtrip.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/forbidden-key-meta-attempts.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/forbidden-key-meta-max-retries.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/forbidden-key-meta-source.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/forbidden-key-meta-ts.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/forbidden-key-timestamp.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/invalid-missing-urn.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/invalid-unknown-schema-version.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/order-created.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/unicode-and-numbers.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/unknown-lang.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/unknown-meta-roundtrip.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/unknown-toplevel-roundtrip.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/fixtures/urn-alias.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/conformance/schema/message-envelope.schema.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/fixtures/dead-lettered.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/fixtures/order-created.json +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_behaviour_conformance.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_celery.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_codec.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_conformance.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_dead_letter.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_django.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_gdpr.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_headers.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_idempotency.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_otel.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_outbox.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_overhead.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_pika_transport.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_redis_transport.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_redrive.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_replay.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_schema.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_schema_conformance.py +0 -0
- {babelqueue-1.15.0 → babelqueue-1.16.0}/tests/test_shutdown.py +0 -0
|
@@ -17,10 +17,10 @@ jobs:
|
|
|
17
17
|
matrix:
|
|
18
18
|
python: ['3.9', '3.10', '3.11', '3.12', '3.13']
|
|
19
19
|
steps:
|
|
20
|
-
- uses: actions/checkout@
|
|
20
|
+
- uses: actions/checkout@v7
|
|
21
21
|
|
|
22
22
|
- name: Setup Python
|
|
23
|
-
uses: actions/setup-python@
|
|
23
|
+
uses: actions/setup-python@v7
|
|
24
24
|
with:
|
|
25
25
|
python-version: ${{ matrix.python }}
|
|
26
26
|
|
|
@@ -36,9 +36,9 @@ jobs:
|
|
|
36
36
|
name: Static analysis (ruff + mypy)
|
|
37
37
|
runs-on: ubuntu-latest
|
|
38
38
|
steps:
|
|
39
|
-
- uses: actions/checkout@
|
|
39
|
+
- uses: actions/checkout@v7
|
|
40
40
|
- name: Setup Python
|
|
41
|
-
uses: actions/setup-python@
|
|
41
|
+
uses: actions/setup-python@v7
|
|
42
42
|
with:
|
|
43
43
|
python-version: '3.12'
|
|
44
44
|
- name: Install (dev + all adapters for type context)
|
|
@@ -79,10 +79,10 @@ jobs:
|
|
|
79
79
|
ports:
|
|
80
80
|
- 9324:9324
|
|
81
81
|
steps:
|
|
82
|
-
- uses: actions/checkout@
|
|
82
|
+
- uses: actions/checkout@v7
|
|
83
83
|
|
|
84
84
|
- name: Setup Python
|
|
85
|
-
uses: actions/setup-python@
|
|
85
|
+
uses: actions/setup-python@v7
|
|
86
86
|
with:
|
|
87
87
|
python-version: '3.12'
|
|
88
88
|
|
|
@@ -115,10 +115,10 @@ jobs:
|
|
|
115
115
|
name: Conformance suite in sync
|
|
116
116
|
runs-on: ubuntu-latest
|
|
117
117
|
steps:
|
|
118
|
-
- uses: actions/checkout@
|
|
118
|
+
- uses: actions/checkout@v7
|
|
119
119
|
- name: Verify vendored conformance matches the canonical suite
|
|
120
120
|
run: |
|
|
121
|
-
git clone --depth 1 --branch v1.
|
|
121
|
+
git clone --depth 1 --branch v1.1.0 https://github.com/BabelQueue/conformance.git "$RUNNER_TEMP/conformance"
|
|
122
122
|
diff -u "$RUNNER_TEMP/conformance/CONFORMANCE_VERSION" "tests/conformance/CONFORMANCE_VERSION"
|
|
123
123
|
diff -ru "$RUNNER_TEMP/conformance/manifest.json" "tests/conformance/manifest.json"
|
|
124
124
|
diff -ru "$RUNNER_TEMP/conformance/fixtures" "tests/conformance/fixtures"
|
|
@@ -18,10 +18,10 @@ jobs:
|
|
|
18
18
|
id-token: write # PyPI Trusted Publishing (OIDC) — no API token needed
|
|
19
19
|
contents: write # create the GitHub release
|
|
20
20
|
steps:
|
|
21
|
-
- uses: actions/checkout@
|
|
21
|
+
- uses: actions/checkout@v7
|
|
22
22
|
|
|
23
23
|
- name: Setup Python
|
|
24
|
-
uses: actions/setup-python@
|
|
24
|
+
uses: actions/setup-python@v7
|
|
25
25
|
with:
|
|
26
26
|
python-version: '3.12'
|
|
27
27
|
|
|
@@ -41,6 +41,6 @@ jobs:
|
|
|
41
41
|
# password: ${{ secrets.PYPI_API_TOKEN }}
|
|
42
42
|
|
|
43
43
|
- name: Create GitHub release
|
|
44
|
-
uses: softprops/action-gh-release@
|
|
44
|
+
uses: softprops/action-gh-release@v3
|
|
45
45
|
with:
|
|
46
46
|
generate_release_notes: true
|
|
@@ -9,6 +9,40 @@ The envelope wire format is versioned separately by `meta.schema_version`
|
|
|
9
9
|
|
|
10
10
|
## [Unreleased]
|
|
11
11
|
|
|
12
|
+
## [1.16.0] - 2026-10-10
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
- New optional `Rewinder` protocol (`rewind(message)`), implemented by `KafkaTransport` and
|
|
16
|
+
exported from `babelqueue` (public API, MINOR). The runtime calls it after a failed
|
|
17
|
+
version-gate dead-letter write and then pauses briefly (1 s) before the next poll.
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
- **`bq-schema-version` pre-decode gate on the SQS, Azure Service Bus, Kafka and Artemis
|
|
21
|
+
transports** (`pop`). The broker's schema-version property is read **before** the body is
|
|
22
|
+
decoded (broker-bindings §3.7 / §4.7 / §6.7 / §7.7), mirroring the existing Pulsar gate:
|
|
23
|
+
a missing or blank value (empty or only ASCII whitespace) takes the normal path; anything
|
|
24
|
+
else is compared untrimmed to `"1"` and a mismatch is returned with the raw body and the
|
|
25
|
+
reserved route `babelqueue:unsupported-schema-version:<v>` (version percent-escaped) without
|
|
26
|
+
decoding it. SQS/ASB/Kafka read `bq-schema-version`, Artemis `bq_schema_version`
|
|
27
|
+
(ADR-0017); ASB and Artemis also accept an integer `1` (a `bool` is not `1`). The conformance suite now reads each binding's
|
|
28
|
+
`schema_version_gate` table (including Pulsar's). No wire change.
|
|
29
|
+
- **A version-gated delivery is dead-lettered directly, never retried.** `BabelQueue` now
|
|
30
|
+
sends a message under the reserved gate route straight to the DLQ (reason `unknown_urn`,
|
|
31
|
+
body untouched) when `dead_letter` is enabled and drops it otherwise, regardless of
|
|
32
|
+
`max_attempts` / `on_unknown_urn`. A retry would re-publish through paths that rebuild the
|
|
33
|
+
version header from the (valid) body (ASB/Kafka/Artemis) or never advance the attempt
|
|
34
|
+
counter (SQS), letting the gate be bypassed or the message loop. Applies to Pulsar too
|
|
35
|
+
(previously it retried up to `max_attempts` first).
|
|
36
|
+
If the dead-letter write itself fails, the message is still never retried: it is left
|
|
37
|
+
unacknowledged (an error is logged) so the broker redelivers it and the gate rejects it again
|
|
38
|
+
(SQS at the visibility timeout, Azure Service Bus at lock expiry, Artemis when the link or
|
|
39
|
+
connection closes). Kafka commits offsets cumulatively, so merely not acknowledging the
|
|
40
|
+
record would lose it once a later record is committed: `KafkaTransport` now `seek`s the
|
|
41
|
+
partition back to the failed record's offset so it is polled again and the dead-letter write
|
|
42
|
+
is retried (no commit moves past it).
|
|
43
|
+
- The rejected `bq-schema-version` value is capped at 64 characters plus `...` in the log and
|
|
44
|
+
the route.
|
|
45
|
+
|
|
12
46
|
## [1.15.0] - 2026-10-10
|
|
13
47
|
|
|
14
48
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: babelqueue
|
|
3
|
-
Version: 1.
|
|
3
|
+
Version: 1.16.0
|
|
4
4
|
Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
|
|
5
5
|
Project-URL: Homepage, https://babelqueue.com
|
|
6
6
|
Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
|
|
@@ -178,6 +178,43 @@ app.run() # consume forever (Ctrl-C to stop)
|
|
|
178
178
|
the message unchanged once `shutdown_timeout` expires. Delivery is
|
|
179
179
|
**at-least-once** — keep handlers idempotent (see the idempotency helper).
|
|
180
180
|
|
|
181
|
+
### Unsupported `schema_version` (SQS, Azure Service Bus, Kafka, Artemis, Pulsar)
|
|
182
|
+
|
|
183
|
+
These transports read the binding's schema-version property (`bq-schema-version`; Artemis
|
|
184
|
+
`bq_schema_version`) **before** decoding the body. A missing or blank value (empty or only
|
|
185
|
+
ASCII whitespace) follows the normal path; any other value must be exactly `"1"` (compared
|
|
186
|
+
untrimmed; Azure Service Bus and Artemis also accept the integer `1`). A message carrying
|
|
187
|
+
anything else is not decoded: it reaches the runtime on the reserved route
|
|
188
|
+
`babelqueue:unsupported-schema-version:<v>` (the version percent-escaped) and is **never
|
|
189
|
+
retried** — it goes straight to the dead-letter queue (reason `unknown_urn`, the body kept
|
|
190
|
+
verbatim) when `dead_letter` is enabled, and is dropped otherwise, whatever `max_attempts` is.
|
|
191
|
+
It does not depend on a broker `RedrivePolicy`. If the dead-letter write fails the message is
|
|
192
|
+
left unacknowledged (an error is logged), is never retried and never reaches a handler; the
|
|
193
|
+
dead-letter write is attempted again when the broker brings it back, which depends on the
|
|
194
|
+
broker:
|
|
195
|
+
|
|
196
|
+
- **SQS** — redelivered when the visibility timeout expires.
|
|
197
|
+
- **Azure Service Bus** — redelivered when the message lock expires.
|
|
198
|
+
- **Artemis** — an unsettled AMQP 1.0 delivery is returned only when the link or connection
|
|
199
|
+
closes, so it stays unsettled until then.
|
|
200
|
+
- **Kafka** — offsets are committed cumulatively per partition, so merely skipping the commit
|
|
201
|
+
would lose the record once a later one is committed. The transport therefore implements
|
|
202
|
+
the optional `Rewinder` protocol: it `seek`s the partition back to the failed record's
|
|
203
|
+
offset and pauses `_rewind_pause` (1 s by default, so an unavailable DLQ is not hammered)
|
|
204
|
+
before the record is polled again; the gate rejects it again and the dead-letter write
|
|
205
|
+
is retried. While the dead-letter write keeps failing (for example the `.dlq` topic is not
|
|
206
|
+
writable) the partition **stalls on that record** (head-of-line blocking, consumer lag
|
|
207
|
+
grows) and the worker waits the pause on every attempt, which also delays its other
|
|
208
|
+
partitions and queues. The commit does not move past the record as long as the seek
|
|
209
|
+
succeeds; if the seek itself fails, an error naming the topic, partition and offset is
|
|
210
|
+
logged and a later commit on that partition may skip the record, so recover it by hand from
|
|
211
|
+
that offset. If the partition was lost to a rebalance, the new owner resumes from the last
|
|
212
|
+
committed offset and the record is not lost.
|
|
213
|
+
- **Pulsar** — acknowledgement is per message; the unacknowledged message is redelivered by
|
|
214
|
+
the broker.
|
|
215
|
+
|
|
216
|
+
A warning naming the version (cut at 64 characters plus `...`) and queue is logged.
|
|
217
|
+
|
|
181
218
|
### SQS release and poison messages
|
|
182
219
|
|
|
183
220
|
On a handler failure, unknown-URN `release` or shutdown release, the SQS
|
|
@@ -124,6 +124,43 @@ app.run() # consume forever (Ctrl-C to stop)
|
|
|
124
124
|
the message unchanged once `shutdown_timeout` expires. Delivery is
|
|
125
125
|
**at-least-once** — keep handlers idempotent (see the idempotency helper).
|
|
126
126
|
|
|
127
|
+
### Unsupported `schema_version` (SQS, Azure Service Bus, Kafka, Artemis, Pulsar)
|
|
128
|
+
|
|
129
|
+
These transports read the binding's schema-version property (`bq-schema-version`; Artemis
|
|
130
|
+
`bq_schema_version`) **before** decoding the body. A missing or blank value (empty or only
|
|
131
|
+
ASCII whitespace) follows the normal path; any other value must be exactly `"1"` (compared
|
|
132
|
+
untrimmed; Azure Service Bus and Artemis also accept the integer `1`). A message carrying
|
|
133
|
+
anything else is not decoded: it reaches the runtime on the reserved route
|
|
134
|
+
`babelqueue:unsupported-schema-version:<v>` (the version percent-escaped) and is **never
|
|
135
|
+
retried** — it goes straight to the dead-letter queue (reason `unknown_urn`, the body kept
|
|
136
|
+
verbatim) when `dead_letter` is enabled, and is dropped otherwise, whatever `max_attempts` is.
|
|
137
|
+
It does not depend on a broker `RedrivePolicy`. If the dead-letter write fails the message is
|
|
138
|
+
left unacknowledged (an error is logged), is never retried and never reaches a handler; the
|
|
139
|
+
dead-letter write is attempted again when the broker brings it back, which depends on the
|
|
140
|
+
broker:
|
|
141
|
+
|
|
142
|
+
- **SQS** — redelivered when the visibility timeout expires.
|
|
143
|
+
- **Azure Service Bus** — redelivered when the message lock expires.
|
|
144
|
+
- **Artemis** — an unsettled AMQP 1.0 delivery is returned only when the link or connection
|
|
145
|
+
closes, so it stays unsettled until then.
|
|
146
|
+
- **Kafka** — offsets are committed cumulatively per partition, so merely skipping the commit
|
|
147
|
+
would lose the record once a later one is committed. The transport therefore implements
|
|
148
|
+
the optional `Rewinder` protocol: it `seek`s the partition back to the failed record's
|
|
149
|
+
offset and pauses `_rewind_pause` (1 s by default, so an unavailable DLQ is not hammered)
|
|
150
|
+
before the record is polled again; the gate rejects it again and the dead-letter write
|
|
151
|
+
is retried. While the dead-letter write keeps failing (for example the `.dlq` topic is not
|
|
152
|
+
writable) the partition **stalls on that record** (head-of-line blocking, consumer lag
|
|
153
|
+
grows) and the worker waits the pause on every attempt, which also delays its other
|
|
154
|
+
partitions and queues. The commit does not move past the record as long as the seek
|
|
155
|
+
succeeds; if the seek itself fails, an error naming the topic, partition and offset is
|
|
156
|
+
logged and a later commit on that partition may skip the record, so recover it by hand from
|
|
157
|
+
that offset. If the partition was lost to a rebalance, the new owner resumes from the last
|
|
158
|
+
committed offset and the record is not lost.
|
|
159
|
+
- **Pulsar** — acknowledgement is per message; the unacknowledged message is redelivered by
|
|
160
|
+
the broker.
|
|
161
|
+
|
|
162
|
+
A warning naming the version (cut at 64 characters plus `...`) and queue is logged.
|
|
163
|
+
|
|
127
164
|
### SQS release and poison messages
|
|
128
165
|
|
|
129
166
|
On a handler failure, unknown-URN `release` or shutdown release, the SQS
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "babelqueue"
|
|
7
|
-
version = "1.
|
|
7
|
+
version = "1.16.0"
|
|
8
8
|
description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.9"
|
|
@@ -34,11 +34,12 @@ from .transport import (
|
|
|
34
34
|
InMemoryTransport,
|
|
35
35
|
ReceivedMessage,
|
|
36
36
|
Redeliverer,
|
|
37
|
+
Rewinder,
|
|
37
38
|
RoutedPublisher,
|
|
38
39
|
Transport,
|
|
39
40
|
)
|
|
40
41
|
|
|
41
|
-
__version__ = "1.
|
|
42
|
+
__version__ = "1.16.0"
|
|
42
43
|
|
|
43
44
|
__all__ = [
|
|
44
45
|
"BabelQueue",
|
|
@@ -53,6 +54,7 @@ __all__ = [
|
|
|
53
54
|
"ReceivedMessage",
|
|
54
55
|
"HeaderPublisher",
|
|
55
56
|
"Redeliverer",
|
|
57
|
+
"Rewinder",
|
|
56
58
|
"RoutedPublisher",
|
|
57
59
|
"BabelQueueError",
|
|
58
60
|
"UnknownUrnError",
|
|
@@ -43,6 +43,7 @@ from .transport import (
|
|
|
43
43
|
HeaderPublisher,
|
|
44
44
|
ReceivedMessage,
|
|
45
45
|
Redeliverer,
|
|
46
|
+
Rewinder,
|
|
46
47
|
Transport,
|
|
47
48
|
make_transport,
|
|
48
49
|
publish_keeping_route,
|
|
@@ -111,6 +112,7 @@ class BabelQueue:
|
|
|
111
112
|
self.shutdown_timeout = float(shutdown_timeout)
|
|
112
113
|
self._handlers: Dict[str, Handler] = {}
|
|
113
114
|
self._stop = threading.Event()
|
|
115
|
+
self._rewind_pause = 1.0 # seconds to wait before a rewound gated message is re-polled
|
|
114
116
|
self._phase = _IDLE
|
|
115
117
|
self._last_phase = _IDLE
|
|
116
118
|
self._signalled = False
|
|
@@ -295,6 +297,20 @@ class BabelQueue:
|
|
|
295
297
|
self._phase = _SETTLING
|
|
296
298
|
except Exception as exc: # noqa: BLE001 - one bad message must not kill the loop
|
|
297
299
|
self._phase = _SETTLING
|
|
300
|
+
if received.route_urn.startswith(_UNSUPPORTED_SCHEMA_ROUTE_PREFIX):
|
|
301
|
+
# A version-gated delivery never takes the generic retry path, even when its
|
|
302
|
+
# dead-letter write fails: a retry re-publish may rebuild the version header
|
|
303
|
+
# from the (valid) body and walk the message past the gate into a handler.
|
|
304
|
+
# It is left unacked; the broker (or a rewind, for cumulative-offset brokers
|
|
305
|
+
# such as Kafka) brings it back and the gate rejects it again.
|
|
306
|
+
logger.error(
|
|
307
|
+
"Dead-lettering a version-gated message on queue %r failed; it was NOT "
|
|
308
|
+
"retried or acknowledged; it will come back and the dead-letter write will be retried.",
|
|
309
|
+
received.queue,
|
|
310
|
+
exc_info=exc,
|
|
311
|
+
)
|
|
312
|
+
self._rewind_unsettled(received)
|
|
313
|
+
return
|
|
298
314
|
self._retry_or_dead_letter(received, envelope, exc)
|
|
299
315
|
else:
|
|
300
316
|
# Outside the handler's try: a failed ack is not a handler failure, so it must
|
|
@@ -417,6 +433,23 @@ class BabelQueue:
|
|
|
417
433
|
exc_info=True,
|
|
418
434
|
)
|
|
419
435
|
|
|
436
|
+
def _rewind_unsettled(self, received: ReceivedMessage) -> None:
|
|
437
|
+
"""Make a deliberately unsettled message come back (cumulative-offset brokers such as Kafka
|
|
438
|
+
would otherwise lose it when a later record on the partition is committed), after a short
|
|
439
|
+
pause so an unavailable DLQ is not hammered in a hot loop."""
|
|
440
|
+
if not isinstance(self.transport, Rewinder):
|
|
441
|
+
return
|
|
442
|
+
try:
|
|
443
|
+
self.transport.rewind(received)
|
|
444
|
+
except Exception: # noqa: BLE001 - reported, never kills the loop
|
|
445
|
+
logger.error(
|
|
446
|
+
"Failed to rewind an unsettled version-gated message on queue %r.",
|
|
447
|
+
received.queue,
|
|
448
|
+
exc_info=True,
|
|
449
|
+
)
|
|
450
|
+
return
|
|
451
|
+
self._stop.wait(self._rewind_pause)
|
|
452
|
+
|
|
420
453
|
def _release_unprocessed(self, received: ReceivedMessage) -> None:
|
|
421
454
|
"""Release a message whose handler never completed (shutdown): unchanged body, no
|
|
422
455
|
backoff, and the runtime adds no ``attempts`` increment. A broker that counts
|
|
@@ -440,23 +473,35 @@ class BabelQueue:
|
|
|
440
473
|
if strategy == UnknownUrnStrategy.DELETE:
|
|
441
474
|
self._ack_settled(received)
|
|
442
475
|
return
|
|
443
|
-
# A transport's version-gated delivery
|
|
444
|
-
#
|
|
445
|
-
#
|
|
446
|
-
#
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
476
|
+
# A transport's version-gated delivery (unsupported ``bq-schema-version``) can never gain
|
|
477
|
+
# a handler and must not be retried: a retry re-publishes through paths that may rebuild
|
|
478
|
+
# the version header from the (valid) body, or never advance the attempt counter, so the
|
|
479
|
+
# gate would be bypassed or the message would loop. It goes straight to the DLQ (or is
|
|
480
|
+
# dropped when no DLQ is configured).
|
|
481
|
+
if received.route_urn.startswith(_UNSUPPORTED_SCHEMA_ROUTE_PREFIX):
|
|
482
|
+
self._settle_version_gated(received, envelope)
|
|
483
|
+
return
|
|
484
|
+
# An undecodable body or a body naming no URN can never gain a handler either, so
|
|
485
|
+
# releasing it unchanged would redeliver it forever. It takes the bounded FAIL path
|
|
486
|
+
# instead (retry up to max_attempts, then dead-letter or drop); only a genuinely unknown
|
|
487
|
+
# URN keeps the verbatim release.
|
|
488
|
+
if strategy == UnknownUrnStrategy.RELEASE and urn:
|
|
450
489
|
self._release(received, received.body, self.unknown_urn_release_delay)
|
|
451
490
|
return
|
|
452
491
|
if strategy == UnknownUrnStrategy.DEAD_LETTER:
|
|
453
492
|
self._dead_letter(received, dict(envelope), "unknown_urn", None)
|
|
454
493
|
return
|
|
455
494
|
# FAIL (and RELEASE of a poison message) — surfaced through the retry/dead-letter path
|
|
456
|
-
# (never kills the loop).
|
|
457
|
-
raise UnknownUrnError(
|
|
458
|
-
|
|
459
|
-
|
|
495
|
+
# (never kills the loop).
|
|
496
|
+
raise UnknownUrnError(f"No handler mapped for URN [{urn or '(empty)'}].")
|
|
497
|
+
|
|
498
|
+
def _settle_version_gated(self, received: ReceivedMessage, envelope: Mapping[str, Any]) -> None:
|
|
499
|
+
"""Dead-letter a version-gated delivery directly (no retry); drop it when no DLQ is set."""
|
|
500
|
+
if self.dead_letter_enabled or self.on_unknown_urn == UnknownUrnStrategy.DEAD_LETTER:
|
|
501
|
+
error = UnknownUrnError(f"No handler mapped for URN [{received.route_urn}].")
|
|
502
|
+
self._dead_letter(received, dict(envelope), "unknown_urn", error)
|
|
503
|
+
return
|
|
504
|
+
self._ack_settled(received)
|
|
460
505
|
|
|
461
506
|
def _retry_or_dead_letter(
|
|
462
507
|
self, received: ReceivedMessage, envelope: Dict[str, Any], exc: BaseException
|
|
@@ -500,7 +545,8 @@ class BabelQueue:
|
|
|
500
545
|
#: Reserved ``ReceivedMessage.route_urn`` prefix a transport uses to hand over a delivery it
|
|
501
546
|
#: version-gated (broker-bindings §5.7 step 3, e.g. the pulsar transport's ``bq-schema-version``
|
|
502
547
|
#: check). Not a ``urn:babel:`` URN, so no handler owns it. Must equal
|
|
503
|
-
#: ``
|
|
548
|
+
#: ``UNSUPPORTED_SCHEMA_ROUTE_PREFIX`` of the pulsar, sqs, asb, kafka and artemis transports (locked
|
|
549
|
+
#: by a test); never reaches the wire. A delivery under it is dead-lettered directly, never retried.
|
|
504
550
|
_UNSUPPORTED_SCHEMA_ROUTE_PREFIX = "babelqueue:unsupported-schema-version:"
|
|
505
551
|
|
|
506
552
|
|
|
@@ -32,10 +32,24 @@ transport directly and pass it via ``BabelQueue(transport=...)`` or
|
|
|
32
32
|
|
|
33
33
|
from __future__ import annotations
|
|
34
34
|
|
|
35
|
+
import logging
|
|
36
|
+
import re
|
|
35
37
|
from typing import Any, Dict, Optional
|
|
38
|
+
from urllib.parse import quote
|
|
36
39
|
|
|
37
|
-
from .codec import EnvelopeCodec, parse_envelope
|
|
38
|
-
from .transport import ReceivedMessage, Transport
|
|
40
|
+
from .codec import SCHEMA_VERSION, EnvelopeCodec, parse_envelope
|
|
41
|
+
from .transport import ReceivedMessage, Transport, truncate_declared_version
|
|
42
|
+
|
|
43
|
+
_logger = logging.getLogger("babelqueue.artemis")
|
|
44
|
+
|
|
45
|
+
#: A blank ``bq_schema_version``: empty or only ASCII whitespace (space, \t, \n, \v, \f, \r) — the
|
|
46
|
+
#: same set in every SDK (GR-5). Deliberately not ``str.strip()``, which also strips U+001C-U+001F,
|
|
47
|
+
#: U+0085, U+00A0, U+2028, U+3000 ...
|
|
48
|
+
_BLANK_ASCII_PATTERN = re.compile(r"[ \t\n\x0b\x0c\r]*")
|
|
49
|
+
#: Prefix of the reserved ``route_urn`` a version-gated message is delivered under. It is not a
|
|
50
|
+
#: ``urn:babel:`` URN, so no handler can legitimately own it; the runtime reports it as an
|
|
51
|
+
#: unknown URN, which names the rejected ``bq_schema_version``.
|
|
52
|
+
UNSUPPORTED_SCHEMA_ROUTE_PREFIX = "babelqueue:unsupported-schema-version:"
|
|
39
53
|
|
|
40
54
|
JMS_TYPE_ANNOTATION = "x-opt-jms-type"
|
|
41
55
|
APP_ID = "babelqueue"
|
|
@@ -187,6 +201,33 @@ class ArtemisTransport(Transport):
|
|
|
187
201
|
except (TypeError, ValueError): # pragma: no cover - defensive
|
|
188
202
|
return 0
|
|
189
203
|
|
|
204
|
+
@staticmethod
|
|
205
|
+
def _unsupported_schema_version(properties: Any) -> Optional[str]:
|
|
206
|
+
"""The §7.7 step-3 gate on the ``bq_schema_version`` application property (underscores —
|
|
207
|
+
ADR-0017): the offending raw value, or ``None``.
|
|
208
|
+
|
|
209
|
+
A missing property, a null value or a blank value (the empty string or only ASCII
|
|
210
|
+
whitespace — the same set in every SDK, GR-5, not ``str.strip()``) is not gated: the usual
|
|
211
|
+
decode path runs. A ``bytes`` value is decoded as UTF-8. A present value is accepted only
|
|
212
|
+
when it is exactly the core's supported ``SCHEMA_VERSION``: the string ``"1"`` or the
|
|
213
|
+
integer ``1`` (a ``bool`` is not an integer here, though it is one in Python); ``"2"``,
|
|
214
|
+
``"x"``, ``"01"``, ``" 1"``, ``0``, ``2``, ``1.5``, ``True`` and any other type are
|
|
215
|
+
rejected."""
|
|
216
|
+
if not isinstance(properties, dict):
|
|
217
|
+
return None
|
|
218
|
+
raw = properties.get("bq_schema_version")
|
|
219
|
+
if raw is None:
|
|
220
|
+
return None
|
|
221
|
+
if isinstance(raw, (bytes, bytearray)):
|
|
222
|
+
raw = bytes(raw).decode("utf-8", "replace")
|
|
223
|
+
if isinstance(raw, str):
|
|
224
|
+
if _BLANK_ASCII_PATTERN.fullmatch(raw) is not None or raw == str(SCHEMA_VERSION):
|
|
225
|
+
return None
|
|
226
|
+
return raw
|
|
227
|
+
if isinstance(raw, int) and not isinstance(raw, bool) and raw == SCHEMA_VERSION:
|
|
228
|
+
return None
|
|
229
|
+
return str(raw)
|
|
230
|
+
|
|
190
231
|
@staticmethod
|
|
191
232
|
def _payload(message: Any) -> str:
|
|
192
233
|
body = getattr(message, "body", None)
|
|
@@ -212,6 +253,26 @@ class ArtemisTransport(Transport):
|
|
|
212
253
|
raise
|
|
213
254
|
if message is None: # pragma: no cover - defensive (real client raises instead)
|
|
214
255
|
return None
|
|
256
|
+
# §7.7 step 3: gate on the ``bq_schema_version`` property BEFORE any body decode (the
|
|
257
|
+
# attempts reconcile below decodes the body). A rejected message keeps its raw body and the
|
|
258
|
+
# reserved route; the runtime then dead-letters it directly (no retry).
|
|
259
|
+
unsupported = truncate_declared_version(
|
|
260
|
+
self._unsupported_schema_version(getattr(message, "properties", None))
|
|
261
|
+
)
|
|
262
|
+
if unsupported is not None:
|
|
263
|
+
_logger.warning(
|
|
264
|
+
"Artemis message on %r carries unsupported bq_schema_version %r (supported: %s); "
|
|
265
|
+
"not decoding it.",
|
|
266
|
+
queue,
|
|
267
|
+
unsupported,
|
|
268
|
+
SCHEMA_VERSION,
|
|
269
|
+
)
|
|
270
|
+
return ReceivedMessage(
|
|
271
|
+
body=self._payload(message),
|
|
272
|
+
queue=queue,
|
|
273
|
+
handle=receiver,
|
|
274
|
+
route_urn=UNSUPPORTED_SCHEMA_ROUTE_PREFIX + quote(unsupported, safe=""),
|
|
275
|
+
)
|
|
215
276
|
body = self._reconcile(self._payload(message), self._delivery_count(message))
|
|
216
277
|
return ReceivedMessage(body=body, queue=queue, handle=receiver)
|
|
217
278
|
|
|
@@ -23,11 +23,24 @@ transport directly and pass it via ``BabelQueue(transport=...)`` or
|
|
|
23
23
|
|
|
24
24
|
from __future__ import annotations
|
|
25
25
|
|
|
26
|
+
import logging
|
|
27
|
+
import re
|
|
26
28
|
from typing import Any, Dict, Optional
|
|
27
|
-
from urllib.parse import urlsplit
|
|
29
|
+
from urllib.parse import quote, urlsplit
|
|
28
30
|
|
|
29
|
-
from .codec import EnvelopeCodec, parse_envelope
|
|
30
|
-
from .transport import ReceivedMessage, Transport
|
|
31
|
+
from .codec import SCHEMA_VERSION, EnvelopeCodec, parse_envelope
|
|
32
|
+
from .transport import ReceivedMessage, Transport, truncate_declared_version
|
|
33
|
+
|
|
34
|
+
_logger = logging.getLogger("babelqueue.asb")
|
|
35
|
+
|
|
36
|
+
#: A blank ``bq-schema-version``: empty or only ASCII whitespace (space, \t, \n, \v, \f, \r) — the
|
|
37
|
+
#: same set in every SDK (GR-5). Deliberately not ``str.strip()``, which also strips U+001C-U+001F,
|
|
38
|
+
#: U+0085, U+00A0, U+2028, U+3000 ...
|
|
39
|
+
_BLANK_ASCII_PATTERN = re.compile(r"[ \t\n\x0b\x0c\r]*")
|
|
40
|
+
#: Prefix of the reserved ``route_urn`` a version-gated message is delivered under. It is not a
|
|
41
|
+
#: ``urn:babel:`` URN, so no handler can legitimately own it; the runtime reports it as an
|
|
42
|
+
#: unknown URN, which names the rejected ``bq-schema-version``.
|
|
43
|
+
UNSUPPORTED_SCHEMA_ROUTE_PREFIX = "babelqueue:unsupported-schema-version:"
|
|
31
44
|
|
|
32
45
|
|
|
33
46
|
class AsbTransport(Transport):
|
|
@@ -141,6 +154,42 @@ class AsbTransport(Transport):
|
|
|
141
154
|
env["attempts"] = native
|
|
142
155
|
return EnvelopeCodec.encode(env)
|
|
143
156
|
|
|
157
|
+
@staticmethod
|
|
158
|
+
def _unsupported_schema_version(properties: Any) -> Optional[str]:
|
|
159
|
+
"""The §4.7 step-3 gate on the ``bq-schema-version`` application property: the offending
|
|
160
|
+
raw value as text, or ``None``.
|
|
161
|
+
|
|
162
|
+
The AMQP client may hand keys and values over as ``bytes``; both are decoded as UTF-8. A
|
|
163
|
+
missing property, or a blank string (the empty string or only ASCII whitespace — the same
|
|
164
|
+
set in every SDK, GR-5, not ``str.strip()``) is not gated: the usual decode path runs. A
|
|
165
|
+
present value is accepted only when it is exactly the core's supported ``SCHEMA_VERSION``:
|
|
166
|
+
the string ``"1"`` or the integer ``1`` (a ``bool`` is not an integer here, though it is
|
|
167
|
+
one in Python). ``"2"``, ``"x"``, ``"01"``, ``" 1"``, ``0``, ``2``, ``1.5``, ``True`` and
|
|
168
|
+
any other type are rejected."""
|
|
169
|
+
if not properties:
|
|
170
|
+
return None
|
|
171
|
+
raw: Any = None
|
|
172
|
+
try:
|
|
173
|
+
items = list(properties.items())
|
|
174
|
+
except (AttributeError, TypeError):
|
|
175
|
+
return None
|
|
176
|
+
for key, value in items:
|
|
177
|
+
name = key.decode("utf-8", "replace") if isinstance(key, (bytes, bytearray)) else key
|
|
178
|
+
if name == "bq-schema-version":
|
|
179
|
+
raw = value
|
|
180
|
+
break
|
|
181
|
+
if raw is None:
|
|
182
|
+
return None
|
|
183
|
+
if isinstance(raw, (bytes, bytearray)):
|
|
184
|
+
raw = bytes(raw).decode("utf-8", "replace")
|
|
185
|
+
if isinstance(raw, str):
|
|
186
|
+
if _BLANK_ASCII_PATTERN.fullmatch(raw) is not None or raw == str(SCHEMA_VERSION):
|
|
187
|
+
return None
|
|
188
|
+
return raw
|
|
189
|
+
if isinstance(raw, int) and not isinstance(raw, bool) and raw == SCHEMA_VERSION:
|
|
190
|
+
return None
|
|
191
|
+
return str(raw)
|
|
192
|
+
|
|
144
193
|
# -- Transport ----------------------------------------------------------
|
|
145
194
|
|
|
146
195
|
def publish(self, queue: str, body: str) -> None:
|
|
@@ -157,6 +206,27 @@ class AsbTransport(Transport):
|
|
|
157
206
|
if not messages:
|
|
158
207
|
return None
|
|
159
208
|
message = messages[0]
|
|
209
|
+
# §4.7 step 3: gate on ``bq-schema-version`` BEFORE any body decode (the attempts reconcile
|
|
210
|
+
# below decodes the body). A rejected message keeps its raw body and the reserved route.
|
|
211
|
+
unsupported = truncate_declared_version(
|
|
212
|
+
self._unsupported_schema_version(
|
|
213
|
+
getattr(message, "application_properties", None)
|
|
214
|
+
)
|
|
215
|
+
)
|
|
216
|
+
if unsupported is not None:
|
|
217
|
+
_logger.warning(
|
|
218
|
+
"Azure Service Bus message on %r carries unsupported bq-schema-version %r "
|
|
219
|
+
"(supported: %s); not decoding it.",
|
|
220
|
+
queue,
|
|
221
|
+
unsupported,
|
|
222
|
+
SCHEMA_VERSION,
|
|
223
|
+
)
|
|
224
|
+
return ReceivedMessage(
|
|
225
|
+
body=str(message),
|
|
226
|
+
queue=queue,
|
|
227
|
+
handle=message,
|
|
228
|
+
route_urn=UNSUPPORTED_SCHEMA_ROUTE_PREFIX + quote(unsupported, safe=""),
|
|
229
|
+
)
|
|
160
230
|
body = self._reconcile(str(message), getattr(message, "delivery_count", None))
|
|
161
231
|
return ReceivedMessage(body=body, queue=queue, handle=message)
|
|
162
232
|
|