babelqueue 1.21.1__tar.gz → 1.23.0__tar.gz

{babelqueue-1.21.1 → babelqueue-1.23.0}/.github/workflows/ci.yml

@@ -12,7 +12,8 @@ permissions:
 jobs:
   test:
     name: Python ${{ matrix.python }}
-    runs-on: ubuntu-latest
+    # setup-python ships no 3.9 build for ubuntu-26.04 (ubuntu-latest); keep 3.9 on 24.04 while it stays supported.
+    runs-on: ${{ matrix.python == '3.9' && 'ubuntu-24.04' || 'ubuntu-latest' }}
     strategy:
       fail-fast: false
       matrix:
@@ -104,6 +105,8 @@ jobs:
 
       - name: Run full suite with coverage gate (>=90%)
         env:
+          # A real redis:7 is mandatory here: no silent skip or fakeredis fallback (shared runner contract).
+          BQ_REQUIRE_REDIS: "1"
           BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
           BABELQUEUE_TEST_AMQP: amqp://guest:guest@localhost:5672/
           SQS_ENDPOINT: http://localhost:9324
@@ -162,9 +165,19 @@ jobs:
   conformance:
     name: Conformance suite in sync + results.json
     runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
     env:
       # The pinned conformance release (ADR-0032). Bump together with tests/conformance/.
-      BQ_CONFORMANCE_VERSION: 1.5.0
+      BQ_CONFORMANCE_VERSION: 1.7.0
       # sha256 of that release's fetch.sh: the script runs on the runner, so it is pinned rather
       # than trusted from a mutable tag. Bump together with BQ_CONFORMANCE_VERSION.
       BQ_FETCH_SH_SHA256: 8bb6ba567b288fb0ac71a70fd17041b2503df119e01998d76fdad372570bf849
@@ -190,11 +203,13 @@ jobs:
       - name: Install
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[dev,celery,django,otel]" opentelemetry-sdk jsonschema
+          pip install -e ".[dev,celery,django,otel,redis]" opentelemetry-sdk jsonschema
 
       - name: Run the conformance suite and write results.json
         env:
           BQ_CONFORMANCE_RESULTS: ${{ runner.temp }}/results.json
+          BQ_REQUIRE_REDIS: "1"
+          BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
         run: pytest -q tests/test_conformance.py
 
       - name: Validate results.json (schema, bindings, sections)

{babelqueue-1.21.1 → babelqueue-1.23.0}/.github/workflows/release.yml

@@ -14,9 +14,21 @@ jobs:
     # Recommended for PyPI Trusted Publishing — configure a "pypi" environment +
     # a trusted publisher on PyPI (repo BabelQueue/babelqueue-python, this workflow).
     environment: pypi
+    # redis_layout (conformance 1.7.0) drives the real Lua scripts: a real redis:7 is mandatory,
+    # exactly as in ci.yml's `conformance` job — without it the cases are reported as failures.
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
     env:
       # The pinned conformance release (ADR-0032). Keep identical to ci.yml.
-      BQ_CONFORMANCE_VERSION: 1.5.0
+      BQ_CONFORMANCE_VERSION: 1.7.0
       BQ_FETCH_SH_SHA256: 8bb6ba567b288fb0ac71a70fd17041b2503df119e01998d76fdad372570bf849
     permissions:
       id-token: write   # PyPI Trusted Publishing (OIDC) — no API token needed
@@ -40,9 +52,12 @@ jobs:
       - name: Install & test (writes results.json)
         env:
           BQ_CONFORMANCE_RESULTS: ${{ runner.temp }}/results.json
+          # A real redis:7 is mandatory: no silent skip or fakeredis fallback (shared runner contract).
+          BQ_REQUIRE_REDIS: "1"
+          BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
         run: |
           python -m pip install --upgrade pip build
-          pip install -e ".[dev,celery,django,otel]" opentelemetry-sdk jsonschema
+          pip install -e ".[dev,celery,django,otel,redis]" opentelemetry-sdk jsonschema
           pytest
           test -s "$BQ_CONFORMANCE_RESULTS"  # the conformance run must have produced results.json
 

{babelqueue-1.21.1 → babelqueue-1.23.0}/CHANGELOG.md

@@ -9,6 +9,174 @@ The envelope wire format is versioned separately by `meta.schema_version`
 
 ## [Unreleased]
 
+## [1.23.0] - 2026-10-25
+
+Implements ADR-0036 (post-R1 parity) decisions 2, 4, 5 and 6. A MINOR: every change is a
+consumer-side settle decision, so the envelope stays at `schema_version: 1`. The vendored
+conformance suite moves to v1.7.0 (new sections `identity_verdict` and `pulsar_exhaustion`, both
+passing; `conformance-known-gaps.json` stays empty).
+
+### Changed (behaviour)
+- **A body that does not decode, or has no identity, is poison on every transport (ADR-0036
+  decisions 4 and 6).** A JSON object that has neither a valid string `job` nor a valid string
+  `urn` (ADR-0032 section 6) used to be an *unknown URN*, and on SQS, AMQP, Pulsar, Azure Service
+  Bus, Artemis and the memory transport a body that did not decode was decoded leniently into an
+  empty envelope and retried as an unknown URN, finally dead-lettered as `reason: "unknown_urn"`
+  without the original body. Both are now **poison** and take the poison route of Go's
+  `routePoison`: they are never handed to a handler and never take the unknown-URN strategies
+  (those remain for a valid envelope whose URN has no handler). With the DLQ on (`dead_letter=True`
+  or `on_unknown_urn=DEAD_LETTER`) the message is written to the DLQ at once, with no retry, as the
+  ADR-0035 record (`dead_letter.reason: "poison"`, the original body preserved: a strict JSON
+  object spliced byte for byte, anything else as `raw`, base64 plus `raw_encoding` when it is not
+  valid UTF-8) and acknowledged after the write; `on_unknown_urn=DELETE` drops it (logged); with
+  the DLQ off `RELEASE` is replaced by the bounded `FAIL` path (re-published with `attempts + 1`
+  up to `max_attempts`, then dropped and logged). An operator who relied on a poison body being
+  retried through the unknown-URN strategy, or on a `reason: "unknown_urn"` record for it, sees the
+  new behaviour. The classification is `EnvelopeCodec.poison_cause(raw)` (new, pure): the typed
+  decode of the Go reference plus the identity rule; `decode` / `decode_lenient` are unchanged.
+- **Pulsar poison records carry `reason: "poison"`** (they were `unknown_urn`). The Pulsar
+  transport also reports the reconciled `bq-attempts` / redelivery count to the runtime
+  (`ReceivedMessage.native_attempts`, new), as does SQS (`ApproximateReceiveCount - 1`), so a poison
+  body, which cannot carry `attempts`, is still bounded by `max_attempts`.
+- **Redis: an identity-less body is dead-lettered unconditionally (ADR-0036 decision 4,
+  broker-bindings section 1.1 "Identity-less body").** In both layouts (legacy `:processing`
+  and Laravel-compatible zset) a body that decodes but names no identity (including a bare `null`,
+  which used to follow the bounded `FAIL` path) is handed over as transport poison: it is written
+  to `<queue>.dlq` as the ADR-0035 record with `reason: "poison"` in every configuration
+  (`on_unknown_urn=DELETE` and dead-lettering off change nothing), never handed to a handler and
+  never retried. Nothing else in the Redis transport changes.
+- **Kafka without retry tiers: a poison record no longer blocks the partition (ADR-0036 decision
+  5, broker-bindings section 6.5 degrade-to-delete).** With no `retry_tiers_ms`, dead-lettering off
+  and `on_unknown_urn` not `DELETE`, the bounded retry of a poison record (including an
+  identity-less body) needs a tier and raised, leaving the record uncommitted with `bq-attempts`
+  never advancing. It is now committed, dropped and logged, and the partition advances. A failing
+  handler and `RELEASE` of a valid envelope keep the section 6.5 rule (they still raise
+  `BabelQueueError` without tiers). The start-up warning for a tierless transport now says so; enable
+  `dead_letter` to keep such records in `<topic>.dlq`.
+- A message dropped after its retries are exhausted with no DLQ configured is now logged
+  (`babelqueue.app`, WARNING) instead of being dropped silently.
+
+- **Pulsar: a failed DLQ write or retry publish is negatively acknowledged (Go `Rewind`).**
+  `PulsarTransport` is now a `Rewinder` (`negative_acknowledge`), so the unsettled original is
+  redelivered by the broker instead of staying un-acknowledged for the life of the consumer. A
+  graceful-shutdown release of an in-flight Pulsar message is now a nack as well rather than a
+  re-publish (on a `Shared` subscription the native redelivery count advances; on `Exclusive` /
+  `Failover` it does not, which is harmless: `bq-attempts` carries the attempt limit).
+
+### Fixed
+- **Bytes that are not valid UTF-8 no longer crash the AMQP, Pulsar and Artemis consumers.** They
+  were decoded strictly in `pop`; they are now kept losslessly (`surrogateescape`). Invalid UTF-8
+  is poison wherever it sits, **also inside a JSON string** (`EnvelopeCodec.poison_cause`,
+  error-handling section 7): it is never handed to a handler and is dead-lettered as the ADR-0035
+  base64 record, like on Redis and Kafka.
+
+### Added
+- `EnvelopeCodec.poison_cause(raw, *, require_identity=True)`; `Transport.classifies_poison`
+  (Kafka and Redis classify poison themselves and opt out of the runtime's classification);
+  `ReceivedMessage.native_attempts`.
+- Conformance runners for `identity_verdict` (the core verdict on all 97 bodies, plus the fate on
+  the memory transport) and `pulsar_exhaustion` (all 29 rows, driven through the real
+  `PulsarTransport` and runtime over a duck-typed client), and per-transport poison tests.
+- `ci.yml` / `release.yml` pin conformance `1.7.0` (the `fetch.sh` SHA-256 is unchanged).
+
+## [1.22.0] - 2026-10-24
+
+Implements ADR-0033 (Redis) and ADR-0034 (Kafka tiers, backoff, poison). The envelope stays at
+`schema_version: 1`; the vendored conformance suite moves to v1.6.0 and
+`conformance-known-gaps.json` is now empty.
+
+### Changed
+- The Redis poison dead-letter element is now the ADR-0035 record (raw splice, or the `raw`/`raw_encoding` wrapper) carrying a `dead_letter` block, instead of the bare element (broker-bindings section 1). A legacy `__bq_frame` header is unwrapped, not re-applied. Dropped the dead `_reject_json_constant` helper in the Kafka transport.
+- **Redis: a stored value that is not valid UTF-8 no longer crashes the reserve.** The client built from a URL now uses `encoding_errors="surrogateescape"`, replies are decoded losslessly, and the settle / release / dead-letter calls send the exact stored bytes back, so such a poison body becomes the ADR-0035 base64 wrapper (no U+FFFD) and its reservation is actually removed. A client you pass in with `decode_responses=True` should set `encoding_errors="surrogateescape"` as well (redis-py's default strict decoding raises on such a reply).
+- **Poison DLQ record shape follows ADR-0035** (Kafka): a strict JSON object body is spliced byte for byte with a `dead_letter` member, anything else is wrapped as `{"raw": ..., "raw_encoding"?: "base64", "dead_letter": ...}`; a body that is not valid UTF-8 is stored losslessly as base64 instead of with U+FFFD, and the conformance `poison_dlq` section runs.
+
+### Fixed
+- **Invalid UTF-8 is poison in both Redis layouts (broker-bindings section 1.1).** A stored value (or zset `:reserved` member) that is not valid UTF-8 is dead-lettered on first delivery as the ADR-0035 `raw`/`raw_encoding: "base64"` wrapper, even when the codec or `cjson` could decode it: it is never handed to a handler and never retried, released or re-published, and the exact stored bytes are used for the settle script, so no byte is rewritten.
+- **Redis, Laravel-compatible layout: a poison reservation is dead-lettered in every
+  configuration.** The DLQ write no longer depends on `dead_letter_enabled` (default `False`) or
+  `on_unknown_urn`: an undecodable body or one without a numeric `attempts` is written raw to
+  `<queue>.dlq` and acked instead of being logged and dropped (broker-bindings section 1.1,
+  ADR-0033; same as the Go reference). Other version-gated routes keep their previous behaviour.
+- **Redis, Laravel-compatible layout: a job with a non-numeric `attempts` or an undecodable body no
+  longer aborts the reserve script.** It is reserved raw (`pcall` guard), dead-lettered byte for
+  byte and acked, leaving nothing in `:reserved` (ADR-0033 (c)).
+- **Redis, legacy layout: a `kill -9` no longer strands a message in `<queue>:processing`.** Every
+  reservation writes a `<token>:<raw>` member into `<queue>:processing:deadlines`; a sweep before
+  each reserve moves lapsed reservations back to the end of `<queue>`. Ack, release and dead-letter
+  are gated on their own reservation, so a late ack never touches a newer one. The legacy mode
+  remains the default and the `<queue>:processing` key name is unchanged.
+- **Kafka without retry tiers: a failing handler no longer stops the worker** (Go
+  `leaveRepublishUnsettled` parity). The transport still raises `BabelQueueError` (section 6.5), but
+  the runtime now catches a failed retry release, logs it (`babelqueue.app`, ERROR, "NOT
+  acknowledged"), leaves the record unsettled (no commit; a rewind-capable transport seeks back and
+  the loop pauses `_rewind_pause` first) and keeps consuming. Previously the error escaped
+  `consume()`. Applies to any transport whose retry release fails.
+- **Kafka poison follows Go `routePoison` when no DLQ is configured.** With the DLQ on
+  (`dead_letter=True` or `on_unknown_urn=DEAD_LETTER`) the wrapped record is written to
+  `<topic>.dlq` (`dead_letter.reason: "poison"`) and then committed. With the DLQ off the poison
+  record is **not** written anywhere: `on_unknown_urn=DELETE` drops it, every other strategy takes
+  the bounded FAIL path (retried through the retry tiers up to `max_attempts` with a bare
+  `{"attempts": N}` body, then dropped; without tiers the record is redelivered in place).
+  Operational note: when the DLQ is on, `<topic>.dlq` must exist; a failed write leaves the record
+  uncommitted and rewound (it is never dropped).
+- **Unknown-URN `RELEASE` and the retry release leave the delivery unsettled on failure** (Go
+  `leaveRepublishUnsettled`): a failed re-publish of an unknown-URN message is no longer counted as
+  a handler failure and can no longer lead to the record being acked and lost.
+- **Redis, legacy `:processing` layout: an undecodable body is dead-lettered on its first
+  delivery.** A body Go's typed `Decode` rejects (neither a JSON object nor `null`, or wrong member types such as
+  a string `attempts`/`meta`; same criterion as Kafka poison) is written verbatim to `<queue>.dlq`
+  through the gated settle script and removed from `:processing`, in every configuration, instead of
+  being retried until `max_attempts`. A valid JSON object without an identity keeps the existing
+  bounded path.
+- **Kafka: a record without a valid `job` / `urn` is poison** (ADR-0034 section 10), not an
+  unknown URN: with the DLQ on it is dead-lettered wrapped with `dead_letter.reason: "poison"` and
+  never retried; with the DLQ off it follows Go `routePoison` (see above).
+
+- **Redis, Laravel-compatible layout: the reserve script no longer aborts on a body that decodes
+  but cannot be re-encoded** (`cjson.encode` of `inf`/`NaN`, e.g. `1e400`, raised after the `LPOP`
+  and lost the job). The encode is now inside `pcall`; the job is reserved raw and takes the poison
+  path. Note: fakeredis' `cjson` serialises `inf` as `Infinity` and masks this; it is proven by a
+  real-Redis test (CI `redis:7`) and by a Lua test against a failing `cjson` stub.
+- **`on_unknown_urn="delete"` no longer discards a Redis poison reservation.** It is dead-lettered
+  byte for byte like the Kafka poison record already was; a genuine unknown URN is still deleted.
+- **Redis, Laravel-compatible layout: `dead_letter` is atomic and gated.** One script `ZREM`s the
+  reservation and writes the dead-letter record and its notify entry; an expired reservation writes
+  nothing (and logs a warning) instead of publishing a duplicate.
+- **A message dropped because it is undeliverable and no dead-letter queue is configured is
+  logged** (`babelqueue.app`, WARNING) instead of disappearing silently.
+- **Kafka `RetryReinjector` no longer loses records:** it kept one held record per partition, so a
+  second record on the same partition overwrote the first. It now keeps all of them in order.
+- **A worker on a tier-less Kafka transport with `max_attempts` > 1 warns at start-up** that a
+  failing handler cannot be retried: the record is left uncommitted and redelivered.
+- Internal: the legacy `:processing` settle script takes `[token, raw]` and builds the deadline
+  member itself.
+
+### Added
+- `RedisTransport(retry_after=90, migration_batch_size=-1)` (also `?retry_after=` in the URL;
+  `None` / `null` disables recovery). The Laravel-compatible `retry_after` default is now 90 s
+  (was 60 s).
+- SQS: `SqsTransport.publish(..., delay_seconds=)` (`DelaySeconds` 0..900; out of range or FIFO
+  raises `BabelQueueError`).
+- Kafka: `KafkaTransport(retry_tiers_ms=...)` with `DEFAULT_RETRY_TIERS_MS`, `retry_topic()` and
+  the `babelqueue.kafka_retry.RetryReinjector` consumer that moves parked records from
+  `<topic>.retry.<n>` back to the work topic (broker-bindings section 6.4). Opt-in: without tiers a
+  retry / `publish_delayed` raises `BabelQueueError` (the transport raises; the worker logs it and
+  leaves the record uncommitted, see Fixed).
+- Worker-side retry backoff: `BabelQueue(retry_backoff=, retry_backoff_multiplier=,
+  retry_backoff_max=, retry_jitter=)` (ADR-0034 section 9); the defaults keep an immediate release.
+- Conformance runner covers the v1.6.0 sections (`redis_layout`, `redis_delay`, `retry_backoff`,
+  `kafka_poison`, ...). CI runs the Redis cases against a real `redis:7` with `BQ_REQUIRE_REDIS=1`.
+
+### Changed
+- **Kafka retries go to the retry tiers.** With `retry_tiers_ms` set a failed message is parked on
+  `<queue>.retry.<n>` and needs a running `RetryReinjector`; the retry topics must exist. Without
+  tiers a retry now raises from the transport instead of republishing onto the work topic; the
+  runtime logs it and redelivers the record rather than stopping the worker.
+
+### Documentation
+- README: Redis layouts, the limit that the Laravel-compatible layout carries no out-of-band headers (ADR-0033 §5),
+  Kafka retry tiers, `LogAppendTime` on the tier topics, retry backoff, SQS `delay_seconds`.
+
 ## [1.21.1] - 2026-10-20
 
 ### Fixed

{babelqueue-1.21.1 → babelqueue-1.23.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: babelqueue
-Version: 1.21.1
+Version: 1.23.0
 Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
 Project-URL: Homepage, https://babelqueue.com
 Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
@@ -195,7 +195,7 @@ app.run()                               # consume forever (Ctrl-C to stop)
   messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
 - **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
   Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
-  see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
+  see below), `kafka://` (opt-in retry tiers, see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
   and `memory://` (in-process, great for tests/local). Bring your own by passing
   `transport=...`.
 
@@ -248,7 +248,10 @@ transport **always** releases with `ChangeMessageVisibility` (broker-bindings
 attempt counter, so a shutdown release also consumes an attempt. The default
 delay is **0 s** (`retry_backoff` / `unknown_urn_release_delay`), i.e. the
 message is visible again immediately; delays outside 0..43200 s are clamped with
-a warning.
+a warning. A **per-message delay** at publish time is
+`SqsTransport.publish(queue, body, delay_seconds=n)` (`DelaySeconds`, 0..900 on a
+standard queue; a larger value, or any per-message delay on a FIFO queue, raises
+`BabelQueueError` instead of being truncated or dropped).
 
 **Poison-loop risk:** a message that always fails is redelivered at once until
 `max_attempts` is reached. If the receive count is unavailable the SDK cannot
@@ -257,11 +260,98 @@ advance attempts at all. **Configure a native `RedrivePolicy`**
 queue as the broker-side backstop, and set a non-zero `retry_backoff` if the
 handler's dependencies need time to recover.
 
+### Retry back-off
+
+`BabelQueue(..., retry_backoff=, retry_backoff_multiplier=, retry_backoff_max=, retry_jitter=)`
+set the worker-side retry delay (ADR-0034 section 9): after the n-th failure the message is
+released with `d(n) = min(retry_backoff_max, retry_backoff * retry_backoff_multiplier ** (n - 1))`
+seconds (`retry_backoff_max=0` means no cap) and `retry_jitter="full"` draws `uniform(0, d(n))`.
+The defaults (`retry_backoff=0`, multiplier 1, jitter `"none"`) keep the previous behaviour: an
+immediate release. How the delay is honoured depends on the transport: SQS uses it as the
+visibility timeout, the Laravel-compatible Redis layout as the `:delayed` score and Kafka picks a
+retry tier (see below).
+
+### Kafka retry tiers and the re-inject consumer
+
+Kafka has no per-message delay, so delayed delivery and retry back-off are SDK-owned **retry
+topics** (broker-bindings section 6.4, ADR-0034). They are opt-in: pass `retry_tiers_ms` to the
+transport (the recommended set is `DEFAULT_RETRY_TIERS_MS` = 5 s, 30 s, 5 min, 30 min):
+
+```python
+from babelqueue import BabelQueue
+from babelqueue.kafka_transport import KafkaTransport, DEFAULT_RETRY_TIERS_MS
+from babelqueue.kafka_retry import RetryReinjector
+
+transport = KafkaTransport("kafka://localhost:9092", retry_tiers_ms=DEFAULT_RETRY_TIERS_MS)
+app = BabelQueue(transport=transport, queue="orders", max_attempts=5, dead_letter=True)
+
+# Run one next to the workers, otherwise a parked message is never delivered again:
+RetryReinjector("kafka://localhost:9092", work_topics=["orders"],
+                retry_tiers_ms=DEFAULT_RETRY_TIERS_MS).run()
+```
+
+A failed message is republished (attempts + 1) to `<queue>.retry.<n>`, where
+`n = min(attempts, N - 1) + 1`; `publish_delayed` picks the smallest tier that covers the requested
+delay. Create the `<queue>.retry.<n>` topics up front. The `RetryReinjector` copies each record's
+value and key byte for byte back to its work topic once the tier delay has elapsed (headers
+`bq-delay` / `bq-original-topic` are dropped) and commits only after the broker confirmed the
+write, so delivery stays at-least-once. It holds every record it has accepted per partition (oldest
+first, a record never overtakes the one before it) and a failed write rewinds the partition to the
+failed record. Without `retry_tiers_ms`, `publish_delayed` and a retry
+raise `BabelQueueError` and nothing is produced. The worker does not stop on that error: it logs
+it, leaves the record uncommitted (the consumer seeks back to it and the loop pauses briefly, so
+the record is redelivered and never lost) and keeps consuming. A worker started without tiers
+while `max_attempts` is above 1 logs a warning at start-up: a failing handler cannot be retried
+(the record keeps failing in place) until tiers are configured. The re-inject delay is measured from the record
+timestamp: set `message.timestamp.type=LogAppendTime` on the `.retry.<n>` topics so that
+timestamp is the broker's, not a producer clock's.
+
+**Records without an identity are poison.** A Kafka record whose JSON object has neither a valid
+string `job` nor a valid string `urn` (ADR-0034 section 10) is poison, not an unknown URN. With
+the DLQ on (`dead_letter` or `on_unknown_urn=DEAD_LETTER`) it is dead-lettered on its first delivery
+(wrapped with `dead_letter.reason: "poison"`, key preserved) and never retried: the record is
+written to `<topic>.dlq` (which must exist) and its offset is committed once that write
+is confirmed; a failed write leaves it uncommitted and rewound. With the DLQ off nothing is written:
+`on_unknown_urn=DELETE` drops it, any other strategy retries it through the retry tiers up to
+`max_attempts` and then drops it. **Without retry tiers** that bounded retry has no tier to use, so
+(with the DLQ off and `on_unknown_urn` not `DELETE`) the poison record is committed, dropped and logged
+instead of blocking the partition (section 6.5 degrade-to-delete, ADR-0036 decision 5); enable
+`dead_letter` to keep it in `<topic>.dlq`. A failing handler without tiers still cannot be retried, as above.
+
+### Poison messages on every other transport
+
+A message body that does not decode into an envelope (not a JSON object, malformed, a wrongly typed
+member, or bytes that are not valid UTF-8, even inside a JSON string) or that has neither a valid
+string `job` nor a valid string `urn` is **poison** on every transport (ADR-0036 decisions 2, 4 and
+6; conformance `identity_verdict`). It is never handed to a handler and never takes the
+unknown-URN strategies, which are for a valid envelope whose URN has no handler:
+
+- DLQ on (`dead_letter` or `on_unknown_urn=DEAD_LETTER`): written to `<queue>.dlq` at once, with no
+  retry, as the ADR-0035 record (`dead_letter.reason: "poison"`, the original body preserved,
+  base64 plus `raw_encoding` when it is not UTF-8), then acknowledged. A failed write leaves it
+  unacknowledged.
+- `on_unknown_urn=DELETE`: acknowledged and dropped (logged).
+- DLQ off: the bounded `FAIL` path (`RELEASE` is replaced by it): retried up to `max_attempts`, then
+  dropped and logged. The count comes from the broker where it has one (SQS receive count, Pulsar
+  `bq-attempts` / redelivery count), otherwise from a bare `{"attempts": n}` body the retry leaves.
+
+On Redis (both layouts) an identity-less body is dead-lettered as poison in **every** configuration,
+including `on_unknown_urn=DELETE` and dead-lettering off (broker-bindings section 1.1). On the legacy
+(`<queue>:processing`) layout a body that Go's typed decode rejects is dead-lettered raw on its first
+delivery and removed from `:processing`.
+
 ### Sharing a Redis queue with Laravel
 
 By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
-`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
-**shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
+`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). Each reservation also
+writes a `<token>:<raw>` member (32 lowercase hex characters of token) into the
+`<queue>:processing:deadlines` sorted set, and before every reserve a sweep moves the lapsed
+reservations back to the end of the list, so a `kill -9` no longer strands a message in
+`:processing`; a late ack of a swept reservation touches nothing of the newer one. `retry_after`
+(default **90** seconds; `None` / `?retry_after=null` disables recovery) sets that deadline, and
+`migration_batch_size` (`-1` = all, otherwise `>= 1`) bounds how many due members one sweep or
+migration moves. To consume a **shared** Laravel BabelQueue Redis queue instead, enable
+Laravel-compatible mode:
 
 ```python
 app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
@@ -278,8 +368,21 @@ member a Python worker writes is identical to a Laravel worker's — either side
 ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
 one queue without losing or double-processing messages. Before each pop, expired
 reserved/delayed jobs migrate back to the ready list, so a crashed worker's
-in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
-(or the `key_prefix` / `retry_after` constructor args).
+in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=90`
+(or the `key_prefix` / `retry_after` / `migration_batch_size` constructor args). A job whose body
+does not decode, or whose `attempts` is not a number, is reserved raw, dead-lettered byte for byte
+and acked, so it never lingers in `:reserved`. That holds when the body cannot be re-encoded
+either (for example a number such as `1e400`, which real Redis' `cjson` refuses to serialise), and
+it holds under `on_unknown_urn="delete"` too: such a message is dead-lettered, never deleted. Without
+a dead-letter queue it is dropped, and the drop is logged as a warning. Dead-lettering a
+reserved message is one atomic script (`ZREM` the reservation, then `RPUSH` the dead-letter
+record and its notify entry); if the reservation expired first, nothing is written and a warning
+says so, because the job was already migrated back and will be delivered again.
+
+**Limit (A3):** the Laravel-compatible layout stores the bare envelope (Laravel's reserve script
+has to decode it), so out-of-band transport headers such as the W3C `traceparent` cannot be
+carried there. They are dropped without an error; the `trace_id` inside the envelope is still
+preserved. The default (legacy) layout carries them.
 
 ## Framework adapters — Celery & Django
 

{babelqueue-1.21.1 → babelqueue-1.23.0}/README.md

@@ -141,7 +141,7 @@ app.run()                               # consume forever (Ctrl-C to stop)
   messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
 - **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
   Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
-  see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
+  see below), `kafka://` (opt-in retry tiers, see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
   and `memory://` (in-process, great for tests/local). Bring your own by passing
   `transport=...`.
 
@@ -194,7 +194,10 @@ transport **always** releases with `ChangeMessageVisibility` (broker-bindings
 attempt counter, so a shutdown release also consumes an attempt. The default
 delay is **0 s** (`retry_backoff` / `unknown_urn_release_delay`), i.e. the
 message is visible again immediately; delays outside 0..43200 s are clamped with
-a warning.
+a warning. A **per-message delay** at publish time is
+`SqsTransport.publish(queue, body, delay_seconds=n)` (`DelaySeconds`, 0..900 on a
+standard queue; a larger value, or any per-message delay on a FIFO queue, raises
+`BabelQueueError` instead of being truncated or dropped).
 
 **Poison-loop risk:** a message that always fails is redelivered at once until
 `max_attempts` is reached. If the receive count is unavailable the SDK cannot
@@ -203,11 +206,98 @@ advance attempts at all. **Configure a native `RedrivePolicy`**
 queue as the broker-side backstop, and set a non-zero `retry_backoff` if the
 handler's dependencies need time to recover.
 
+### Retry back-off
+
+`BabelQueue(..., retry_backoff=, retry_backoff_multiplier=, retry_backoff_max=, retry_jitter=)`
+set the worker-side retry delay (ADR-0034 section 9): after the n-th failure the message is
+released with `d(n) = min(retry_backoff_max, retry_backoff * retry_backoff_multiplier ** (n - 1))`
+seconds (`retry_backoff_max=0` means no cap) and `retry_jitter="full"` draws `uniform(0, d(n))`.
+The defaults (`retry_backoff=0`, multiplier 1, jitter `"none"`) keep the previous behaviour: an
+immediate release. How the delay is honoured depends on the transport: SQS uses it as the
+visibility timeout, the Laravel-compatible Redis layout as the `:delayed` score and Kafka picks a
+retry tier (see below).
+
+### Kafka retry tiers and the re-inject consumer
+
+Kafka has no per-message delay, so delayed delivery and retry back-off are SDK-owned **retry
+topics** (broker-bindings section 6.4, ADR-0034). They are opt-in: pass `retry_tiers_ms` to the
+transport (the recommended set is `DEFAULT_RETRY_TIERS_MS` = 5 s, 30 s, 5 min, 30 min):
+
+```python
+from babelqueue import BabelQueue
+from babelqueue.kafka_transport import KafkaTransport, DEFAULT_RETRY_TIERS_MS
+from babelqueue.kafka_retry import RetryReinjector
+
+transport = KafkaTransport("kafka://localhost:9092", retry_tiers_ms=DEFAULT_RETRY_TIERS_MS)
+app = BabelQueue(transport=transport, queue="orders", max_attempts=5, dead_letter=True)
+
+# Run one next to the workers, otherwise a parked message is never delivered again:
+RetryReinjector("kafka://localhost:9092", work_topics=["orders"],
+                retry_tiers_ms=DEFAULT_RETRY_TIERS_MS).run()
+```
+
+A failed message is republished (attempts + 1) to `<queue>.retry.<n>`, where
+`n = min(attempts, N - 1) + 1`; `publish_delayed` picks the smallest tier that covers the requested
+delay. Create the `<queue>.retry.<n>` topics up front. The `RetryReinjector` copies each record's
+value and key byte for byte back to its work topic once the tier delay has elapsed (headers
+`bq-delay` / `bq-original-topic` are dropped) and commits only after the broker confirmed the
+write, so delivery stays at-least-once. It holds every record it has accepted per partition (oldest
+first, a record never overtakes the one before it) and a failed write rewinds the partition to the
+failed record. Without `retry_tiers_ms`, `publish_delayed` and a retry
+raise `BabelQueueError` and nothing is produced. The worker does not stop on that error: it logs
+it, leaves the record uncommitted (the consumer seeks back to it and the loop pauses briefly, so
+the record is redelivered and never lost) and keeps consuming. A worker started without tiers
+while `max_attempts` is above 1 logs a warning at start-up: a failing handler cannot be retried
+(the record keeps failing in place) until tiers are configured. The re-inject delay is measured from the record
+timestamp: set `message.timestamp.type=LogAppendTime` on the `.retry.<n>` topics so that
+timestamp is the broker's, not a producer clock's.
+
+**Records without an identity are poison.** A Kafka record whose JSON object has neither a valid
+string `job` nor a valid string `urn` (ADR-0034 section 10) is poison, not an unknown URN. With
+the DLQ on (`dead_letter` or `on_unknown_urn=DEAD_LETTER`) it is dead-lettered on its first delivery
+(wrapped with `dead_letter.reason: "poison"`, key preserved) and never retried: the record is
+written to `<topic>.dlq` (which must exist) and its offset is committed once that write
+is confirmed; a failed write leaves it uncommitted and rewound. With the DLQ off nothing is written:
+`on_unknown_urn=DELETE` drops it, any other strategy retries it through the retry tiers up to
+`max_attempts` and then drops it. **Without retry tiers** that bounded retry has no tier to use, so
+(with the DLQ off and `on_unknown_urn` not `DELETE`) the poison record is committed, dropped and logged
+instead of blocking the partition (section 6.5 degrade-to-delete, ADR-0036 decision 5); enable
+`dead_letter` to keep it in `<topic>.dlq`. A failing handler without tiers still cannot be retried, as above.
+
+### Poison messages on every other transport
+
+A message body that does not decode into an envelope (not a JSON object, malformed, a wrongly typed
+member, or bytes that are not valid UTF-8, even inside a JSON string) or that has neither a valid
+string `job` nor a valid string `urn` is **poison** on every transport (ADR-0036 decisions 2, 4 and
+6; conformance `identity_verdict`). It is never handed to a handler and never takes the
+unknown-URN strategies, which are for a valid envelope whose URN has no handler:
+
+- DLQ on (`dead_letter` or `on_unknown_urn=DEAD_LETTER`): written to `<queue>.dlq` at once, with no
+  retry, as the ADR-0035 record (`dead_letter.reason: "poison"`, the original body preserved,
+  base64 plus `raw_encoding` when it is not UTF-8), then acknowledged. A failed write leaves it
+  unacknowledged.
+- `on_unknown_urn=DELETE`: acknowledged and dropped (logged).
+- DLQ off: the bounded `FAIL` path (`RELEASE` is replaced by it): retried up to `max_attempts`, then
+  dropped and logged. The count comes from the broker where it has one (SQS receive count, Pulsar
+  `bq-attempts` / redelivery count), otherwise from a bare `{"attempts": n}` body the retry leaves.
+
+On Redis (both layouts) an identity-less body is dead-lettered as poison in **every** configuration,
+including `on_unknown_urn=DELETE` and dead-lettering off (broker-bindings section 1.1). On the legacy
+(`<queue>:processing`) layout a body that Go's typed decode rejects is dead-lettered raw on its first
+delivery and removed from `:processing`.
+
 ### Sharing a Redis queue with Laravel
 
 By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
-`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
-**shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
+`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). Each reservation also
+writes a `<token>:<raw>` member (32 lowercase hex characters of token) into the
+`<queue>:processing:deadlines` sorted set, and before every reserve a sweep moves the lapsed
+reservations back to the end of the list, so a `kill -9` no longer strands a message in
+`:processing`; a late ack of a swept reservation touches nothing of the newer one. `retry_after`
+(default **90** seconds; `None` / `?retry_after=null` disables recovery) sets that deadline, and
+`migration_batch_size` (`-1` = all, otherwise `>= 1`) bounds how many due members one sweep or
+migration moves. To consume a **shared** Laravel BabelQueue Redis queue instead, enable
+Laravel-compatible mode:
 
 ```python
 app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
@@ -224,8 +314,21 @@ member a Python worker writes is identical to a Laravel worker's — either side
 ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
 one queue without losing or double-processing messages. Before each pop, expired
 reserved/delayed jobs migrate back to the ready list, so a crashed worker's
-in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
-(or the `key_prefix` / `retry_after` constructor args).
+in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=90`
+(or the `key_prefix` / `retry_after` / `migration_batch_size` constructor args). A job whose body
+does not decode, or whose `attempts` is not a number, is reserved raw, dead-lettered byte for byte
+and acked, so it never lingers in `:reserved`. That holds when the body cannot be re-encoded
+either (for example a number such as `1e400`, which real Redis' `cjson` refuses to serialise), and
+it holds under `on_unknown_urn="delete"` too: such a message is dead-lettered, never deleted. Without
+a dead-letter queue it is dropped, and the drop is logged as a warning. Dead-lettering a
+reserved message is one atomic script (`ZREM` the reservation, then `RPUSH` the dead-letter
+record and its notify entry); if the reservation expired first, nothing is written and a warning
+says so, because the job was already migrated back and will be delivered again.
+
+**Limit (A3):** the Laravel-compatible layout stores the bare envelope (Laravel's reserve script
+has to decode it), so out-of-band transport headers such as the W3C `traceparent` cannot be
+carried there. They are dropped without an error; the `trace_id` inside the envelope is still
+preserved. The default (legacy) layout carries them.
 
 ## Framework adapters — Celery & Django
 

babelqueue-1.23.0/conformance-known-gaps.json

@@ -0,0 +1 @@
+{"gaps": []}

{babelqueue-1.21.1 → babelqueue-1.23.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "babelqueue"
-version = "1.21.1"
+version = "1.23.0"
 description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
 readme = "README.md"
 requires-python = ">=3.9"

{babelqueue-1.21.1 → babelqueue-1.23.0}/src/babelqueue/__init__.py

@@ -31,6 +31,7 @@ from .replay import HEADER_REPLAY_BYPASS, bypass_external_effects, is_replay
 from .routing import UnknownUrnStrategy
 from .transport import (
     BatchPopper,
+    DeadLetterWriter,
     HeaderPublisher,
     InMemoryTransport,
     ReceivedMessage,
@@ -41,7 +42,7 @@ from .transport import (
     Transport,
 )
 
-__version__ = "1.21.1"
+__version__ = "1.23.0"
 
 __all__ = [
     "BabelQueue",
@@ -55,6 +56,7 @@ __all__ = [
     "InMemoryTransport",
     "ReceivedMessage",
     "BatchPopper",
+    "DeadLetterWriter",
     "HeaderPublisher",
     "Redeliverer",
     "Rewinder",

{babelqueue-1.21.1 → babelqueue-1.23.0}/src/babelqueue/_envelope_shape.py

@@ -5,7 +5,7 @@ Go reference's ``babelqueue.Decode``, except that member names are case-sensitiv
 
 * the body must be exactly one JSON value (no BOM, trailing tokens, ``NaN`` / ``Infinity``, malformed
   numbers); a bare ``null`` decodes to an empty envelope, any other non-object root is rejected;
-* ``job``, ``trace_id``, ``meta.id|queue|lang`` and ``dead_letter.reason|error|exception|original_queue|lang``
+* ``trace_id``, ``meta.id|queue|lang`` and ``dead_letter.reason|error|exception|original_queue|lang``
   are string-or-null;
 * ``attempts``, ``meta.schema_version|created_at`` and ``dead_letter.failed_at|attempts`` are
   integer-or-null: an integer literal (no fraction or exponent) that fits ``int64``;
@@ -14,6 +14,8 @@ Go reference's ``babelqueue.Decode``, except that member names are case-sensitiv
 * member names are matched exactly as spelled (case-sensitive, as the contract schema and the core
   decoder read them) and every occurrence of a duplicated key is checked; any other member (a
   differently-cased name or a non-string ``urn`` alias included) is only checked for syntax.
+  A non-string ``job`` is **not** poison either: the identity resolver ignores it (ADR-0032 section 6),
+  so the body is readable and simply carries no identity (conformance ``kafka_poison``, v1.6.0, and ``identity_verdict``, v1.7.0).
 
 A well-typed but non-conformant envelope is not poison: Go's consume path never calls ``Accepts``.
 """
@@ -64,7 +66,6 @@ _DEAD_LETTER = {
     "attempts": _INTEGER,
 }
 _TOP = {
-    "job": _STRING,
     "trace_id": _STRING,
     "attempts": _INTEGER,
     "data": _DATA,