babelqueue 1.21.0__tar.gz → 1.22.0__tar.gz

{babelqueue-1.21.0 → babelqueue-1.22.0}/.github/workflows/ci.yml

@@ -12,7 +12,8 @@ permissions:
 jobs:
   test:
     name: Python ${{ matrix.python }}
-    runs-on: ubuntu-latest
+    # setup-python ships no 3.9 build for ubuntu-26.04 (ubuntu-latest); keep 3.9 on 24.04 while it stays supported.
+    runs-on: ${{ matrix.python == '3.9' && 'ubuntu-24.04' || 'ubuntu-latest' }}
     strategy:
       fail-fast: false
       matrix:
@@ -104,6 +105,8 @@ jobs:
 
       - name: Run full suite with coverage gate (>=90%)
         env:
+          # A real redis:7 is mandatory here: no silent skip or fakeredis fallback (shared runner contract).
+          BQ_REQUIRE_REDIS: "1"
           BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
           BABELQUEUE_TEST_AMQP: amqp://guest:guest@localhost:5672/
           SQS_ENDPOINT: http://localhost:9324
@@ -162,9 +165,19 @@ jobs:
   conformance:
     name: Conformance suite in sync + results.json
     runs-on: ubuntu-latest
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
     env:
       # The pinned conformance release (ADR-0032). Bump together with tests/conformance/.
-      BQ_CONFORMANCE_VERSION: 1.5.0
+      BQ_CONFORMANCE_VERSION: 1.6.0
       # sha256 of that release's fetch.sh: the script runs on the runner, so it is pinned rather
       # than trusted from a mutable tag. Bump together with BQ_CONFORMANCE_VERSION.
       BQ_FETCH_SH_SHA256: 8bb6ba567b288fb0ac71a70fd17041b2503df119e01998d76fdad372570bf849
@@ -190,11 +203,13 @@ jobs:
       - name: Install
         run: |
           python -m pip install --upgrade pip
-          pip install -e ".[dev,celery,django,otel]" opentelemetry-sdk jsonschema
+          pip install -e ".[dev,celery,django,otel,redis]" opentelemetry-sdk jsonschema
 
       - name: Run the conformance suite and write results.json
         env:
           BQ_CONFORMANCE_RESULTS: ${{ runner.temp }}/results.json
+          BQ_REQUIRE_REDIS: "1"
+          BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
         run: pytest -q tests/test_conformance.py
 
       - name: Validate results.json (schema, bindings, sections)

{babelqueue-1.21.0 → babelqueue-1.22.0}/.github/workflows/release.yml

@@ -14,9 +14,21 @@ jobs:
     # Recommended for PyPI Trusted Publishing — configure a "pypi" environment +
     # a trusted publisher on PyPI (repo BabelQueue/babelqueue-python, this workflow).
     environment: pypi
+    # redis_layout (conformance 1.6.0) drives the real Lua scripts: a real redis:7 is mandatory,
+    # exactly as in ci.yml's `conformance` job — without it the cases are reported as failures.
+    services:
+      redis:
+        image: redis:7
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 5s
+          --health-timeout 3s
+          --health-retries 10
     env:
       # The pinned conformance release (ADR-0032). Keep identical to ci.yml.
-      BQ_CONFORMANCE_VERSION: 1.5.0
+      BQ_CONFORMANCE_VERSION: 1.6.0
       BQ_FETCH_SH_SHA256: 8bb6ba567b288fb0ac71a70fd17041b2503df119e01998d76fdad372570bf849
     permissions:
       id-token: write   # PyPI Trusted Publishing (OIDC) — no API token needed
@@ -40,9 +52,12 @@ jobs:
       - name: Install & test (writes results.json)
         env:
           BQ_CONFORMANCE_RESULTS: ${{ runner.temp }}/results.json
+          # A real redis:7 is mandatory: no silent skip or fakeredis fallback (shared runner contract).
+          BQ_REQUIRE_REDIS: "1"
+          BABELQUEUE_TEST_REDIS: redis://localhost:6379/0
         run: |
           python -m pip install --upgrade pip build
-          pip install -e ".[dev,celery,django,otel]" opentelemetry-sdk jsonschema
+          pip install -e ".[dev,celery,django,otel,redis]" opentelemetry-sdk jsonschema
           pytest
           test -s "$BQ_CONFORMANCE_RESULTS"  # the conformance run must have produced results.json
 

{babelqueue-1.21.0 → babelqueue-1.22.0}/CHANGELOG.md

@@ -9,6 +9,113 @@ The envelope wire format is versioned separately by `meta.schema_version`
 
 ## [Unreleased]
 
+## [1.22.0] - 2026-10-24
+
+Implements ADR-0033 (Redis) and ADR-0034 (Kafka tiers, backoff, poison). The envelope stays at
+`schema_version: 1`; the vendored conformance suite moves to v1.6.0 and
+`conformance-known-gaps.json` is now empty.
+
+### Changed
+- The Redis poison dead-letter element is now the ADR-0035 record (raw splice, or the `raw`/`raw_encoding` wrapper) carrying a `dead_letter` block, instead of the bare element (broker-bindings section 1). A legacy `__bq_frame` header is unwrapped, not re-applied. Dropped the dead `_reject_json_constant` helper in the Kafka transport.
+- **Redis: a stored value that is not valid UTF-8 no longer crashes the reserve.** The client built from a URL now uses `encoding_errors="surrogateescape"`, replies are decoded losslessly, and the settle / release / dead-letter calls send the exact stored bytes back, so such a poison body becomes the ADR-0035 base64 wrapper (no U+FFFD) and its reservation is actually removed. A client you pass in with `decode_responses=True` should set `encoding_errors="surrogateescape"` as well (redis-py's default strict decoding raises on such a reply).
+- **Poison DLQ record shape follows ADR-0035** (Kafka): a strict JSON object body is spliced byte for byte with a `dead_letter` member, anything else is wrapped as `{"raw": ..., "raw_encoding"?: "base64", "dead_letter": ...}`; a body that is not valid UTF-8 is stored losslessly as base64 instead of with U+FFFD, and the conformance `poison_dlq` section runs.
+
+### Fixed
+- **Invalid UTF-8 is poison in both Redis layouts (broker-bindings section 1.1).** A stored value (or zset `:reserved` member) that is not valid UTF-8 is dead-lettered on first delivery as the ADR-0035 `raw`/`raw_encoding: "base64"` wrapper, even when the codec or `cjson` could decode it: it is never handed to a handler and never retried, released or re-published, and the exact stored bytes are used for the settle script, so no byte is rewritten.
+- **Redis, Laravel-compatible layout: a poison reservation is dead-lettered in every
+  configuration.** The DLQ write no longer depends on `dead_letter_enabled` (default `False`) or
+  `on_unknown_urn`: an undecodable body or one without a numeric `attempts` is written raw to
+  `<queue>.dlq` and acked instead of being logged and dropped (broker-bindings section 1.1,
+  ADR-0033; same as the Go reference). Other version-gated routes keep their previous behaviour.
+- **Redis, Laravel-compatible layout: a job with a non-numeric `attempts` or an undecodable body no
+  longer aborts the reserve script.** It is reserved raw (`pcall` guard), dead-lettered byte for
+  byte and acked, leaving nothing in `:reserved` (ADR-0033 (c)).
+- **Redis, legacy layout: a `kill -9` no longer strands a message in `<queue>:processing`.** Every
+  reservation writes a `<token>:<raw>` member into `<queue>:processing:deadlines`; a sweep before
+  each reserve moves lapsed reservations back to the end of `<queue>`. Ack, release and dead-letter
+  are gated on their own reservation, so a late ack never touches a newer one. The legacy mode
+  remains the default and the `<queue>:processing` key name is unchanged.
+- **Kafka without retry tiers: a failing handler no longer stops the worker** (Go
+  `leaveRepublishUnsettled` parity). The transport still raises `BabelQueueError` (section 6.5), but
+  the runtime now catches a failed retry release, logs it (`babelqueue.app`, ERROR, "NOT
+  acknowledged"), leaves the record unsettled (no commit; a rewind-capable transport seeks back and
+  the loop pauses `_rewind_pause` first) and keeps consuming. Previously the error escaped
+  `consume()`. Applies to any transport whose retry release fails.
+- **Kafka poison follows Go `routePoison` when no DLQ is configured.** With the DLQ on
+  (`dead_letter=True` or `on_unknown_urn=DEAD_LETTER`) the wrapped record is written to
+  `<topic>.dlq` (`dead_letter.reason: "poison"`) and then committed. With the DLQ off the poison
+  record is **not** written anywhere: `on_unknown_urn=DELETE` drops it, every other strategy takes
+  the bounded FAIL path (retried through the retry tiers up to `max_attempts` with a bare
+  `{"attempts": N}` body, then dropped; without tiers the record is redelivered in place).
+  Operational note: when the DLQ is on, `<topic>.dlq` must exist; a failed write leaves the record
+  uncommitted and rewound (it is never dropped).
+- **Unknown-URN `RELEASE` and the retry release leave the delivery unsettled on failure** (Go
+  `leaveRepublishUnsettled`): a failed re-publish of an unknown-URN message is no longer counted as
+  a handler failure and can no longer lead to the record being acked and lost.
+- **Redis, legacy `:processing` layout: an undecodable body is dead-lettered on its first
+  delivery.** A body Go's typed `Decode` rejects (neither a JSON object nor `null`, or wrong member types such as
+  a string `attempts`/`meta`; same criterion as Kafka poison) is written verbatim to `<queue>.dlq`
+  through the gated settle script and removed from `:processing`, in every configuration, instead of
+  being retried until `max_attempts`. A valid JSON object without an identity keeps the existing
+  bounded path.
+- **Kafka: a record without a valid `job` / `urn` is poison** (ADR-0034 section 10), not an
+  unknown URN: with the DLQ on it is dead-lettered wrapped with `dead_letter.reason: "poison"` and
+  never retried; with the DLQ off it follows Go `routePoison` (see above).
+
+- **Redis, Laravel-compatible layout: the reserve script no longer aborts on a body that decodes
+  but cannot be re-encoded** (`cjson.encode` of `inf`/`NaN`, e.g. `1e400`, raised after the `LPOP`
+  and lost the job). The encode is now inside `pcall`; the job is reserved raw and takes the poison
+  path. Note: fakeredis' `cjson` serialises `inf` as `Infinity` and masks this; it is proven by a
+  real-Redis test (CI `redis:7`) and by a Lua test against a failing `cjson` stub.
+- **`on_unknown_urn="delete"` no longer discards a Redis poison reservation.** It is dead-lettered
+  byte for byte like the Kafka poison record already was; a genuine unknown URN is still deleted.
+- **Redis, Laravel-compatible layout: `dead_letter` is atomic and gated.** One script `ZREM`s the
+  reservation and writes the dead-letter record and its notify entry; an expired reservation writes
+  nothing (and logs a warning) instead of publishing a duplicate.
+- **A message dropped because it is undeliverable and no dead-letter queue is configured is
+  logged** (`babelqueue.app`, WARNING) instead of disappearing silently.
+- **Kafka `RetryReinjector` no longer loses records:** it kept one held record per partition, so a
+  second record on the same partition overwrote the first. It now keeps all of them in order.
+- **A worker on a tier-less Kafka transport with `max_attempts` > 1 warns at start-up** that a
+  failing handler cannot be retried: the record is left uncommitted and redelivered.
+- Internal: the legacy `:processing` settle script takes `[token, raw]` and builds the deadline
+  member itself.
+
+### Added
+- `RedisTransport(retry_after=90, migration_batch_size=-1)` (also `?retry_after=` in the URL;
+  `None` / `null` disables recovery). The Laravel-compatible `retry_after` default is now 90 s
+  (was 60 s).
+- SQS: `SqsTransport.publish(..., delay_seconds=)` (`DelaySeconds` 0..900; out of range or FIFO
+  raises `BabelQueueError`).
+- Kafka: `KafkaTransport(retry_tiers_ms=...)` with `DEFAULT_RETRY_TIERS_MS`, `retry_topic()` and
+  the `babelqueue.kafka_retry.RetryReinjector` consumer that moves parked records from
+  `<topic>.retry.<n>` back to the work topic (broker-bindings section 6.4). Opt-in: without tiers a
+  retry / `publish_delayed` raises `BabelQueueError` (the transport raises; the worker logs it and
+  leaves the record uncommitted, see Fixed).
+- Worker-side retry backoff: `BabelQueue(retry_backoff=, retry_backoff_multiplier=,
+  retry_backoff_max=, retry_jitter=)` (ADR-0034 section 9); the defaults keep an immediate release.
+- Conformance runner covers the v1.6.0 sections (`redis_layout`, `redis_delay`, `retry_backoff`,
+  `kafka_poison`, ...). CI runs the Redis cases against a real `redis:7` with `BQ_REQUIRE_REDIS=1`.
+
+### Changed
+- **Kafka retries go to the retry tiers.** With `retry_tiers_ms` set a failed message is parked on
+  `<queue>.retry.<n>` and needs a running `RetryReinjector`; the retry topics must exist. Without
+  tiers a retry now raises from the transport instead of republishing onto the work topic; the
+  runtime logs it and redelivers the record rather than stopping the worker.
+
+### Documentation
+- README: Redis layouts, the limit that the Laravel-compatible layout carries no out-of-band headers (ADR-0033 §5),
+  Kafka retry tiers, `LogAppendTime` on the tier topics, retry backoff, SQS `delay_seconds`.
+
+## [1.21.1] - 2026-10-20
+
+### Fixed
+- **Azure Service Bus: `publish` no longer fails with `Value supplied for int invalid`.** The pure-Python AMQP
+  stack in `azure-servicebus` encodes a bare Python int as AMQP int32, so the millisecond `bq-created-at`
+  application property (broker-bindings.md §4.3) overflowed at send time. Out-of-int32 integer properties
+  are now sent as AMQP `long`, matching the Go, Java, .NET and Node transports. The native projection and
+  the envelope body are unchanged.
+
 ## [1.21.0] - 2026-10-19
 
 Implements ADR-0032 (conformance packaging, results format, ASCII trim and the non-string `job` rule).

{babelqueue-1.21.0 → babelqueue-1.22.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: babelqueue
-Version: 1.21.0
+Version: 1.22.0
 Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
 Project-URL: Homepage, https://babelqueue.com
 Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
@@ -195,7 +195,7 @@ app.run()                               # consume forever (Ctrl-C to stop)
   messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
 - **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
   Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
-  see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
+  see below), `kafka://` (opt-in retry tiers, see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
   and `memory://` (in-process, great for tests/local). Bring your own by passing
   `transport=...`.
 
@@ -248,7 +248,10 @@ transport **always** releases with `ChangeMessageVisibility` (broker-bindings
 attempt counter, so a shutdown release also consumes an attempt. The default
 delay is **0 s** (`retry_backoff` / `unknown_urn_release_delay`), i.e. the
 message is visible again immediately; delays outside 0..43200 s are clamped with
-a warning.
+a warning. A **per-message delay** at publish time is
+`SqsTransport.publish(queue, body, delay_seconds=n)` (`DelaySeconds`, 0..900 on a
+standard queue; a larger value, or any per-message delay on a FIFO queue, raises
+`BabelQueueError` instead of being truncated or dropped).
 
 **Poison-loop risk:** a message that always fails is redelivered at once until
 `max_attempts` is reached. If the receive count is unavailable the SDK cannot
@@ -257,11 +260,76 @@ advance attempts at all. **Configure a native `RedrivePolicy`**
 queue as the broker-side backstop, and set a non-zero `retry_backoff` if the
 handler's dependencies need time to recover.
 
+### Retry back-off
+
+`BabelQueue(..., retry_backoff=, retry_backoff_multiplier=, retry_backoff_max=, retry_jitter=)`
+set the worker-side retry delay (ADR-0034 section 9): after the n-th failure the message is
+released with `d(n) = min(retry_backoff_max, retry_backoff * retry_backoff_multiplier ** (n - 1))`
+seconds (`retry_backoff_max=0` means no cap) and `retry_jitter="full"` draws `uniform(0, d(n))`.
+The defaults (`retry_backoff=0`, multiplier 1, jitter `"none"`) keep the previous behaviour: an
+immediate release. How the delay is honoured depends on the transport: SQS uses it as the
+visibility timeout, the Laravel-compatible Redis layout as the `:delayed` score and Kafka picks a
+retry tier (see below).
+
+### Kafka retry tiers and the re-inject consumer
+
+Kafka has no per-message delay, so delayed delivery and retry back-off are SDK-owned **retry
+topics** (broker-bindings section 6.4, ADR-0034). They are opt-in: pass `retry_tiers_ms` to the
+transport (the recommended set is `DEFAULT_RETRY_TIERS_MS` = 5 s, 30 s, 5 min, 30 min):
+
+```python
+from babelqueue import BabelQueue
+from babelqueue.kafka_transport import KafkaTransport, DEFAULT_RETRY_TIERS_MS
+from babelqueue.kafka_retry import RetryReinjector
+
+transport = KafkaTransport("kafka://localhost:9092", retry_tiers_ms=DEFAULT_RETRY_TIERS_MS)
+app = BabelQueue(transport=transport, queue="orders", max_attempts=5, dead_letter=True)
+
+# Run one next to the workers, otherwise a parked message is never delivered again:
+RetryReinjector("kafka://localhost:9092", work_topics=["orders"],
+                retry_tiers_ms=DEFAULT_RETRY_TIERS_MS).run()
+```
+
+A failed message is republished (attempts + 1) to `<queue>.retry.<n>`, where
+`n = min(attempts, N - 1) + 1`; `publish_delayed` picks the smallest tier that covers the requested
+delay. Create the `<queue>.retry.<n>` topics up front. The `RetryReinjector` copies each record's
+value and key byte for byte back to its work topic once the tier delay has elapsed (headers
+`bq-delay` / `bq-original-topic` are dropped) and commits only after the broker confirmed the
+write, so delivery stays at-least-once. It holds every record it has accepted per partition (oldest
+first, a record never overtakes the one before it) and a failed write rewinds the partition to the
+failed record. Without `retry_tiers_ms`, `publish_delayed` and a retry
+raise `BabelQueueError` and nothing is produced. The worker does not stop on that error: it logs
+it, leaves the record uncommitted (the consumer seeks back to it and the loop pauses briefly, so
+the record is redelivered and never lost) and keeps consuming. A worker started without tiers
+while `max_attempts` is above 1 logs a warning at start-up: a failing handler cannot be retried
+(the record keeps failing in place) until tiers are configured. The re-inject delay is measured from the record
+timestamp: set `message.timestamp.type=LogAppendTime` on the `.retry.<n>` topics so that
+timestamp is the broker's, not a producer clock's.
+
+**Records without an identity are poison.** A Kafka record whose JSON object has neither a valid
+string `job` nor a valid string `urn` (ADR-0034 section 10) is poison, not an unknown URN. With
+the DLQ on (`dead_letter` or `on_unknown_urn=DEAD_LETTER`) it is dead-lettered on its first delivery
+(wrapped with `dead_letter.reason: "poison"`, key preserved) and never retried: the record is
+written to `<topic>.dlq` (which must exist) and its offset is committed once that write
+is confirmed; a failed write leaves it uncommitted and rewound. With the DLQ off nothing is written:
+`on_unknown_urn=DELETE` drops it, any other strategy retries it through the retry tiers up to
+`max_attempts` and then drops it (without tiers the record is redelivered in place, as above). On
+the Redis legacy (`<queue>:processing`) layout a body that Go's typed decode rejects (neither a JSON
+object nor `null`, or wrong member types) is dead-lettered raw on its first
+delivery and removed from `:processing`.
+
 ### Sharing a Redis queue with Laravel
 
 By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
-`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
-**shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
+`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). Each reservation also
+writes a `<token>:<raw>` member (32 lowercase hex characters of token) into the
+`<queue>:processing:deadlines` sorted set, and before every reserve a sweep moves the lapsed
+reservations back to the end of the list, so a `kill -9` no longer strands a message in
+`:processing`; a late ack of a swept reservation touches nothing of the newer one. `retry_after`
+(default **90** seconds; `None` / `?retry_after=null` disables recovery) sets that deadline, and
+`migration_batch_size` (`-1` = all, otherwise `>= 1`) bounds how many due members one sweep or
+migration moves. To consume a **shared** Laravel BabelQueue Redis queue instead, enable
+Laravel-compatible mode:
 
 ```python
 app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
@@ -278,8 +346,21 @@ member a Python worker writes is identical to a Laravel worker's — either side
 ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
 one queue without losing or double-processing messages. Before each pop, expired
 reserved/delayed jobs migrate back to the ready list, so a crashed worker's
-in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
-(or the `key_prefix` / `retry_after` constructor args).
+in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=90`
+(or the `key_prefix` / `retry_after` / `migration_batch_size` constructor args). A job whose body
+does not decode, or whose `attempts` is not a number, is reserved raw, dead-lettered byte for byte
+and acked, so it never lingers in `:reserved`. That holds when the body cannot be re-encoded
+either (for example a number such as `1e400`, which real Redis' `cjson` refuses to serialise), and
+it holds under `on_unknown_urn="delete"` too: such a message is dead-lettered, never deleted. Without
+a dead-letter queue it is dropped, and the drop is logged as a warning. Dead-lettering a
+reserved message is one atomic script (`ZREM` the reservation, then `RPUSH` the dead-letter
+record and its notify entry); if the reservation expired first, nothing is written and a warning
+says so, because the job was already migrated back and will be delivered again.
+
+**Limit (A3):** the Laravel-compatible layout stores the bare envelope (Laravel's reserve script
+has to decode it), so out-of-band transport headers such as the W3C `traceparent` cannot be
+carried there. They are dropped without an error; the `trace_id` inside the envelope is still
+preserved. The default (legacy) layout carries them.
 
 ## Framework adapters — Celery & Django
 

{babelqueue-1.21.0 → babelqueue-1.22.0}/README.md

@@ -141,7 +141,7 @@ app.run()                               # consume forever (Ctrl-C to stop)
   messages on `<queue>.dlq`. `on_unknown_urn` = `fail` | `delete` | `release` | `dead_letter`.
 - **Transports:** `redis://` (reliable-queue pattern; add `?laravel=1` to share a
   Redis queue with a Laravel BabelQueue worker using its reserved-set semantics —
-  see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
+  see below), `kafka://` (opt-in retry tiers, see below), `amqp://` (RabbitMQ via `pika`, with the contract AMQP properties)
   and `memory://` (in-process, great for tests/local). Bring your own by passing
   `transport=...`.
 
@@ -194,7 +194,10 @@ transport **always** releases with `ChangeMessageVisibility` (broker-bindings
 attempt counter, so a shutdown release also consumes an attempt. The default
 delay is **0 s** (`retry_backoff` / `unknown_urn_release_delay`), i.e. the
 message is visible again immediately; delays outside 0..43200 s are clamped with
-a warning.
+a warning. A **per-message delay** at publish time is
+`SqsTransport.publish(queue, body, delay_seconds=n)` (`DelaySeconds`, 0..900 on a
+standard queue; a larger value, or any per-message delay on a FIFO queue, raises
+`BabelQueueError` instead of being truncated or dropped).
 
 **Poison-loop risk:** a message that always fails is redelivered at once until
 `max_attempts` is reached. If the receive count is unavailable the SDK cannot
@@ -203,11 +206,76 @@ advance attempts at all. **Configure a native `RedrivePolicy`**
 queue as the broker-side backstop, and set a non-zero `retry_backoff` if the
 handler's dependencies need time to recover.
 
+### Retry back-off
+
+`BabelQueue(..., retry_backoff=, retry_backoff_multiplier=, retry_backoff_max=, retry_jitter=)`
+set the worker-side retry delay (ADR-0034 section 9): after the n-th failure the message is
+released with `d(n) = min(retry_backoff_max, retry_backoff * retry_backoff_multiplier ** (n - 1))`
+seconds (`retry_backoff_max=0` means no cap) and `retry_jitter="full"` draws `uniform(0, d(n))`.
+The defaults (`retry_backoff=0`, multiplier 1, jitter `"none"`) keep the previous behaviour: an
+immediate release. How the delay is honoured depends on the transport: SQS uses it as the
+visibility timeout, the Laravel-compatible Redis layout as the `:delayed` score and Kafka picks a
+retry tier (see below).
+
+### Kafka retry tiers and the re-inject consumer
+
+Kafka has no per-message delay, so delayed delivery and retry back-off are SDK-owned **retry
+topics** (broker-bindings section 6.4, ADR-0034). They are opt-in: pass `retry_tiers_ms` to the
+transport (the recommended set is `DEFAULT_RETRY_TIERS_MS` = 5 s, 30 s, 5 min, 30 min):
+
+```python
+from babelqueue import BabelQueue
+from babelqueue.kafka_transport import KafkaTransport, DEFAULT_RETRY_TIERS_MS
+from babelqueue.kafka_retry import RetryReinjector
+
+transport = KafkaTransport("kafka://localhost:9092", retry_tiers_ms=DEFAULT_RETRY_TIERS_MS)
+app = BabelQueue(transport=transport, queue="orders", max_attempts=5, dead_letter=True)
+
+# Run one next to the workers, otherwise a parked message is never delivered again:
+RetryReinjector("kafka://localhost:9092", work_topics=["orders"],
+                retry_tiers_ms=DEFAULT_RETRY_TIERS_MS).run()
+```
+
+A failed message is republished (attempts + 1) to `<queue>.retry.<n>`, where
+`n = min(attempts, N - 1) + 1`; `publish_delayed` picks the smallest tier that covers the requested
+delay. Create the `<queue>.retry.<n>` topics up front. The `RetryReinjector` copies each record's
+value and key byte for byte back to its work topic once the tier delay has elapsed (headers
+`bq-delay` / `bq-original-topic` are dropped) and commits only after the broker confirmed the
+write, so delivery stays at-least-once. It holds every record it has accepted per partition (oldest
+first, a record never overtakes the one before it) and a failed write rewinds the partition to the
+failed record. Without `retry_tiers_ms`, `publish_delayed` and a retry
+raise `BabelQueueError` and nothing is produced. The worker does not stop on that error: it logs
+it, leaves the record uncommitted (the consumer seeks back to it and the loop pauses briefly, so
+the record is redelivered and never lost) and keeps consuming. A worker started without tiers
+while `max_attempts` is above 1 logs a warning at start-up: a failing handler cannot be retried
+(the record keeps failing in place) until tiers are configured. The re-inject delay is measured from the record
+timestamp: set `message.timestamp.type=LogAppendTime` on the `.retry.<n>` topics so that
+timestamp is the broker's, not a producer clock's.
+
+**Records without an identity are poison.** A Kafka record whose JSON object has neither a valid
+string `job` nor a valid string `urn` (ADR-0034 section 10) is poison, not an unknown URN. With
+the DLQ on (`dead_letter` or `on_unknown_urn=DEAD_LETTER`) it is dead-lettered on its first delivery
+(wrapped with `dead_letter.reason: "poison"`, key preserved) and never retried: the record is
+written to `<topic>.dlq` (which must exist) and its offset is committed once that write
+is confirmed; a failed write leaves it uncommitted and rewound. With the DLQ off nothing is written:
+`on_unknown_urn=DELETE` drops it, any other strategy retries it through the retry tiers up to
+`max_attempts` and then drops it (without tiers the record is redelivered in place, as above). On
+the Redis legacy (`<queue>:processing`) layout a body that Go's typed decode rejects (neither a JSON
+object nor `null`, or wrong member types) is dead-lettered raw on its first
+delivery and removed from `:processing`.
+
 ### Sharing a Redis queue with Laravel
 
 By default the Redis transport owns its queue end-to-end (`RPUSH` to produce;
-`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). To consume a
-**shared** Laravel BabelQueue Redis queue instead, enable Laravel-compatible mode:
+`BLMOVE` into a `<queue>:processing` list to reserve; `LREM` on ack). Each reservation also
+writes a `<token>:<raw>` member (32 lowercase hex characters of token) into the
+`<queue>:processing:deadlines` sorted set, and before every reserve a sweep moves the lapsed
+reservations back to the end of the list, so a `kill -9` no longer strands a message in
+`:processing`; a late ack of a swept reservation touches nothing of the newer one. `retry_after`
+(default **90** seconds; `None` / `?retry_after=null` disables recovery) sets that deadline, and
+`migration_batch_size` (`-1` = all, otherwise `>= 1`) bounds how many due members one sweep or
+migration moves. To consume a **shared** Laravel BabelQueue Redis queue instead, enable
+Laravel-compatible mode:
 
 ```python
 app = BabelQueue("redis://localhost:6379/0?laravel=1", queue="orders")
@@ -224,8 +292,21 @@ member a Python worker writes is identical to a Laravel worker's — either side
 ack (`ZREM`) the other's reservation, so a Python worker and a Laravel worker share
 one queue without losing or double-processing messages. Before each pop, expired
 reserved/delayed jobs migrate back to the ready list, so a crashed worker's
-in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=60`
-(or the `key_prefix` / `retry_after` constructor args).
+in-flight job is re-reserved. Options: `?prefix=queues:` and `?retry_after=90`
+(or the `key_prefix` / `retry_after` / `migration_batch_size` constructor args). A job whose body
+does not decode, or whose `attempts` is not a number, is reserved raw, dead-lettered byte for byte
+and acked, so it never lingers in `:reserved`. That holds when the body cannot be re-encoded
+either (for example a number such as `1e400`, which real Redis' `cjson` refuses to serialise), and
+it holds under `on_unknown_urn="delete"` too: such a message is dead-lettered, never deleted. Without
+a dead-letter queue it is dropped, and the drop is logged as a warning. Dead-lettering a
+reserved message is one atomic script (`ZREM` the reservation, then `RPUSH` the dead-letter
+record and its notify entry); if the reservation expired first, nothing is written and a warning
+says so, because the job was already migrated back and will be delivered again.
+
+**Limit (A3):** the Laravel-compatible layout stores the bare envelope (Laravel's reserve script
+has to decode it), so out-of-band transport headers such as the W3C `traceparent` cannot be
+carried there. They are dropped without an error; the `trace_id` inside the envelope is still
+preserved. The default (legacy) layout carries them.
 
 ## Framework adapters — Celery & Django
 

babelqueue-1.22.0/conformance-known-gaps.json

@@ -0,0 +1 @@
+{"gaps": []}

{babelqueue-1.21.0 → babelqueue-1.22.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "babelqueue"
-version = "1.21.0"
+version = "1.22.0"
 description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
 readme = "README.md"
 requires-python = ">=3.9"

{babelqueue-1.21.0 → babelqueue-1.22.0}/src/babelqueue/__init__.py

@@ -31,6 +31,7 @@ from .replay import HEADER_REPLAY_BYPASS, bypass_external_effects, is_replay
 from .routing import UnknownUrnStrategy
 from .transport import (
     BatchPopper,
+    DeadLetterWriter,
     HeaderPublisher,
     InMemoryTransport,
     ReceivedMessage,
@@ -41,7 +42,7 @@ from .transport import (
     Transport,
 )
 
-__version__ = "1.21.0"
+__version__ = "1.22.0"
 
 __all__ = [
     "BabelQueue",
@@ -55,6 +56,7 @@ __all__ = [
     "InMemoryTransport",
     "ReceivedMessage",
     "BatchPopper",
+    "DeadLetterWriter",
     "HeaderPublisher",
     "Redeliverer",
     "Rewinder",

{babelqueue-1.21.0 → babelqueue-1.22.0}/src/babelqueue/_envelope_shape.py

@@ -5,7 +5,7 @@ Go reference's ``babelqueue.Decode``, except that member names are case-sensitiv
 
 * the body must be exactly one JSON value (no BOM, trailing tokens, ``NaN`` / ``Infinity``, malformed
   numbers); a bare ``null`` decodes to an empty envelope, any other non-object root is rejected;
-* ``job``, ``trace_id``, ``meta.id|queue|lang`` and ``dead_letter.reason|error|exception|original_queue|lang``
+* ``trace_id``, ``meta.id|queue|lang`` and ``dead_letter.reason|error|exception|original_queue|lang``
   are string-or-null;
 * ``attempts``, ``meta.schema_version|created_at`` and ``dead_letter.failed_at|attempts`` are
   integer-or-null: an integer literal (no fraction or exponent) that fits ``int64``;
@@ -14,6 +14,8 @@ Go reference's ``babelqueue.Decode``, except that member names are case-sensitiv
 * member names are matched exactly as spelled (case-sensitive, as the contract schema and the core
   decoder read them) and every occurrence of a duplicated key is checked; any other member (a
   differently-cased name or a non-string ``urn`` alias included) is only checked for syntax.
+  A non-string ``job`` is **not** poison either: the identity resolver ignores it (ADR-0032 section 6),
+  so the body is readable and simply carries no identity (conformance ``kafka_poison``, v1.6.0).
 
 A well-typed but non-conformant envelope is not poison: Go's consume path never calls ``Accepts``.
 """
@@ -64,7 +66,6 @@ _DEAD_LETTER = {
     "attempts": _INTEGER,
 }
 _TOP = {
-    "job": _STRING,
     "trace_id": _STRING,
     "attempts": _INTEGER,
     "data": _DATA,

babelqueue-1.22.0/src/babelqueue/_poison_record.py

@@ -0,0 +1,92 @@
+"""The DLQ record of a poison body (ADR-0035, error-handling.md section 6).
+
+A body that is a strict RFC 8259 JSON object (valid UTF-8, no ``NaN`` / ``Infinity`` literal, no
+unpaired surrogate escape, no top-level ``dead_letter`` member) is *spliced* at byte level: every
+original character is kept and only a ``dead_letter`` member is appended, so no value is re-encoded
+(``1e400`` and a 64-bit integer stay as written, a repeated member name is not collapsed). Anything
+else is wrapped as ``{"raw": <string>, "raw_encoding"?: "base64", "dead_letter": ...}``: ``raw`` is
+the body as plain text when it is valid UTF-8, otherwise the standard base64 of the exact bytes with
+``"raw_encoding": "base64"``. U+FFFD substitution never happens.
+"""
+
+from __future__ import annotations
+
+import base64
+import json
+import re
+from typing import Any, Dict, List, Optional, Tuple
+
+_JSON_WHITESPACE = " \t\r\n"
+_STRING_LITERAL = re.compile(r'"(?:[^"\\]|\\.)*"', re.DOTALL)
+_ESCAPE = re.compile(r"\\(u[0-9a-fA-F]{4}|.)", re.DOTALL)
+
+
+def _reject_constant(name: str) -> Any:
+    raise ValueError("invalid JSON constant " + name)
+
+
+def _has_unpaired_surrogate_escape(text: str) -> bool:
+    """True when a string literal of the (already syntax-valid) JSON ``text`` holds a ``\\uD800``-style
+    escape that is not half of a high+low pair."""
+    for literal in _STRING_LITERAL.finditer(text):
+        pending_high = False
+        for escape in _ESCAPE.finditer(literal.group(0)):
+            token = escape.group(1)
+            code = int(token[1:], 16) if len(token) == 5 and token[0] == "u" else -1
+            is_high = 0xD800 <= code <= 0xDBFF
+            is_low = 0xDC00 <= code <= 0xDFFF
+            if pending_high and not is_low:
+                return True
+            if is_low and not pending_high:
+                return True
+            pending_high = is_high
+        if pending_high:
+            return True
+    return False
+
+
+def _is_spliceable_object(text: str) -> bool:
+    """Whether ``text`` is one strict JSON object that can be spliced (ADR-0035 section 1)."""
+    last_pairs: List[Optional[List[Tuple[str, Any]]]] = [None]
+
+    def hook(pairs: List[Tuple[str, Any]]) -> Dict[str, Any]:
+        last_pairs[0] = pairs  # the outermost object is the last one the decoder completes
+        return {}
+
+    try:
+        parsed = json.loads(text, parse_constant=_reject_constant, object_pairs_hook=hook)
+    except (ValueError, RecursionError):
+        return False
+    if not isinstance(parsed, dict) or last_pairs[0] is None:
+        return False
+    if any(key == "dead_letter" for key, _ in last_pairs[0]):
+        return False
+    return not _has_unpaired_surrogate_escape(text)
+
+
+def build_poison_record(text: str, dead_letter_block: Dict[str, Any]) -> str:
+    """The DLQ record body for the poison body ``text``.
+
+    ``text`` is the body as the transport carries it: bytes that were not UTF-8 appear as
+    ``surrogateescape`` code points. ``dead_letter_block`` is the ``dead_letter`` member's value."""
+    try:
+        data = text.encode("utf-8", "surrogateescape")
+    except UnicodeEncodeError:
+        data = text.encode("utf-8", "surrogatepass")
+    try:
+        decoded: Optional[str] = data.decode("utf-8")
+    except UnicodeDecodeError:
+        decoded = None
+
+    block = json.dumps(dead_letter_block, ensure_ascii=False, separators=(",", ":"), allow_nan=False)
+    if decoded is not None and _is_spliceable_object(decoded):
+        trimmed = decoded.strip(_JSON_WHITESPACE)
+        if trimmed[1:-1].strip(_JSON_WHITESPACE) == "":
+            return '{"dead_letter":' + block + "}"
+        return trimmed[:-1] + ',"dead_letter":' + block + "}"
+
+    if decoded is not None:
+        raw = json.dumps(decoded, ensure_ascii=False)
+        return '{"raw":' + raw + ',"dead_letter":' + block + "}"
+    encoded = base64.b64encode(data).decode("ascii")
+    return '{"raw":"' + encoded + '","raw_encoding":"base64","dead_letter":' + block + "}"