babelqueue 1.3.0__tar.gz → 1.4.0__tar.gz

{babelqueue-1.3.0 → babelqueue-1.4.0}/.gitignore

@@ -12,3 +12,6 @@ venv/
 .coverage
 coverage.xml
 htmlcov/
+
+# CommitBrief local config and cache
+.commitbrief/

{babelqueue-1.3.0 → babelqueue-1.4.0}/CHANGELOG.md

@@ -7,6 +7,23 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 The envelope wire format is versioned separately by `meta.schema_version`
 (currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
 
+## [1.4.0] - 2026-06-13
+
+### Added
+- **Apache Kafka transport** (`babelqueue[kafka]`, `confluent-kafka`) — `KafkaTransport`,
+  selected by the `kafka://` URL scheme (e.g. `kafka://host:9092`; or pass an injected
+  `producer` + `consumer_factory`). Implements [§6 of the broker-bindings
+  contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#apache-kafka): the record
+  **value** is the canonical envelope, projected onto native Kafka record headers (UTF-8 byte
+  strings) — `bq-job` = URN, `bq-trace-id`, `bq-message-id`, plus `bq-schema-version`/
+  `bq-source-lang`/`bq-attempts` — with the record timestamp mirroring `meta.created_at`.
+  Consume is **process-then-commit** (`pop` reserves via `poll` with `enable.auto.commit=false`,
+  `ack` commits the offset); the **`bq-attempts` header is the authoritative attempt counter**
+  (the body's `attempts` is the fallback for non-BabelQueue producers). The projection +
+  reconciliation + publish/pop/ack flow are unit-tested with no broker and no `confluent-kafka`
+  (the kafka import is lazy; the transport talks to injected producer/consumer fakes). The
+  envelope is unchanged (`schema_version: 1`); Apache Kafka is purely additive. Ships as a MINOR.
+
 ## [1.3.0] - 2026-06-13
 
 ### Added

{babelqueue-1.3.0 → babelqueue-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: babelqueue
-Version: 1.3.0
+Version: 1.4.0
 Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
 Project-URL: Homepage, https://babelqueue.com
 Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
@@ -36,6 +36,8 @@ Requires-Dist: pytest>=7; extra == 'dev'
 Requires-Dist: ruff>=0.5; extra == 'dev'
 Provides-Extra: django
 Requires-Dist: django>=4.2; extra == 'django'
+Provides-Extra: kafka
+Requires-Dist: confluent-kafka>=2.3; extra == 'kafka'
 Provides-Extra: pulsar
 Requires-Dist: pulsar-client>=3.4; extra == 'pulsar'
 Provides-Extra: redis

{babelqueue-1.3.0 → babelqueue-1.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "babelqueue"
-version = "1.3.0"
+version = "1.4.0"
 description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -34,6 +34,7 @@ amqp = ["pika>=1.3"]
 sqs = ["boto3>=1.26"]
 azureservicebus = ["azure-servicebus>=7.11", "azure-identity>=1.15"]
 pulsar = ["pulsar-client>=3.4"]
+kafka = ["confluent-kafka>=2.3"]
 celery = ["celery>=5"]
 django = ["django>=4.2"]
 dev = ["pytest>=7", "pytest-cov>=4", "mypy>=1.8", "ruff>=0.5"]

babelqueue-1.4.0/src/babelqueue/kafka_transport.py

@@ -0,0 +1,181 @@
+"""Apache Kafka transport. Requires the ``kafka`` extra:
+
+    pip install "babelqueue[kafka]"
+
+Producing writes the canonical envelope as the record value and projects the contract
+envelope fields onto native Kafka record headers (UTF-8 byte strings): ``bq-job`` = URN,
+``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
+``bq-source-lang`` / ``bq-attempts`` — so a Java/.NET/... peer can route on ``bq-job``
+without parsing the body — with the record timestamp mirroring ``meta.created_at``.
+Consuming is process-then-commit: ``poll`` reserves a record (``enable.auto.commit=false``)
+and ``ack`` commits the offset only after the handler returns (at-least-once). Kafka has no
+native delivery count, so the ``bq-attempts`` header is the authoritative retry counter (the
+body's ``attempts`` is the fallback for non-BabelQueue producers); the runtime owns retry by
+republishing with attempts+1 and dead-letters to ``<queue>.dlq``.
+
+This implements §6 of the broker-bindings contract. The envelope is unchanged
+(``schema_version`` stays 1); Apache Kafka is purely additive.
+
+URL form: ``kafka://host:9092[,host2:9092]``. The default consumer group is ``babelqueue``.
+For a custom client, build the transport directly and pass it via ``BabelQueue(transport=...)``
+or ``KafkaTransport(producer=..., consumer_factory=...)``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from .codec import EnvelopeCodec
+from .transport import ReceivedMessage, Transport
+
+Headers = List[Tuple[str, bytes]]
+
+
+def _brokers_from_url(url: str) -> str:
+    rest = url.split("://", 1)[1] if "://" in url else url
+    return rest.split("/", 1)[0] or "localhost:9092"
+
+
+class KafkaTransport(Transport):
+    def __init__(
+        self,
+        url: str = "kafka://localhost:9092",
+        *,
+        producer: Any = None,
+        consumer_factory: Optional[Callable[[str], Any]] = None,
+        group_id: str = "babelqueue",
+        **client_config: Any,
+    ) -> None:
+        self._brokers = _brokers_from_url(url or "kafka://localhost:9092")
+        self._group_id = group_id
+        self._client_config = client_config
+        self._producer = producer
+        self._consumer_factory = consumer_factory
+        self._consumers: Dict[str, Any] = {}
+
+    # -- helpers ------------------------------------------------------------
+
+    def _producer_(self) -> Any:
+        if self._producer is None:
+            self._producer = self._build_producer()  # pragma: no cover - needs Kafka / network
+        return self._producer
+
+    def _build_producer(self) -> Any:  # pragma: no cover - needs Kafka / network
+        from confluent_kafka import Producer
+
+        return Producer({"bootstrap.servers": self._brokers, **self._client_config})
+
+    def _consumer(self, queue: str) -> Any:
+        consumer = self._consumers.get(queue)
+        if consumer is None:
+            if self._consumer_factory is not None:
+                consumer = self._consumer_factory(queue)
+            else:
+                consumer = self._build_consumer(queue)  # pragma: no cover - needs Kafka / network
+            self._consumers[queue] = consumer
+        return consumer
+
+    def _build_consumer(self, queue: str) -> Any:  # pragma: no cover - needs Kafka / network
+        from confluent_kafka import Consumer
+
+        consumer = Consumer(
+            {
+                "bootstrap.servers": self._brokers,
+                "group.id": self._group_id,
+                "enable.auto.commit": False,
+                "auto.offset.reset": "earliest",
+                **self._client_config,
+            }
+        )
+        consumer.subscribe([queue])
+        return consumer
+
+    @staticmethod
+    def _projection(body: str) -> Headers:
+        """Native Kafka record headers (UTF-8 byte values) — a redundant, routable view of the
+        body: bq-job/bq-trace-id/bq-message-id + bq-schema-version/lang/attempts. §6.3."""
+        env = EnvelopeCodec.decode(body)
+        if not env:
+            return []
+        meta = env.get("meta") or {}
+
+        headers: Headers = []
+
+        def add(key: str, value: Any) -> None:
+            if value is not None and value != "":
+                headers.append((key, str(value).encode("utf-8")))
+
+        add("bq-job", env.get("job"))
+        add("bq-trace-id", env.get("trace_id"))
+        add("bq-message-id", meta.get("id"))
+        if meta.get("schema_version") is not None:
+            headers.append(("bq-schema-version", str(meta["schema_version"]).encode("utf-8")))
+        add("bq-source-lang", meta.get("lang"))
+        headers.append(("bq-attempts", str(int(env.get("attempts", 0) or 0)).encode("utf-8")))
+        return headers
+
+    @staticmethod
+    def _reconcile(body: str, headers: Any) -> str:
+        """Set attempts to the authoritative bq-attempts header (falling back to the body's own
+        attempts when the header is absent/unparseable — a non-BabelQueue producer). §6.5."""
+        env = EnvelopeCodec.decode(body)
+        if not env:
+            return body
+        attempts = int(env.get("attempts", 0) or 0)
+        for key, value in headers or []:
+            if key == "bq-attempts":
+                raw = value.decode("utf-8") if isinstance(value, (bytes, bytearray)) else str(value)
+                try:
+                    attempts = int(raw)
+                except (ValueError, TypeError):
+                    pass
+                break
+        if attempts == int(env.get("attempts", 0) or 0):
+            return body
+        env["attempts"] = attempts
+        return EnvelopeCodec.encode(env)
+
+    @staticmethod
+    def _payload(message: Any) -> str:
+        value = message.value()
+        if isinstance(value, (bytes, bytearray)):
+            return value.decode("utf-8")
+        return str(value) if value is not None else ""
+
+    # -- Transport ----------------------------------------------------------
+
+    def publish(self, queue: str, body: str) -> None:
+        env = EnvelopeCodec.decode(body)
+        meta = env.get("meta") or {}
+        producer = self._producer_()
+        kwargs: Dict[str, Any] = {"value": body.encode("utf-8"), "headers": self._projection(body)}
+        created_at = meta.get("created_at")
+        if created_at:
+            kwargs["timestamp"] = int(created_at)
+        producer.produce(queue, **kwargs)
+        producer.poll(0)
+
+    def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
+        wait = timeout if timeout and timeout > 0 else 1.0
+        message = self._consumer(queue).poll(wait)
+        if message is None or message.error() is not None:
+            return None
+        body = self._reconcile(self._payload(message), message.headers())
+        return ReceivedMessage(body=body, queue=queue, handle=message)
+
+    def ack(self, message: ReceivedMessage) -> None:
+        if message.handle is None:
+            return
+        self._consumer(message.queue).commit(message=message.handle, asynchronous=False)
+
+    def close(self) -> None:  # pragma: no cover - resource cleanup
+        try:
+            if self._producer is not None:
+                self._producer.flush()
+        except Exception:  # noqa: BLE001 - best-effort cleanup
+            pass
+        for consumer in self._consumers.values():
+            try:
+                consumer.close()
+            except Exception:  # noqa: BLE001 - best-effort cleanup
+                pass

{babelqueue-1.3.0 → babelqueue-1.4.0}/src/babelqueue/transport.py

@@ -91,9 +91,13 @@ def make_transport(broker_url: str) -> Transport:
         from .pulsar_transport import PulsarTransport
 
         return PulsarTransport(broker_url)
+    if scheme == "kafka":
+        from .kafka_transport import KafkaTransport
+
+        return KafkaTransport(broker_url)
 
     raise BabelQueueError(
         f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://', "
-        "'amqp://', 'sqs://', 'sb://' or 'pulsar://', or pass your own Transport via "
-        "BabelQueue(transport=...)."
+        "'amqp://', 'sqs://', 'sb://', 'pulsar://' or 'kafka://', or pass your own Transport "
+        "via BabelQueue(transport=...)."
     )

{babelqueue-1.3.0 → babelqueue-1.4.0}/tests/conformance/manifest.json

@@ -148,5 +148,30 @@
         { "name": "native-equals-body",        "body_attempts": 3, "redelivery_count": 3, "expected_attempts": 3 }
       ]
     }
+  },
+  "kafka": {
+    "description": "Apache Kafka binding conformance (broker-bindings.md §6). Every SDK that ships a Kafka transport must satisfy these. The record value is the byte-identical envelope (the 'cases' above); these lock the bq- header projection + the bq-attempts-authoritative reconciliation the binding adds. Per-message values reuse fixtures/order-created.json so the expected projection is deterministic.",
+    "property_projection": {
+      "description": "On produce, the transport MUST project these native Kafka record headers from the envelope, all UTF-8 byte strings (Kafka headers are bytes, so integers are stringified): bq-job = job (the URN), bq-trace-id = trace_id, bq-message-id = meta.id, bq-schema-version = str(meta.schema_version), bq-source-lang = meta.lang, bq-attempts = str(attempts). The record value is the byte-identical envelope and the record timestamp mirrors meta.created_at (Unix ms). Applies to every Kafka-producing SDK.",
+      "envelope_file": "fixtures/order-created.json",
+      "headers": {
+        "bq-job":            "urn:babel:orders:created",
+        "bq-trace-id":       "7b3f9c2a-e41d-4f88-9b2a-1c0d5e6f7a8b",
+        "bq-message-id":     "f1e2d3c4-b5a6-4789-90ab-cdef01234567",
+        "bq-schema-version": "1",
+        "bq-source-lang":    "php",
+        "bq-attempts":       "0"
+      }
+    },
+    "attempts_reconciliation": {
+      "description": "On consume, attempts = the bq-attempts header when present (AUTHORITATIVE — Kafka has no native delivery count), else the body's own attempts (the fallback for a non-BabelQueue producer). This is NOT a max: the header overrides the body even when lower. A null header_attempts means the header is absent. The rule is identical across all five Kafka SDKs.",
+      "cases": [
+        { "name": "header-matches-body",            "body_attempts": 0, "header_attempts": 0,    "expected_attempts": 0 },
+        { "name": "header-present",                 "body_attempts": 0, "header_attempts": 2,    "expected_attempts": 2 },
+        { "name": "header-absent-falls-back",       "body_attempts": 3, "header_attempts": null, "expected_attempts": 3 },
+        { "name": "header-authoritative-overrides", "body_attempts": 5, "header_attempts": 1,    "expected_attempts": 1 },
+        { "name": "header-higher-than-body",        "body_attempts": 1, "header_attempts": 4,    "expected_attempts": 4 }
+      ]
+    }
   }
 }

{babelqueue-1.3.0 → babelqueue-1.4.0}/tests/test_app.py

@@ -118,7 +118,7 @@ class AppTest(unittest.TestCase):
         from babelqueue import BabelQueueError
 
         with self.assertRaises(BabelQueueError):
-            BabelQueue("kafka://localhost:9092")
+            BabelQueue("frobnicate://localhost:9092")
 
 
 if __name__ == "__main__":

babelqueue-1.4.0/tests/test_kafka_conformance.py

@@ -0,0 +1,52 @@
+"""Apache Kafka binding conformance: the vendored manifest's ``kafka`` block locks the §6
+header projection (bq-* UTF-8 byte strings) and the ``attempts`` reconciliation (the
+``bq-attempts`` header is authoritative when present, else the body — NOT a max). No
+confluent-kafka, no broker — pure transport logic against golden values."""
+
+from __future__ import annotations
+
+import json
+import os
+import unittest
+
+from babelqueue import EnvelopeCodec
+from babelqueue.kafka_transport import KafkaTransport
+
+CONFORMANCE = os.path.join(os.path.dirname(__file__), "conformance")
+
+
+class KafkaConformanceTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        with open(os.path.join(CONFORMANCE, "manifest.json"), encoding="utf-8") as fh:
+            cls.kafka = json.load(fh)["kafka"]
+
+    def test_property_projection(self) -> None:
+        projection = self.kafka["property_projection"]
+        with open(os.path.join(CONFORMANCE, projection["envelope_file"]), encoding="utf-8") as fh:
+            body = fh.read()
+
+        got = {key: value.decode("utf-8") for key, value in KafkaTransport._projection(body)}
+        self.assertEqual(got, projection["headers"])
+
+    def test_attempts_reconciliation(self) -> None:
+        for case in self.kafka["attempts_reconciliation"]["cases"]:
+            env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1})
+            env["attempts"] = case["body_attempts"]
+            body = EnvelopeCodec.encode(env)
+
+            headers = []
+            if case["header_attempts"] is not None:
+                headers = [("bq-attempts", str(case["header_attempts"]).encode("utf-8"))]
+
+            out = KafkaTransport._reconcile(body, headers)
+
+            self.assertEqual(
+                EnvelopeCodec.decode(out)["attempts"],
+                case["expected_attempts"],
+                case["name"],
+            )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()

babelqueue-1.4.0/tests/test_kafka_transport.py

@@ -0,0 +1,171 @@
+"""Apache Kafka transport tests.
+
+The header projection and the ``attempts`` reconciliation (authoritative ``bq-attempts``
+header, body fallback) — and the full publish / pop / ack flow — run with no broker and no
+``confluent-kafka`` (the kafka import is lazy and the transport talks to injected
+producer/consumer fakes). Only ``_build_producer``/``_build_consumer`` need the real package.
+"""
+
+from __future__ import annotations
+
+import unittest
+
+from babelqueue import EnvelopeCodec
+from babelqueue.kafka_transport import KafkaTransport
+from babelqueue.transport import ReceivedMessage, make_transport
+
+URN = "urn:babel:orders:created"
+
+
+def body(attempts: int = 0) -> str:
+    env = EnvelopeCodec.decode(
+        EnvelopeCodec.encode(EnvelopeCodec.make(URN, {"order_id": 1042}, queue="orders"))
+    )
+    env["attempts"] = attempts
+    return EnvelopeCodec.encode(env)
+
+
+def header_map(headers):
+    return {k: (v.decode("utf-8") if isinstance(v, (bytes, bytearray)) else v) for k, v in headers}
+
+
+class FakeMessage:
+    def __init__(self, value, headers=None, err=None) -> None:
+        self._value = value
+        self._headers = headers or []
+        self._err = err
+
+    def value(self):
+        return self._value
+
+    def headers(self):
+        return self._headers
+
+    def error(self):
+        return self._err
+
+
+class FakeProducer:
+    def __init__(self) -> None:
+        self.produced = []
+
+    def produce(self, topic, value=None, headers=None, timestamp=None, **kw) -> None:
+        self.produced.append({"topic": topic, "value": value, "headers": headers, "timestamp": timestamp})
+
+    def poll(self, _t):
+        return 0
+
+    def flush(self, *a):
+        return 0
+
+
+class FakeConsumer:
+    def __init__(self, messages=()) -> None:
+        self._messages = list(messages)
+        self.committed = []
+
+    def poll(self, timeout=None):
+        return self._messages.pop(0) if self._messages else None
+
+    def commit(self, message=None, asynchronous=True) -> None:
+        self.committed.append(message)
+
+    def close(self) -> None:
+        pass
+
+
+def transport(producer=None, consumer=None) -> KafkaTransport:
+    producer = producer if producer is not None else FakeProducer()
+    consumer = consumer if consumer is not None else FakeConsumer()
+    return KafkaTransport(producer=producer, consumer_factory=lambda _q: consumer)
+
+
+class KafkaProjectionTest(unittest.TestCase):
+    def test_projection_maps_contract_headers(self):
+        raw = body()
+        env = EnvelopeCodec.decode(raw)
+        headers = header_map(KafkaTransport._projection(raw))
+
+        self.assertEqual(headers["bq-job"], URN)
+        self.assertEqual(headers["bq-trace-id"], env["trace_id"])
+        self.assertEqual(headers["bq-message-id"], env["meta"]["id"])
+        self.assertEqual(headers["bq-schema-version"], str(env["meta"]["schema_version"]))
+        self.assertEqual(headers["bq-source-lang"], env["meta"]["lang"])
+        self.assertEqual(headers["bq-attempts"], "0")
+
+    def test_projection_values_are_bytes(self):
+        for _key, value in KafkaTransport._projection(body(3)):
+            self.assertIsInstance(value, bytes)
+
+    def test_projection_of_garbage_is_empty(self):
+        self.assertEqual(KafkaTransport._projection("not-json"), [])
+
+    def test_reconcile_header_is_authoritative(self):
+        out = KafkaTransport._reconcile(body(0), [("bq-attempts", b"2")])
+        self.assertEqual(EnvelopeCodec.decode(out)["attempts"], 2)
+
+    def test_reconcile_absent_header_falls_back_to_body(self):
+        raw = body(3)
+        self.assertEqual(KafkaTransport._reconcile(raw, []), raw)
+
+    def test_reconcile_garbage_header_falls_back(self):
+        raw = body(3)
+        self.assertEqual(KafkaTransport._reconcile(raw, [("bq-attempts", b"x")]), raw)
+
+
+class KafkaPublishTest(unittest.TestCase):
+    def test_publish_writes_value_headers_and_timestamp(self):
+        raw = body()
+        env = EnvelopeCodec.decode(raw)
+        producer = FakeProducer()
+        transport(producer=producer).publish("orders", raw)
+
+        self.assertEqual(len(producer.produced), 1)
+        rec = producer.produced[0]
+        self.assertEqual(rec["topic"], "orders")
+        self.assertEqual(rec["value"], raw.encode("utf-8"))
+        self.assertEqual(rec["timestamp"], env["meta"]["created_at"])
+        self.assertEqual(header_map(rec["headers"])["bq-job"], URN)
+
+
+class KafkaConsumeTest(unittest.TestCase):
+    def test_pop_reconciles_attempts_and_returns_handle(self):
+        message = FakeMessage(body(0).encode("utf-8"), headers=[("bq-attempts", b"2")])
+        consumer = FakeConsumer([message])
+        tr = transport(consumer=consumer)
+
+        msg = tr.pop("orders")
+
+        self.assertIsNotNone(msg)
+        self.assertIs(msg.handle, message)
+        self.assertEqual(EnvelopeCodec.decode(msg.body)["attempts"], 2)
+
+    def test_pop_empty_returns_none(self):
+        self.assertIsNone(transport(consumer=FakeConsumer([])).pop("orders"))
+
+    def test_pop_error_returns_none(self):
+        message = FakeMessage(body().encode("utf-8"), err="partition eof")
+        self.assertIsNone(transport(consumer=FakeConsumer([message])).pop("orders"))
+
+    def test_ack_commits_the_reserved_message(self):
+        message = FakeMessage(body().encode("utf-8"))
+        consumer = FakeConsumer([message])
+        tr = transport(consumer=consumer)
+
+        tr.ack(tr.pop("orders"))
+
+        self.assertEqual(consumer.committed, [message])
+
+    def test_ack_noop_without_handle(self):
+        transport().ack(ReceivedMessage(body="", queue="orders", handle=None))  # no error
+
+
+class KafkaSchemeTest(unittest.TestCase):
+    def test_make_transport_routes_kafka_scheme_lazily(self):
+        # Construction is lazy (no client built), so it succeeds with no confluent-kafka.
+        tr = make_transport("kafka://localhost:9092")
+        self.assertIsInstance(tr, KafkaTransport)
+
+
+if __name__ == "__main__":
+    unittest.main()