babelqueue 1.2.0__tar.gz → 1.4.0__tar.gz

{babelqueue-1.2.0 → babelqueue-1.4.0}/.gitignore

@@ -12,3 +12,6 @@ venv/
 .coverage
 coverage.xml
 htmlcov/
+
+# CommitBrief local config and cache
+.commitbrief/

{babelqueue-1.2.0 → babelqueue-1.4.0}/CHANGELOG.md

@@ -7,6 +7,41 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 The envelope wire format is versioned separately by `meta.schema_version`
 (currently **1**) — see the contract at [babelqueue.com](https://babelqueue.com).
 
+## [1.4.0] - 2026-06-13
+
+### Added
+- **Apache Kafka transport** (`babelqueue[kafka]`, `confluent-kafka`) — `KafkaTransport`,
+  selected by the `kafka://` URL scheme (e.g. `kafka://host:9092`; or pass an injected
+  `producer` + `consumer_factory`). Implements [§6 of the broker-bindings
+  contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#apache-kafka): the record
+  **value** is the canonical envelope, projected onto native Kafka record headers (UTF-8 byte
+  strings) — `bq-job` = URN, `bq-trace-id`, `bq-message-id`, plus `bq-schema-version`/
+  `bq-source-lang`/`bq-attempts` — with the record timestamp mirroring `meta.created_at`.
+  Consume is **process-then-commit** (`pop` reserves via `poll` with `enable.auto.commit=false`,
+  `ack` commits the offset); the **`bq-attempts` header is the authoritative attempt counter**
+  (the body's `attempts` is the fallback for non-BabelQueue producers). The projection +
+  reconciliation + publish/pop/ack flow are unit-tested with no broker and no `confluent-kafka`
+  (the kafka import is lazy; the transport talks to injected producer/consumer fakes). The
+  envelope is unchanged (`schema_version: 1`); Apache Kafka is purely additive. Ships as a MINOR.
+
+## [1.3.0] - 2026-06-13
+
+### Added
+- **Apache Pulsar transport** (`babelqueue[pulsar]`, `pulsar-client`) — `PulsarTransport`,
+  selected by the `pulsar://` (or `pulsar+ssl://`) URL scheme (e.g.
+  `pulsar://localhost:6650`; or pass a built `client`). Implements [§5 of the broker-bindings
+  contract](https://babelqueue.com/docs/spec/1.x/broker-bindings#apache-pulsar): the canonical
+  envelope is the message payload, projected onto native Pulsar message properties
+  (string→string) — `bq-job` = URN, `bq-trace-id` = `trace_id`, `bq-message-id` = `meta.id`,
+  plus `bq-schema-version`/`bq-source-lang`/`bq-attempts`; receive → `acknowledge`; `attempts`
+  reconciled to `max(bq-attempts, redelivery_count)` (Pulsar's redelivery count is 0-based, so
+  it maps directly with no −1, and the `max` never lowers a higher body count carried by a
+  republish-driven retry). Default `Shared` subscription named `babelqueue`; a client can be
+  injected for tests/DI. The projection + reconciliation + publish/pop/ack flow are unit-tested
+  with no broker and no `pulsar-client` (the pulsar import is lazy and publishing sends raw
+  bytes). The envelope is unchanged (`schema_version: 1`); Apache Pulsar is purely additive.
+  Ships as a MINOR.
+
 ## [1.2.0] - 2026-06-13
 
 ### Added

{babelqueue-1.2.0 → babelqueue-1.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: babelqueue
-Version: 1.2.0
+Version: 1.4.0
 Summary: Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers.
 Project-URL: Homepage, https://babelqueue.com
 Project-URL: Source, https://github.com/BabelQueue/babelqueue-python
@@ -36,6 +36,10 @@ Requires-Dist: pytest>=7; extra == 'dev'
 Requires-Dist: ruff>=0.5; extra == 'dev'
 Provides-Extra: django
 Requires-Dist: django>=4.2; extra == 'django'
+Provides-Extra: kafka
+Requires-Dist: confluent-kafka>=2.3; extra == 'kafka'
+Provides-Extra: pulsar
+Requires-Dist: pulsar-client>=3.4; extra == 'pulsar'
 Provides-Extra: redis
 Requires-Dist: redis>=4; extra == 'redis'
 Provides-Extra: sqs

{babelqueue-1.2.0 → babelqueue-1.4.0}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
 
 [project]
 name = "babelqueue"
-version = "1.2.0"
+version = "1.4.0"
 description = "Polyglot Queues, Simplified — the Python core: the canonical BabelQueue wire-envelope codec, contracts and dead-letter helpers."
 readme = "README.md"
 requires-python = ">=3.9"
@@ -33,6 +33,8 @@ redis = ["redis>=4"]
 amqp = ["pika>=1.3"]
 sqs = ["boto3>=1.26"]
 azureservicebus = ["azure-servicebus>=7.11", "azure-identity>=1.15"]
+pulsar = ["pulsar-client>=3.4"]
+kafka = ["confluent-kafka>=2.3"]
 celery = ["celery>=5"]
 django = ["django>=4.2"]
 dev = ["pytest>=7", "pytest-cov>=4", "mypy>=1.8", "ruff>=0.5"]

babelqueue-1.4.0/src/babelqueue/kafka_transport.py

@@ -0,0 +1,181 @@
+"""Apache Kafka transport. Requires the ``kafka`` extra:
+
+    pip install "babelqueue[kafka]"
+
+Producing writes the canonical envelope as the record value and projects the contract
+envelope fields onto native Kafka record headers (UTF-8 byte strings): ``bq-job`` = URN,
+``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
+``bq-source-lang`` / ``bq-attempts`` — so a Java/.NET/... peer can route on ``bq-job``
+without parsing the body — with the record timestamp mirroring ``meta.created_at``.
+Consuming is process-then-commit: ``poll`` reserves a record (``enable.auto.commit=false``)
+and ``ack`` commits the offset only after the handler returns (at-least-once). Kafka has no
+native delivery count, so the ``bq-attempts`` header is the authoritative retry counter (the
+body's ``attempts`` is the fallback for non-BabelQueue producers); the runtime owns retry by
+republishing with attempts+1 and dead-letters to ``<queue>.dlq``.
+
+This implements §6 of the broker-bindings contract. The envelope is unchanged
+(``schema_version`` stays 1); Apache Kafka is purely additive.
+
+URL form: ``kafka://host:9092[,host2:9092]``. The default consumer group is ``babelqueue``.
+For a custom client, build the transport directly and pass it via ``BabelQueue(transport=...)``
+or ``KafkaTransport(producer=..., consumer_factory=...)``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Callable, Dict, List, Optional, Tuple
+
+from .codec import EnvelopeCodec
+from .transport import ReceivedMessage, Transport
+
+Headers = List[Tuple[str, bytes]]
+
+
+def _brokers_from_url(url: str) -> str:
+    rest = url.split("://", 1)[1] if "://" in url else url
+    return rest.split("/", 1)[0] or "localhost:9092"
+
+
+class KafkaTransport(Transport):
+    def __init__(
+        self,
+        url: str = "kafka://localhost:9092",
+        *,
+        producer: Any = None,
+        consumer_factory: Optional[Callable[[str], Any]] = None,
+        group_id: str = "babelqueue",
+        **client_config: Any,
+    ) -> None:
+        self._brokers = _brokers_from_url(url or "kafka://localhost:9092")
+        self._group_id = group_id
+        self._client_config = client_config
+        self._producer = producer
+        self._consumer_factory = consumer_factory
+        self._consumers: Dict[str, Any] = {}
+
+    # -- helpers ------------------------------------------------------------
+
+    def _producer_(self) -> Any:
+        if self._producer is None:
+            self._producer = self._build_producer()  # pragma: no cover - needs Kafka / network
+        return self._producer
+
+    def _build_producer(self) -> Any:  # pragma: no cover - needs Kafka / network
+        from confluent_kafka import Producer
+
+        return Producer({"bootstrap.servers": self._brokers, **self._client_config})
+
+    def _consumer(self, queue: str) -> Any:
+        consumer = self._consumers.get(queue)
+        if consumer is None:
+            if self._consumer_factory is not None:
+                consumer = self._consumer_factory(queue)
+            else:
+                consumer = self._build_consumer(queue)  # pragma: no cover - needs Kafka / network
+            self._consumers[queue] = consumer
+        return consumer
+
+    def _build_consumer(self, queue: str) -> Any:  # pragma: no cover - needs Kafka / network
+        from confluent_kafka import Consumer
+
+        consumer = Consumer(
+            {
+                "bootstrap.servers": self._brokers,
+                "group.id": self._group_id,
+                "enable.auto.commit": False,
+                "auto.offset.reset": "earliest",
+                **self._client_config,
+            }
+        )
+        consumer.subscribe([queue])
+        return consumer
+
+    @staticmethod
+    def _projection(body: str) -> Headers:
+        """Native Kafka record headers (UTF-8 byte values) — a redundant, routable view of the
+        body: bq-job/bq-trace-id/bq-message-id + bq-schema-version/lang/attempts. §6.3."""
+        env = EnvelopeCodec.decode(body)
+        if not env:
+            return []
+        meta = env.get("meta") or {}
+
+        headers: Headers = []
+
+        def add(key: str, value: Any) -> None:
+            if value is not None and value != "":
+                headers.append((key, str(value).encode("utf-8")))
+
+        add("bq-job", env.get("job"))
+        add("bq-trace-id", env.get("trace_id"))
+        add("bq-message-id", meta.get("id"))
+        if meta.get("schema_version") is not None:
+            headers.append(("bq-schema-version", str(meta["schema_version"]).encode("utf-8")))
+        add("bq-source-lang", meta.get("lang"))
+        headers.append(("bq-attempts", str(int(env.get("attempts", 0) or 0)).encode("utf-8")))
+        return headers
+
+    @staticmethod
+    def _reconcile(body: str, headers: Any) -> str:
+        """Set attempts to the authoritative bq-attempts header (falling back to the body's own
+        attempts when the header is absent/unparseable — a non-BabelQueue producer). §6.5."""
+        env = EnvelopeCodec.decode(body)
+        if not env:
+            return body
+        attempts = int(env.get("attempts", 0) or 0)
+        for key, value in headers or []:
+            if key == "bq-attempts":
+                raw = value.decode("utf-8") if isinstance(value, (bytes, bytearray)) else str(value)
+                try:
+                    attempts = int(raw)
+                except (ValueError, TypeError):
+                    pass
+                break
+        if attempts == int(env.get("attempts", 0) or 0):
+            return body
+        env["attempts"] = attempts
+        return EnvelopeCodec.encode(env)
+
+    @staticmethod
+    def _payload(message: Any) -> str:
+        value = message.value()
+        if isinstance(value, (bytes, bytearray)):
+            return value.decode("utf-8")
+        return str(value) if value is not None else ""
+
+    # -- Transport ----------------------------------------------------------
+
+    def publish(self, queue: str, body: str) -> None:
+        env = EnvelopeCodec.decode(body)
+        meta = env.get("meta") or {}
+        producer = self._producer_()
+        kwargs: Dict[str, Any] = {"value": body.encode("utf-8"), "headers": self._projection(body)}
+        created_at = meta.get("created_at")
+        if created_at:
+            kwargs["timestamp"] = int(created_at)
+        producer.produce(queue, **kwargs)
+        producer.poll(0)
+
+    def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
+        wait = timeout if timeout and timeout > 0 else 1.0
+        message = self._consumer(queue).poll(wait)
+        if message is None or message.error() is not None:
+            return None
+        body = self._reconcile(self._payload(message), message.headers())
+        return ReceivedMessage(body=body, queue=queue, handle=message)
+
+    def ack(self, message: ReceivedMessage) -> None:
+        if message.handle is None:
+            return
+        self._consumer(message.queue).commit(message=message.handle, asynchronous=False)
+
+    def close(self) -> None:  # pragma: no cover - resource cleanup
+        try:
+            if self._producer is not None:
+                self._producer.flush()
+        except Exception:  # noqa: BLE001 - best-effort cleanup
+            pass
+        for consumer in self._consumers.values():
+            try:
+                consumer.close()
+            except Exception:  # noqa: BLE001 - best-effort cleanup
+                pass

babelqueue-1.4.0/src/babelqueue/pulsar_transport.py

@@ -0,0 +1,202 @@
+"""Apache Pulsar transport. Requires the ``pulsar`` extra:
+
+    pip install "babelqueue[pulsar]"
+
+Producing sends the canonical envelope as the message payload and projects the contract
+envelope fields onto native Pulsar message properties (string->string): ``bq-job`` = URN,
+``bq-trace-id`` = trace_id, ``bq-message-id`` = meta.id, plus ``bq-schema-version`` /
+``bq-source-lang`` / ``bq-attempts`` — so a Java/.NET/... peer can route on ``bq-job``
+without parsing the body. Consuming receives one message at a time
+(``receive`` -> process -> ``acknowledge``); the authoritative attempt count is the
+envelope's ``bq-attempts`` (the body), cross-checked against the broker's native
+``redelivery_count`` (0-based) and reconciled onto the envelope as
+``attempts = max(body, redelivery_count)`` — no -1, because Pulsar's redelivery count is
+0-based (0 on first delivery). A message left un-acked is redelivered (at-least-once).
+
+This implements §5 of the broker-bindings contract. The envelope is unchanged
+(``schema_version`` stays 1); Apache Pulsar is purely additive.
+
+URL form: ``pulsar://host:6650`` (or ``pulsar+ssl://``). The default subscription name is
+``babelqueue`` over a ``Shared`` subscription. For a custom client/subscription, build the
+transport directly and pass it via ``BabelQueue(transport=...)`` or
+``PulsarTransport(client=...)``.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, Optional
+
+from .codec import EnvelopeCodec
+from .transport import ReceivedMessage, Transport
+
+
+class PulsarTransport(Transport):
+    def __init__(
+        self,
+        url: str = "pulsar://localhost:6650",
+        *,
+        client: Any = None,
+        subscription: str = "babelqueue",
+        consumer_type: Any = None,
+        topic_prefix: str = "",
+        receive_timeout_millis: int = 1000,
+        **subscribe_options: Any,
+    ) -> None:
+        self._url = url or "pulsar://localhost:6650"
+        self._subscription = subscription
+        self._consumer_type = consumer_type
+        self._topic_prefix = topic_prefix
+        self._receive_timeout_millis = receive_timeout_millis
+        self._subscribe_options = subscribe_options
+        self._producers: Dict[str, Any] = {}
+        self._consumers: Dict[str, Any] = {}
+
+        if client is not None:
+            self._client = client
+            return
+        self._client = self._build_client()  # pragma: no cover - needs Pulsar / network
+
+    def _build_client(self) -> Any:  # pragma: no cover - needs Pulsar / network
+        try:
+            import pulsar
+        except ImportError as exc:
+            raise ImportError(
+                "PulsarTransport requires the 'pulsar-client' package. Install with "
+                'pip install "babelqueue[pulsar]".'
+            ) from exc
+        return pulsar.Client(self._url)
+
+    # -- helpers ------------------------------------------------------------
+
+    def _topic(self, queue: str) -> str:
+        return f"{self._topic_prefix}{queue}" if self._topic_prefix else queue
+
+    def _producer(self, queue: str) -> Any:
+        producer = self._producers.get(queue)
+        if producer is None:
+            producer = self._client.create_producer(self._topic(queue))
+            self._producers[queue] = producer
+        return producer
+
+    def _consumer(self, queue: str) -> Any:
+        consumer = self._consumers.get(queue)
+        if consumer is None:
+            consumer = self._client.subscribe(
+                self._topic(queue), self._subscription, **self._subscribe_kwargs()
+            )
+            self._consumers[queue] = consumer
+        return consumer
+
+    def _subscribe_kwargs(self) -> Dict[str, Any]:
+        kwargs = dict(self._subscribe_options)
+        consumer_type = self._consumer_type
+        if consumer_type is None:
+            consumer_type = self._default_shared_type()
+        if consumer_type is not None:
+            kwargs.setdefault("consumer_type", consumer_type)
+        return kwargs
+
+    @staticmethod
+    def _default_shared_type() -> Any:
+        try:
+            import pulsar
+        except ImportError:  # pragma: no cover - exercised only with pulsar absent
+            return None
+        return pulsar.ConsumerType.Shared  # pragma: no cover - needs Pulsar
+
+    @staticmethod
+    def _projection(body: str) -> Dict[str, str]:
+        """Native Pulsar message properties (string->string) — a redundant, routable view of
+        the body: bq-job/bq-trace-id/bq-message-id + bq-schema-version/lang/attempts. §5.2."""
+        env = EnvelopeCodec.decode(body)
+        if not env:
+            return {}
+        meta = env.get("meta") or {}
+
+        props: Dict[str, str] = {}
+        if env.get("job"):
+            props["bq-job"] = str(env["job"])
+        if env.get("trace_id"):
+            props["bq-trace-id"] = str(env["trace_id"])
+        if meta.get("id"):
+            props["bq-message-id"] = str(meta["id"])
+        if meta.get("schema_version") is not None:
+            props["bq-schema-version"] = str(meta["schema_version"])
+        if meta.get("lang"):
+            props["bq-source-lang"] = str(meta["lang"])
+        props["bq-attempts"] = str(int(env.get("attempts", 0) or 0))
+        return props
+
+    @staticmethod
+    def _reconcile(body: str, redelivery_count: Any) -> str:
+        """Set attempts to max(current, redelivery_count). Pulsar's redelivery count is
+        0-based (0 on first delivery), so it maps directly to attempts with no -1. The
+        runtime retries by republishing with attempts+1 in the body (redelivery count back to
+        0), so a republished message must not have its higher body count lowered."""
+        try:
+            rc = int(redelivery_count)
+        except (ValueError, TypeError):
+            return body
+        if rc <= 0:
+            return body
+        env = EnvelopeCodec.decode(body)
+        if not env or rc <= int(env.get("attempts", 0) or 0):
+            return body
+        env["attempts"] = rc
+        return EnvelopeCodec.encode(env)
+
+    @staticmethod
+    def _redelivery_count(message: Any) -> int:
+        getter = getattr(message, "redelivery_count", None)
+        value = getter() if callable(getter) else getter
+        if value is None:
+            return 0
+        try:
+            return int(value)
+        except (TypeError, ValueError):
+            return 0
+
+    @staticmethod
+    def _payload(message: Any) -> str:
+        getter = getattr(message, "data", None)
+        data = getter() if callable(getter) else getter
+        if isinstance(data, bytes):
+            return data.decode("utf-8")
+        return str(data) if data is not None else ""
+
+    # -- Transport ----------------------------------------------------------
+
+    def publish(self, queue: str, body: str) -> None:
+        self._producer(queue).send(body.encode("utf-8"), properties=self._projection(body))
+
+    def pop(self, queue: str, timeout: float = 1.0) -> Optional[ReceivedMessage]:
+        wait = self._receive_timeout_millis
+        if timeout and timeout > 0:
+            wait = int(timeout * 1000)
+        consumer = self._consumer(queue)
+        try:
+            message = consumer.receive(timeout_millis=wait)
+        except Exception as exc:  # noqa: BLE001 - Pulsar raises pulsar.Timeout on no message
+            if type(exc).__name__ == "Timeout":
+                return None
+            raise
+        if message is None:  # pragma: no cover - defensive (real client raises instead)
+            return None
+        body = self._reconcile(self._payload(message), self._redelivery_count(message))
+        return ReceivedMessage(body=body, queue=queue, handle=message)
+
+    def ack(self, message: ReceivedMessage) -> None:
+        if message.handle is None:
+            return
+        self._consumer(message.queue).acknowledge(message.handle)
+
+    def close(self) -> None:  # pragma: no cover - resource cleanup
+        for resource in (*self._producers.values(), *self._consumers.values()):
+            try:
+                resource.close()
+            except Exception:  # noqa: BLE001 - best-effort cleanup
+                pass
+        try:
+            self._client.close()
+        except Exception:  # noqa: BLE001 - best-effort cleanup
+            pass

{babelqueue-1.2.0 → babelqueue-1.4.0}/src/babelqueue/transport.py

@@ -87,8 +87,17 @@ def make_transport(broker_url: str) -> Transport:
         from .asb_transport import AsbTransport
 
         return AsbTransport(broker_url)
+    if scheme in ("pulsar", "pulsar+ssl"):
+        from .pulsar_transport import PulsarTransport
+
+        return PulsarTransport(broker_url)
+    if scheme == "kafka":
+        from .kafka_transport import KafkaTransport
+
+        return KafkaTransport(broker_url)
 
     raise BabelQueueError(
         f"Unsupported broker scheme {scheme!r}. Use 'memory://', 'redis://', "
-        "'amqp://', 'sqs://' or 'sb://', or pass your own Transport via BabelQueue(transport=...)."
+        "'amqp://', 'sqs://', 'sb://', 'pulsar://' or 'kafka://', or pass your own Transport "
+        "via BabelQueue(transport=...)."
     )

{babelqueue-1.2.0 → babelqueue-1.4.0}/tests/conformance/manifest.json

@@ -122,5 +122,56 @@
         { "name": "zero-count-keeps-body",     "body_attempts": 3, "delivery_count": 0, "expected_attempts": 3 }
       ]
     }
+  },
+  "pulsar": {
+    "description": "Apache Pulsar binding conformance (broker-bindings.md §5). Every SDK that ships a Pulsar transport must satisfy these. The envelope body stays byte-identical (the 'cases' above); these lock the native projection + reconciliation the binding adds. Per-message values reuse fixtures/order-created.json so the expected projection is deterministic.",
+    "property_projection": {
+      "description": "On produce, the transport MUST project these native Pulsar message properties from the envelope, all string->string (Pulsar properties are string-typed, so numbers are stringified): bq-job = job (the URN), bq-trace-id = trace_id, bq-message-id = meta.id, bq-schema-version = str(meta.schema_version), bq-source-lang = meta.lang, bq-attempts = str(attempts). The payload is the byte-identical envelope; the native publishTime mirrors meta.created_at (broker-set, body authoritative). Applies to every Pulsar-producing SDK.",
+      "envelope_file": "fixtures/order-created.json",
+      "properties": {
+        "bq-job":            "urn:babel:orders:created",
+        "bq-trace-id":       "7b3f9c2a-e41d-4f88-9b2a-1c0d5e6f7a8b",
+        "bq-message-id":     "f1e2d3c4-b5a6-4789-90ab-cdef01234567",
+        "bq-schema-version": "1",
+        "bq-source-lang":    "php",
+        "bq-attempts":       "0"
+      }
+    },
+    "attempts_reconciliation": {
+      "description": "On consume, attempts = max(body.attempts, RedeliveryCount): Pulsar's RedeliveryCount is 0-based (0 on first delivery) so it maps directly with NO -1, a runtime-incremented body count is never lowered, and RedeliveryCount 0 leaves the body's own count untouched (the runtime retries by republishing with attempts+1, which resets the broker's redelivery count to 0). The rule is identical across the native-consumer SDKs (.NET/Java/Node) and the Transport+App SDKs (Python/Go).",
+      "cases": [
+        { "name": "first-delivery",            "body_attempts": 0, "redelivery_count": 0, "expected_attempts": 0 },
+        { "name": "third-delivery",            "body_attempts": 0, "redelivery_count": 2, "expected_attempts": 2 },
+        { "name": "native-exceeds-body",       "body_attempts": 2, "redelivery_count": 5, "expected_attempts": 5 },
+        { "name": "never-lower-runtime",       "body_attempts": 5, "redelivery_count": 1, "expected_attempts": 5 },
+        { "name": "first-delivery-keeps-body", "body_attempts": 4, "redelivery_count": 0, "expected_attempts": 4 },
+        { "name": "native-equals-body",        "body_attempts": 3, "redelivery_count": 3, "expected_attempts": 3 }
+      ]
+    }
+  },
+  "kafka": {
+    "description": "Apache Kafka binding conformance (broker-bindings.md §6). Every SDK that ships a Kafka transport must satisfy these. The record value is the byte-identical envelope (the 'cases' above); these lock the bq- header projection + the bq-attempts-authoritative reconciliation the binding adds. Per-message values reuse fixtures/order-created.json so the expected projection is deterministic.",
+    "property_projection": {
+      "description": "On produce, the transport MUST project these native Kafka record headers from the envelope, all UTF-8 byte strings (Kafka headers are bytes, so integers are stringified): bq-job = job (the URN), bq-trace-id = trace_id, bq-message-id = meta.id, bq-schema-version = str(meta.schema_version), bq-source-lang = meta.lang, bq-attempts = str(attempts). The record value is the byte-identical envelope and the record timestamp mirrors meta.created_at (Unix ms). Applies to every Kafka-producing SDK.",
+      "envelope_file": "fixtures/order-created.json",
+      "headers": {
+        "bq-job":            "urn:babel:orders:created",
+        "bq-trace-id":       "7b3f9c2a-e41d-4f88-9b2a-1c0d5e6f7a8b",
+        "bq-message-id":     "f1e2d3c4-b5a6-4789-90ab-cdef01234567",
+        "bq-schema-version": "1",
+        "bq-source-lang":    "php",
+        "bq-attempts":       "0"
+      }
+    },
+    "attempts_reconciliation": {
+      "description": "On consume, attempts = the bq-attempts header when present (AUTHORITATIVE — Kafka has no native delivery count), else the body's own attempts (the fallback for a non-BabelQueue producer). This is NOT a max: the header overrides the body even when lower. A null header_attempts means the header is absent. The rule is identical across all five Kafka SDKs.",
+      "cases": [
+        { "name": "header-matches-body",            "body_attempts": 0, "header_attempts": 0,    "expected_attempts": 0 },
+        { "name": "header-present",                 "body_attempts": 0, "header_attempts": 2,    "expected_attempts": 2 },
+        { "name": "header-absent-falls-back",       "body_attempts": 3, "header_attempts": null, "expected_attempts": 3 },
+        { "name": "header-authoritative-overrides", "body_attempts": 5, "header_attempts": 1,    "expected_attempts": 1 },
+        { "name": "header-higher-than-body",        "body_attempts": 1, "header_attempts": 4,    "expected_attempts": 4 }
+      ]
+    }
   }
 }

{babelqueue-1.2.0 → babelqueue-1.4.0}/tests/test_app.py

@@ -118,7 +118,7 @@ class AppTest(unittest.TestCase):
         from babelqueue import BabelQueueError
 
         with self.assertRaises(BabelQueueError):
-            BabelQueue("kafka://localhost:9092")
+            BabelQueue("frobnicate://localhost:9092")
 
 
 if __name__ == "__main__":

babelqueue-1.4.0/tests/test_kafka_conformance.py

@@ -0,0 +1,52 @@
+"""Apache Kafka binding conformance: the vendored manifest's ``kafka`` block locks the §6
+header projection (bq-* UTF-8 byte strings) and the ``attempts`` reconciliation (the
+``bq-attempts`` header is authoritative when present, else the body — NOT a max). No
+confluent-kafka, no broker — pure transport logic against golden values."""
+
+from __future__ import annotations
+
+import json
+import os
+import unittest
+
+from babelqueue import EnvelopeCodec
+from babelqueue.kafka_transport import KafkaTransport
+
+CONFORMANCE = os.path.join(os.path.dirname(__file__), "conformance")
+
+
+class KafkaConformanceTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        with open(os.path.join(CONFORMANCE, "manifest.json"), encoding="utf-8") as fh:
+            cls.kafka = json.load(fh)["kafka"]
+
+    def test_property_projection(self) -> None:
+        projection = self.kafka["property_projection"]
+        with open(os.path.join(CONFORMANCE, projection["envelope_file"]), encoding="utf-8") as fh:
+            body = fh.read()
+
+        got = {key: value.decode("utf-8") for key, value in KafkaTransport._projection(body)}
+        self.assertEqual(got, projection["headers"])
+
+    def test_attempts_reconciliation(self) -> None:
+        for case in self.kafka["attempts_reconciliation"]["cases"]:
+            env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1})
+            env["attempts"] = case["body_attempts"]
+            body = EnvelopeCodec.encode(env)
+
+            headers = []
+            if case["header_attempts"] is not None:
+                headers = [("bq-attempts", str(case["header_attempts"]).encode("utf-8"))]
+
+            out = KafkaTransport._reconcile(body, headers)
+
+            self.assertEqual(
+                EnvelopeCodec.decode(out)["attempts"],
+                case["expected_attempts"],
+                case["name"],
+            )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()

babelqueue-1.4.0/tests/test_kafka_transport.py

@@ -0,0 +1,171 @@
+"""Apache Kafka transport tests.
+
+The header projection and the ``attempts`` reconciliation (authoritative ``bq-attempts``
+header, body fallback) — and the full publish / pop / ack flow — run with no broker and no
+``confluent-kafka`` (the kafka import is lazy and the transport talks to injected
+producer/consumer fakes). Only ``_build_producer``/``_build_consumer`` need the real package.
+"""
+
+from __future__ import annotations
+
+import unittest
+
+from babelqueue import EnvelopeCodec
+from babelqueue.kafka_transport import KafkaTransport
+from babelqueue.transport import ReceivedMessage, make_transport
+
+URN = "urn:babel:orders:created"
+
+
+def body(attempts: int = 0) -> str:
+    env = EnvelopeCodec.decode(
+        EnvelopeCodec.encode(EnvelopeCodec.make(URN, {"order_id": 1042}, queue="orders"))
+    )
+    env["attempts"] = attempts
+    return EnvelopeCodec.encode(env)
+
+
+def header_map(headers):
+    return {k: (v.decode("utf-8") if isinstance(v, (bytes, bytearray)) else v) for k, v in headers}
+
+
+class FakeMessage:
+    def __init__(self, value, headers=None, err=None) -> None:
+        self._value = value
+        self._headers = headers or []
+        self._err = err
+
+    def value(self):
+        return self._value
+
+    def headers(self):
+        return self._headers
+
+    def error(self):
+        return self._err
+
+
+class FakeProducer:
+    def __init__(self) -> None:
+        self.produced = []
+
+    def produce(self, topic, value=None, headers=None, timestamp=None, **kw) -> None:
+        self.produced.append({"topic": topic, "value": value, "headers": headers, "timestamp": timestamp})
+
+    def poll(self, _t):
+        return 0
+
+    def flush(self, *a):
+        return 0
+
+
+class FakeConsumer:
+    def __init__(self, messages=()) -> None:
+        self._messages = list(messages)
+        self.committed = []
+
+    def poll(self, timeout=None):
+        return self._messages.pop(0) if self._messages else None
+
+    def commit(self, message=None, asynchronous=True) -> None:
+        self.committed.append(message)
+
+    def close(self) -> None:
+        pass
+
+
+def transport(producer=None, consumer=None) -> KafkaTransport:
+    producer = producer if producer is not None else FakeProducer()
+    consumer = consumer if consumer is not None else FakeConsumer()
+    return KafkaTransport(producer=producer, consumer_factory=lambda _q: consumer)
+
+
+class KafkaProjectionTest(unittest.TestCase):
+    def test_projection_maps_contract_headers(self):
+        raw = body()
+        env = EnvelopeCodec.decode(raw)
+        headers = header_map(KafkaTransport._projection(raw))
+
+        self.assertEqual(headers["bq-job"], URN)
+        self.assertEqual(headers["bq-trace-id"], env["trace_id"])
+        self.assertEqual(headers["bq-message-id"], env["meta"]["id"])
+        self.assertEqual(headers["bq-schema-version"], str(env["meta"]["schema_version"]))
+        self.assertEqual(headers["bq-source-lang"], env["meta"]["lang"])
+        self.assertEqual(headers["bq-attempts"], "0")
+
+    def test_projection_values_are_bytes(self):
+        for _key, value in KafkaTransport._projection(body(3)):
+            self.assertIsInstance(value, bytes)
+
+    def test_projection_of_garbage_is_empty(self):
+        self.assertEqual(KafkaTransport._projection("not-json"), [])
+
+    def test_reconcile_header_is_authoritative(self):
+        out = KafkaTransport._reconcile(body(0), [("bq-attempts", b"2")])
+        self.assertEqual(EnvelopeCodec.decode(out)["attempts"], 2)
+
+    def test_reconcile_absent_header_falls_back_to_body(self):
+        raw = body(3)
+        self.assertEqual(KafkaTransport._reconcile(raw, []), raw)
+
+    def test_reconcile_garbage_header_falls_back(self):
+        raw = body(3)
+        self.assertEqual(KafkaTransport._reconcile(raw, [("bq-attempts", b"x")]), raw)
+
+
+class KafkaPublishTest(unittest.TestCase):
+    def test_publish_writes_value_headers_and_timestamp(self):
+        raw = body()
+        env = EnvelopeCodec.decode(raw)
+        producer = FakeProducer()
+        transport(producer=producer).publish("orders", raw)
+
+        self.assertEqual(len(producer.produced), 1)
+        rec = producer.produced[0]
+        self.assertEqual(rec["topic"], "orders")
+        self.assertEqual(rec["value"], raw.encode("utf-8"))
+        self.assertEqual(rec["timestamp"], env["meta"]["created_at"])
+        self.assertEqual(header_map(rec["headers"])["bq-job"], URN)
+
+
+class KafkaConsumeTest(unittest.TestCase):
+    def test_pop_reconciles_attempts_and_returns_handle(self):
+        message = FakeMessage(body(0).encode("utf-8"), headers=[("bq-attempts", b"2")])
+        consumer = FakeConsumer([message])
+        tr = transport(consumer=consumer)
+
+        msg = tr.pop("orders")
+
+        self.assertIsNotNone(msg)
+        self.assertIs(msg.handle, message)
+        self.assertEqual(EnvelopeCodec.decode(msg.body)["attempts"], 2)
+
+    def test_pop_empty_returns_none(self):
+        self.assertIsNone(transport(consumer=FakeConsumer([])).pop("orders"))
+
+    def test_pop_error_returns_none(self):
+        message = FakeMessage(body().encode("utf-8"), err="partition eof")
+        self.assertIsNone(transport(consumer=FakeConsumer([message])).pop("orders"))
+
+    def test_ack_commits_the_reserved_message(self):
+        message = FakeMessage(body().encode("utf-8"))
+        consumer = FakeConsumer([message])
+        tr = transport(consumer=consumer)
+
+        tr.ack(tr.pop("orders"))
+
+        self.assertEqual(consumer.committed, [message])
+
+    def test_ack_noop_without_handle(self):
+        transport().ack(ReceivedMessage(body="", queue="orders", handle=None))  # no error
+
+
+class KafkaSchemeTest(unittest.TestCase):
+    def test_make_transport_routes_kafka_scheme_lazily(self):
+        # Construction is lazy (no client built), so it succeeds with no confluent-kafka.
+        tr = make_transport("kafka://localhost:9092")
+        self.assertIsInstance(tr, KafkaTransport)
+
+
+if __name__ == "__main__":
+    unittest.main()

babelqueue-1.4.0/tests/test_pulsar_conformance.py

@@ -0,0 +1,48 @@
+"""Apache Pulsar binding conformance: the vendored manifest's ``pulsar`` block locks the §5
+property projection (bq-* string->string) and the ``attempts = max(body, redelivery_count)``
+reconciliation (no -1, Pulsar's redelivery count is 0-based). No pulsar-client, no broker —
+pure transport logic against golden values."""
+
+from __future__ import annotations
+
+import json
+import os
+import unittest
+
+from babelqueue import EnvelopeCodec
+from babelqueue.pulsar_transport import PulsarTransport
+
+CONFORMANCE = os.path.join(os.path.dirname(__file__), "conformance")
+
+
+class PulsarConformanceTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls) -> None:
+        with open(os.path.join(CONFORMANCE, "manifest.json"), encoding="utf-8") as fh:
+            cls.pulsar = json.load(fh)["pulsar"]
+
+    def test_property_projection(self) -> None:
+        projection = self.pulsar["property_projection"]
+        with open(os.path.join(CONFORMANCE, projection["envelope_file"]), encoding="utf-8") as fh:
+            body = fh.read()
+
+        got = PulsarTransport._projection(body)
+        self.assertEqual(got, projection["properties"])
+
+    def test_attempts_reconciliation(self) -> None:
+        for case in self.pulsar["attempts_reconciliation"]["cases"]:
+            env = EnvelopeCodec.make("urn:babel:orders:created", {"x": 1})
+            env["attempts"] = case["body_attempts"]
+            body = EnvelopeCodec.encode(env)
+
+            out = PulsarTransport._reconcile(body, case["redelivery_count"])
+
+            self.assertEqual(
+                EnvelopeCodec.decode(out)["attempts"],
+                case["expected_attempts"],
+                case["name"],
+            )
+
+
+if __name__ == "__main__":  # pragma: no cover
+    unittest.main()

babelqueue-1.4.0/tests/test_pulsar_transport.py

@@ -0,0 +1,215 @@
+"""Apache Pulsar transport tests.
+
+The property projection and the ``attempts = max(body, redelivery_count)`` reconciliation —
+and the full publish / pop / ack flow — run with no broker and no ``pulsar-client`` (the
+transport's pulsar import is lazy and the publish path sends raw bytes, so a duck-typed fake
+client covers everything). Only ``_build_client`` needs the real package.
+"""
+
+from __future__ import annotations
+
+import unittest
+
+from babelqueue import EnvelopeCodec
+from babelqueue.pulsar_transport import PulsarTransport
+from babelqueue.transport import ReceivedMessage, make_transport
+
+try:
+    import pulsar  # noqa: F401
+
+    HAVE_PULSAR = True
+except ImportError:
+    HAVE_PULSAR = False
+
+URN = "urn:babel:orders:created"
+
+
+def body(attempts: int = 0) -> str:
+    env = EnvelopeCodec.decode(
+        EnvelopeCodec.encode(EnvelopeCodec.make(URN, {"order_id": 1042}, queue="orders"))
+    )
+    env["attempts"] = attempts
+    return EnvelopeCodec.encode(env)
+
+
+class Timeout(Exception):
+    """Duck-typed stand-in for ``pulsar.Timeout`` (matched by class name)."""
+
+
+class FakeMessage:
+    """Duck-typed pulsar Message: data()/redelivery_count() are callables."""
+
+    def __init__(self, raw: str, redelivery_count: int = 0) -> None:
+        self._raw = raw
+        self._rc = redelivery_count
+
+    def data(self) -> bytes:
+        return self._raw.encode("utf-8")
+
+    def redelivery_count(self) -> int:
+        return self._rc
+
+
+class FakeProducer:
+    def __init__(self) -> None:
+        self.sent: list = []
+
+    def send(self, content, properties=None) -> None:
+        self.sent.append((content, properties))
+
+
+class FakeConsumer:
+    def __init__(self, messages=()) -> None:
+        self._messages = list(messages)
+        self.acked: list = []
+        self.subscribe_kwargs: dict = {}
+
+    def receive(self, timeout_millis=None):
+        if not self._messages:
+            raise Timeout()
+        return self._messages.pop(0)
+
+    def acknowledge(self, message) -> None:
+        self.acked.append(message)
+
+
+class FakeClient:
+    def __init__(self, producer=None, consumer=None) -> None:
+        self.producer = producer if producer is not None else FakeProducer()
+        self.consumer = consumer if consumer is not None else FakeConsumer()
+        self.subscribed: list = []
+
+    def create_producer(self, topic):
+        return self.producer
+
+    def subscribe(self, topic, subscription, **kwargs):
+        self.subscribed.append((topic, subscription, kwargs))
+        self.consumer.subscribe_kwargs = kwargs
+        return self.consumer
+
+
+class PulsarProjectionTest(unittest.TestCase):
+    def test_projection_maps_contract_properties(self):
+        raw = body()
+        env = EnvelopeCodec.decode(raw)
+        props = PulsarTransport._projection(raw)
+
+        self.assertEqual(props["bq-job"], URN)
+        self.assertEqual(props["bq-trace-id"], env["trace_id"])
+        self.assertEqual(props["bq-message-id"], env["meta"]["id"])
+        self.assertEqual(props["bq-schema-version"], str(env["meta"]["schema_version"]))
+        self.assertEqual(props["bq-source-lang"], env["meta"]["lang"])
+        self.assertEqual(props["bq-attempts"], "0")
+
+    def test_projection_values_are_all_strings(self):
+        props = PulsarTransport._projection(body(3))
+        self.assertTrue(all(isinstance(v, str) for v in props.values()))
+        self.assertEqual(props["bq-attempts"], "3")
+
+    def test_projection_of_garbage_is_empty(self):
+        self.assertEqual(PulsarTransport._projection("not-json"), {})
+
+    def test_reconcile_attempts_is_redelivery_count(self):
+        out = PulsarTransport._reconcile(body(0), 2)
+        self.assertEqual(EnvelopeCodec.decode(out)["attempts"], 2)
+
+    def test_reconcile_first_delivery_is_zero(self):
+        out = PulsarTransport._reconcile(body(0), 0)
+        self.assertEqual(EnvelopeCodec.decode(out)["attempts"], 0)
+
+    def test_reconcile_ignores_garbage_count(self):
+        raw = body(5)
+        self.assertEqual(PulsarTransport._reconcile(raw, "not-a-number"), raw)
+
+    def test_reconcile_never_lowers_runtime_count(self):
+        # The runtime retries by republishing with attempts+1 (redelivery count back to 0),
+        # so a higher body count must win over a lower native one.
+        raw = body(5)
+        self.assertEqual(int(EnvelopeCodec.decode(PulsarTransport._reconcile(raw, 1))["attempts"]), 5)
+
+
+class PulsarPublishTest(unittest.TestCase):
+    def test_publish_sends_payload_with_property_projection(self):
+        raw = body()
+        env = EnvelopeCodec.decode(raw)
+        producer = FakeProducer()
+        tr = PulsarTransport(client=FakeClient(producer=producer))
+
+        tr.publish("orders", raw)
+
+        self.assertEqual(len(producer.sent), 1)
+        content, props = producer.sent[0]
+        self.assertEqual(content, raw.encode("utf-8"))
+        self.assertEqual(props["bq-job"], URN)
+        self.assertEqual(props["bq-message-id"], env["meta"]["id"])
+        self.assertEqual(EnvelopeCodec.urn(EnvelopeCodec.decode(content.decode("utf-8"))), URN)
+
+
+class PulsarConsumeTest(unittest.TestCase):
+    def test_pop_reconciles_attempts_and_returns_handle(self):
+        message = FakeMessage(body(0), redelivery_count=2)
+        tr = PulsarTransport(client=FakeClient(consumer=FakeConsumer([message])))
+
+        msg = tr.pop("orders")
+
+        self.assertIsNotNone(msg)
+        self.assertEqual(msg.queue, "orders")
+        self.assertIs(msg.handle, message)
+        self.assertEqual(EnvelopeCodec.decode(msg.body)["attempts"], 2)
+
+    def test_pop_empty_returns_none_on_timeout(self):
+        tr = PulsarTransport(client=FakeClient(consumer=FakeConsumer([])))
+        self.assertIsNone(tr.pop("orders"))
+
+    def test_pop_passes_timeout_to_receive(self):
+        consumer = FakeConsumer([FakeMessage(body(), 0)])
+        captured = {}
+        original = consumer.receive
+
+        def spy(timeout_millis=None):
+            captured["timeout_millis"] = timeout_millis
+            return original(timeout_millis=timeout_millis)
+
+        consumer.receive = spy
+        tr = PulsarTransport(client=FakeClient(consumer=consumer))
+        tr.pop("orders", timeout=2.0)
+        self.assertEqual(captured["timeout_millis"], 2000)
+
+    def test_ack_acknowledges_the_reserved_message(self):
+        message = FakeMessage(body(), 0)
+        consumer = FakeConsumer([message])
+        tr = PulsarTransport(client=FakeClient(consumer=consumer))
+
+        tr.ack(tr.pop("orders"))
+
+        self.assertEqual(consumer.acked, [message])
+
+    def test_ack_noop_without_handle(self):
+        tr = PulsarTransport(client=FakeClient())
+        tr.ack(ReceivedMessage(body="", queue="orders", handle=None))  # no error
+
+    def test_explicit_consumer_type_is_forwarded_to_subscribe(self):
+        client = FakeClient()
+        tr = PulsarTransport(client=client, consumer_type="shared-sentinel")
+        tr.pop("orders")
+        self.assertEqual(client.subscribed[0][2].get("consumer_type"), "shared-sentinel")
+
+    def test_topic_prefix_is_applied(self):
+        client = FakeClient()
+        tr = PulsarTransport(client=client, topic_prefix="persistent://public/default/")
+        tr.publish("orders", body())
+        # The producer is cached per queue; subscribe/produce use the prefixed topic.
+        tr.pop("orders")
+        self.assertEqual(client.subscribed[0][0], "persistent://public/default/orders")
+
+
+class PulsarSchemeTest(unittest.TestCase):
+    def test_make_transport_routes_pulsar_scheme(self):
+        if HAVE_PULSAR:
+            self.skipTest("pulsar-client installed — building a real client would need a broker")
+        with self.assertRaises(ImportError):
+            make_transport("pulsar://localhost:6650")
+
+
+if __name__ == "__main__":
+    unittest.main()