openframe-adapters-queue-kafka 1.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
openframe/__init__.py ADDED
@@ -0,0 +1 @@
1
+ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
@@ -0,0 +1 @@
1
+ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
@@ -0,0 +1 @@
1
+ __path__ = __import__("pkgutil").extend_path(__path__, __name__)
@@ -0,0 +1,49 @@
1
+ """
2
+ openframe.adapters.queue.kafka
3
+ ================================
4
+ Apache Kafka queue adapter for the OpenFrame Microservice Suite.
5
+
6
+ Public API::
7
+
8
+ KafkaSettings — Pydantic Settings subclass for connection config.
9
+ KafkaProducer — Generic async message producer (BaseProducer[T]).
10
+ KafkaConsumer — Generic async message consumer (BaseConsumer[T]).
11
+ KafkaPlugin — OpenFramePlugin implementation for PluginRegistry.
12
+
13
+ Quick start::
14
+
15
+ from openframe.adapters.queue.kafka import (
16
+ KafkaSettings,
17
+ KafkaProducer,
18
+ KafkaConsumer,
19
+ )
20
+
21
+ settings = KafkaSettings(kafka_bootstrap_servers="localhost:9092")
22
+
23
+ # Produce
24
+ producer = KafkaProducer(settings)
25
+ await producer.start()
26
+ await producer.publish({"event": "item.created", "id": "abc"})
27
+ await producer.close()
28
+
29
+ # Consume
30
+ consumer = KafkaConsumer(settings)
31
+
32
+ async def handle(event: dict) -> None:
33
+ print(f"Received: {event}")
34
+
35
+ await consumer.subscribe(handle)
36
+ """
37
+ from __future__ import annotations
38
+
39
+ from .config import KafkaSettings
40
+ from .consumer import KafkaConsumer
41
+ from .plugin import KafkaPlugin
42
+ from .producer import KafkaProducer
43
+
44
+ __all__ = [
45
+ "KafkaSettings",
46
+ "KafkaProducer",
47
+ "KafkaConsumer",
48
+ "KafkaPlugin",
49
+ ]
@@ -0,0 +1,51 @@
1
+ """
2
+ openframe/adapters/queue/kafka/config.py
3
+ ==========================================
4
+ Settings for the Kafka queue adapter, sourced from environment variables
5
+ via ``openframe-core``'s ``BaseAdapterSettings`` (pydantic-settings).
6
+
7
+ Required env vars:
8
+ KAFKA_BOOTSTRAP_SERVERS: Comma-separated broker list.
9
+ Format: host:port,host:port
10
+
11
+ Optional env vars (all have defaults):
12
+ KAFKA_TOPIC: str = "openframe" — default topic
13
+ KAFKA_GROUP_ID: str = "openframe-group" — consumer group
14
+ KAFKA_AUTO_OFFSET_RESET: str = "earliest"
15
+ KAFKA_MAX_POLL_RECORDS: int = 10
16
+ KAFKA_SESSION_TIMEOUT_MS: int = 30000
17
+ KAFKA_REQUEST_TIMEOUT_MS: int = 30000
18
+ KAFKA_SECURITY_PROTOCOL: str = "PLAINTEXT"
19
+ KAFKA_SASL_MECHANISM: str = ""
20
+ KAFKA_SASL_USERNAME: str = ""
21
+ KAFKA_SASL_PASSWORD: str = ""
22
+ ADAPTER_NAME: str = "kafka"
23
+ """
24
+ from __future__ import annotations
25
+
26
+ from openframe.core.config import BaseAdapterSettings
27
+
28
+ __all__ = ["KafkaSettings"]
29
+
30
+
31
+ class KafkaSettings(BaseAdapterSettings):
32
+ """
33
+ Pydantic-settings subclass for the Kafka queue adapter.
34
+
35
+ All fields are read from environment variables with the exact names
36
+ shown above. ``BaseAdapterSettings`` provides ``operation_timeout``
37
+ (default 30.0 s) and ``connection_timeout`` (default 30.0 s).
38
+ """
39
+
40
+ kafka_bootstrap_servers: str
41
+ kafka_topic: str = "openframe"
42
+ kafka_group_id: str = "openframe-group"
43
+ kafka_auto_offset_reset: str = "earliest"
44
+ kafka_max_poll_records: int = 10
45
+ kafka_session_timeout_ms: int = 30_000
46
+ kafka_request_timeout_ms: int = 30_000
47
+ kafka_security_protocol: str = "PLAINTEXT"
48
+ kafka_sasl_mechanism: str = ""
49
+ kafka_sasl_username: str = ""
50
+ kafka_sasl_password: str = ""
51
+ adapter_name: str = "kafka"
@@ -0,0 +1,214 @@
1
+ """
2
+ openframe/adapters/queue/kafka/consumer.py
3
+ ============================================
4
+ Kafka message consumer implementing ``BaseConsumer[T]`` from
5
+ ``openframe-core`` via structural subtyping.
6
+
7
+ Each ``subscribe()`` call creates, starts, and runs a new
8
+ ``AIOKafkaConsumer`` until ``close()`` is called. Manual offset commit
9
+ is used — the offset is committed only after the handler returns
10
+ successfully. If the handler raises, ``nack()`` is called (no commit,
11
+ message will be redelivered on the next poll).
12
+
13
+ Error handling:
14
+ ``KafkaConnectionError`` on startup → ``AdapterConnectionError``.
15
+ ``KafkaError`` on startup → ``AdapterConfigurationError``.
16
+ Handler exceptions are caught, logged, and the loop continues.
17
+ """
18
+ from __future__ import annotations
19
+
20
+ import asyncio
21
+ import json
22
+ import logging
23
+ from typing import Any, Awaitable, Callable, Generic, TypeVar
24
+
25
+ import aiokafka
26
+ import aiokafka.errors
27
+ from aiokafka import AIOKafkaConsumer
28
+
29
+ from openframe.core.exceptions import (
30
+ AdapterConfigurationError,
31
+ AdapterConnectionError,
32
+ )
33
+
34
+ from .config import KafkaSettings
35
+
36
+ __all__ = ["KafkaConsumer"]
37
+
38
+ T = TypeVar("T")
39
+ _logger = logging.getLogger(__name__)
40
+
41
+
42
+ class KafkaConsumer(Generic[T]):
43
+ """
44
+ Kafka message consumer.
45
+
46
+ Implements ``BaseConsumer[T]`` structurally — no inheritance from Protocol.
47
+ Deserialises JSON messages. Manual offset commit after handler success.
48
+
49
+ Each ``subscribe()`` call creates a new ``AIOKafkaConsumer``, starts it,
50
+ polls until ``close()`` is called, then stops it. Handler failures trigger
51
+ ``nack()`` (no commit — message is redelivered).
52
+
53
+ Usage::
54
+
55
+ consumer = KafkaConsumer(settings)
56
+
57
+ async def handle(event: dict) -> None:
58
+ print(f"Received: {event}")
59
+
60
+ await consumer.subscribe(handle) # runs until close() called
61
+
62
+ Structural conformance::
63
+
64
+ assert isinstance(consumer, BaseConsumer)
65
+ """
66
+
67
+ def __init__(self, settings: KafkaSettings) -> None:
68
+ self._settings = settings
69
+ self._consumer: AIOKafkaConsumer | None = None
70
+ self._running: bool = False
71
+
72
+ # ------------------------------------------------------------------
73
+ # Deserialisation (override in typed subclasses)
74
+ # ------------------------------------------------------------------
75
+
76
+ def _deserialise(self, raw: bytes) -> T:
77
+ """
78
+ Deserialise bytes from Kafka to the message type ``T``.
79
+
80
+ Base implementation: ``json.loads(raw.decode("utf-8"))``.
81
+ Subclasses override for custom deserialisation (protobuf, Avro, etc.).
82
+ """
83
+ return json.loads(raw.decode("utf-8")) # type: ignore[return-value]
84
+
85
+ # ------------------------------------------------------------------
86
+ # BaseConsumer[T] interface
87
+ # ------------------------------------------------------------------
88
+
89
+ async def subscribe(
90
+ self,
91
+ handler: Callable[[T], Awaitable[None]],
92
+ ) -> None:
93
+ """
94
+ Start consuming messages and invoke ``handler`` for each one.
95
+
96
+ Runs until ``close()`` is called. Uses manual commit — commits
97
+ the offset only after ``handler`` returns successfully. If
98
+ ``handler`` raises, calls ``nack()`` and continues to the next
99
+ message.
100
+
101
+ Args:
102
+ handler: Async callable that processes each deserialised message.
103
+
104
+ Raises:
105
+ AdapterConnectionError: Cannot reach the Kafka broker.
106
+ AdapterConfigurationError: Invalid topic, group, or credentials.
107
+ """
108
+ kwargs: dict[str, Any] = {
109
+ "bootstrap_servers": self._settings.kafka_bootstrap_servers,
110
+ "group_id": self._settings.kafka_group_id,
111
+ "auto_offset_reset": self._settings.kafka_auto_offset_reset,
112
+ "enable_auto_commit": False,
113
+ "max_poll_records": self._settings.kafka_max_poll_records,
114
+ "session_timeout_ms": self._settings.kafka_session_timeout_ms,
115
+ "request_timeout_ms": self._settings.kafka_request_timeout_ms,
116
+ "security_protocol": self._settings.kafka_security_protocol,
117
+ }
118
+ if self._settings.kafka_sasl_mechanism:
119
+ kwargs["sasl_mechanism"] = self._settings.kafka_sasl_mechanism
120
+ kwargs["sasl_plain_username"] = self._settings.kafka_sasl_username
121
+ kwargs["sasl_plain_password"] = self._settings.kafka_sasl_password
122
+
123
+ try:
124
+ self._consumer = AIOKafkaConsumer(
125
+ self._settings.kafka_topic, **kwargs
126
+ )
127
+ await self._consumer.start()
128
+ self._running = True
129
+ except aiokafka.errors.KafkaConnectionError as exc:
130
+ self._consumer = None
131
+ raise AdapterConnectionError(
132
+ f"Kafka consumer cannot connect to {self._settings.kafka_bootstrap_servers!r}: {exc}",
133
+ adapter=self._settings.adapter_name,
134
+ operation="subscribe",
135
+ cause=exc,
136
+ ) from exc
137
+ except aiokafka.errors.KafkaError as exc:
138
+ self._consumer = None
139
+ raise AdapterConfigurationError(
140
+ f"Kafka consumer configuration error: {exc}",
141
+ adapter=self._settings.adapter_name,
142
+ operation="subscribe",
143
+ ) from exc
144
+
145
+ try:
146
+ async for msg in self._consumer:
147
+ if not self._running:
148
+ break
149
+ try:
150
+ message = self._deserialise(msg.value)
151
+ await handler(message)
152
+ await self.ack(message)
153
+ except Exception as exc: # noqa: BLE001
154
+ _logger.error(
155
+ "KafkaConsumer handler failed for topic %r: %s",
156
+ self._settings.kafka_topic,
157
+ exc,
158
+ )
159
+ await self.nack(message)
160
+ finally:
161
+ if self._consumer is not None:
162
+ try:
163
+ await self._consumer.stop()
164
+ except Exception as exc: # noqa: BLE001
165
+ _logger.error("KafkaConsumer stop error (ignored): %s", exc)
166
+ self._consumer = None
167
+ self._running = False
168
+
169
+ async def ack(self, message: T) -> None:
170
+ """
171
+ Acknowledge a message by committing the current offset.
172
+
173
+ Called automatically after successful handler invocation.
174
+ Can also be called manually for custom acknowledgement flows.
175
+ Never raises.
176
+
177
+ Args:
178
+ message: The message that was successfully processed.
179
+ """
180
+ if self._consumer is not None:
181
+ try:
182
+ await self._consumer.commit()
183
+ except Exception as exc: # noqa: BLE001
184
+ _logger.error("KafkaConsumer ack (commit) error (ignored): %s", exc)
185
+
186
+ async def nack(self, message: T) -> None:
187
+ """
188
+ Negatively acknowledge a message.
189
+
190
+ Does NOT commit the offset — the message will be redelivered
191
+ on the next poll. Logs the failure. Never raises.
192
+
193
+ Args:
194
+ message: The message that failed processing.
195
+ """
196
+ _logger.warning(
197
+ "KafkaConsumer nack — message will be redelivered on topic %r",
198
+ self._settings.kafka_topic,
199
+ )
200
+
201
+ async def close(self) -> None:
202
+ """
203
+ Stop the consumer.
204
+
205
+ Sets ``_running = False`` to break the polling loop. The consumer
206
+ is stopped in the ``finally`` block of ``subscribe()``. Never raises.
207
+ """
208
+ self._running = False
209
+ if self._consumer is not None:
210
+ try:
211
+ await self._consumer.stop()
212
+ except Exception as exc: # noqa: BLE001
213
+ _logger.error("KafkaConsumer close error (ignored): %s", exc)
214
+ self._consumer = None
@@ -0,0 +1,160 @@
1
+ """
2
+ openframe/adapters/queue/kafka/plugin.py
3
+ ==========================================
4
+ OpenFrame plugin wrapper for KafkaProducer and KafkaConsumer.
5
+
6
+ Stability: beta
7
+ Capability: "queue"
8
+
9
+ Manages the ``KafkaProducer`` lifecycle. Consumers are created on-demand
10
+ via ``make_consumer()`` since each consume session is short-lived.
11
+
12
+ Usage via PluginRegistry (optional)::
13
+
14
+ from openframe.core.plugins import PluginRegistry
15
+ from openframe.adapters.queue.kafka import KafkaPlugin, KafkaSettings
16
+
17
+ registry = PluginRegistry()
18
+ registry.register(KafkaPlugin(KafkaSettings()))
19
+ await registry.initialize_all()
20
+
21
+ plugin = registry.get("queue")
22
+ producer = plugin.get_producer()
23
+ await producer.publish({"event": "item.created"})
24
+
25
+ consumer = plugin.make_consumer()
26
+ await consumer.subscribe(handler)
27
+
28
+ Usage via deps.py (unchanged, no plugin needed)::
29
+
30
+ producer = KafkaProducer(KafkaSettings())
31
+ await producer.start()
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import logging
36
+
37
+ from openframe.core.plugins import PluginContext, PluginHealth, PluginStatus
38
+
39
+ from openframe.adapters.queue.kafka.config import KafkaSettings
40
+ from openframe.adapters.queue.kafka.consumer import KafkaConsumer
41
+ from openframe.adapters.queue.kafka.producer import KafkaProducer
42
+
43
+ __all__ = ["KafkaPlugin"]
44
+
45
+ _logger = logging.getLogger(__name__)
46
+
47
+
48
+ class KafkaPlugin:
49
+ """
50
+ Kafka adapter plugin for the OpenFrame plugin registry.
51
+
52
+ Capability: "queue"
53
+
54
+ Lifecycle:
55
+ initialize() — starts the ``KafkaProducer`` and verifies connectivity.
56
+ Raises on failure.
57
+ shutdown() — closes the producer. Never raises.
58
+ health() — checks whether the producer is READY. Never raises.
59
+
60
+ The plugin exposes ``get_producer()`` and ``make_consumer()`` after
61
+ initialization for use in the composition root or ApplicationBootstrap.
62
+ """
63
+
64
+ name: str = "openframe-kafka"
65
+ version: str = "1.1.0"
66
+ capability: str = "queue"
67
+
68
+ def __init__(self, settings: KafkaSettings) -> None:
69
+ self._settings = settings
70
+ self._producer: KafkaProducer | None = None
71
+ self._status = PluginStatus.REGISTERED
72
+
73
+ async def initialize(self, context: PluginContext) -> None:
74
+ """
75
+ Start the KafkaProducer and verify broker connectivity.
76
+
77
+ Args:
78
+ context: Plugin context (config, plugin_name). Unused here —
79
+ settings are provided at construction time.
80
+
81
+ Raises:
82
+ AdapterConnectionError: Broker is unreachable.
83
+ AdapterConfigurationError: Invalid bootstrap servers or settings.
84
+ AdapterTimeoutError: Start exceeded ``connection_timeout``.
85
+ """
86
+ self._status = PluginStatus.INITIALIZED
87
+ try:
88
+ self._producer = KafkaProducer(self._settings)
89
+ await self._producer.start()
90
+ self._status = PluginStatus.READY
91
+ _logger.info(
92
+ "KafkaPlugin initialized — %s",
93
+ self._settings.kafka_bootstrap_servers,
94
+ )
95
+ except Exception:
96
+ self._status = PluginStatus.FAILED
97
+ raise
98
+
99
+ async def shutdown(self) -> None:
100
+ """
101
+ Close the KafkaProducer.
102
+
103
+ Never raises — logs errors and continues.
104
+ """
105
+ self._status = PluginStatus.STOPPING
106
+ try:
107
+ if self._producer is not None:
108
+ await self._producer.close()
109
+ _logger.info("KafkaPlugin shutdown complete.")
110
+ except Exception as exc:
111
+ _logger.error("KafkaPlugin shutdown error (ignored): %s", exc)
112
+ finally:
113
+ self._status = PluginStatus.STOPPED
114
+
115
+ async def health(self) -> PluginHealth:
116
+ """
117
+ Return current health snapshot.
118
+
119
+ Never raises — returns FAILED status on any exception.
120
+ """
121
+ try:
122
+ if self._producer is None or self._status != PluginStatus.READY:
123
+ return PluginHealth(
124
+ status=PluginStatus.FAILED,
125
+ message=f"Plugin status: {self._status.name}",
126
+ )
127
+ return PluginHealth(status=PluginStatus.READY, message="")
128
+ except Exception as exc:
129
+ return PluginHealth(
130
+ status=PluginStatus.FAILED,
131
+ message=str(exc),
132
+ )
133
+
134
+ def get_producer(self) -> KafkaProducer:
135
+ """
136
+ Return the started producer.
137
+
138
+ Only valid after ``registry.initialize_all()`` has been called.
139
+
140
+ Raises:
141
+ RuntimeError: Plugin not yet initialized.
142
+ """
143
+ if self._producer is None or self._status != PluginStatus.READY:
144
+ raise RuntimeError(
145
+ f"KafkaPlugin is not ready (status: {self._status.name}). "
146
+ "Call await registry.initialize_all() first."
147
+ )
148
+ return self._producer
149
+
150
+ def make_consumer(self) -> KafkaConsumer:
151
+ """
152
+ Create a new ``KafkaConsumer`` with this plugin's settings.
153
+
154
+ Consumers are short-lived (one per ``subscribe()`` session) so
155
+ they are created on demand rather than cached.
156
+
157
+ Returns:
158
+ A fresh ``KafkaConsumer`` ready to call ``subscribe()`` on.
159
+ """
160
+ return KafkaConsumer(self._settings)
@@ -0,0 +1,238 @@
1
+ """
2
+ openframe/adapters/queue/kafka/producer.py
3
+ ============================================
4
+ Kafka message producer implementing ``BaseProducer[T]`` from
5
+ ``openframe-core`` via structural subtyping.
6
+
7
+ Messages are serialised to JSON bytes before being sent to Kafka.
8
+ The underlying ``AIOKafkaProducer`` must be explicitly started via
9
+ ``await producer.start()`` before calling ``publish()`` or
10
+ ``publish_batch()``.
11
+
12
+ Error handling:
13
+ Every ``aiokafka.errors.*`` is caught and re-raised as the appropriate
14
+ ``AdapterError`` subclass with cause chaining.
15
+
16
+ Timeout strategy:
17
+ ``asyncio.timeout(settings.operation_timeout)`` wraps every send.
18
+ """
19
+ from __future__ import annotations
20
+
21
+ import asyncio
22
+ import json
23
+ import logging
24
+ from typing import Any, Generic, TypeVar
25
+
26
+ import aiokafka
27
+ import aiokafka.errors
28
+ from aiokafka import AIOKafkaProducer
29
+
30
+ from openframe.core.exceptions import (
31
+ AdapterConfigurationError,
32
+ AdapterConnectionError,
33
+ AdapterQueryError,
34
+ AdapterTimeoutError,
35
+ )
36
+ from openframe.core.ports import BaseProducer
37
+
38
+ from .config import KafkaSettings
39
+
40
+ __all__ = ["KafkaProducer"]
41
+
42
+ T = TypeVar("T")
43
+ _logger = logging.getLogger(__name__)
44
+
45
+
46
+ class KafkaProducer(Generic[T]):
47
+ """
48
+ Kafka message producer.
49
+
50
+ Implements ``BaseProducer[T]`` structurally — no inheritance from Protocol.
51
+ Serialises messages to JSON bytes before publishing to Kafka.
52
+
53
+ The underlying ``AIOKafkaProducer`` must be started before use.
54
+ Call ``await producer.start()`` or use ``KafkaPlugin`` for managed
55
+ lifecycle.
56
+
57
+ Usage::
58
+
59
+ producer = KafkaProducer(settings)
60
+ await producer.start()
61
+ await producer.publish({"event": "item.created", "id": "abc"})
62
+ await producer.close()
63
+
64
+ Structural conformance::
65
+
66
+ assert isinstance(producer, BaseProducer)
67
+ """
68
+
69
+ def __init__(self, settings: KafkaSettings) -> None:
70
+ self._settings = settings
71
+ self._producer: AIOKafkaProducer | None = None
72
+
73
+ # ------------------------------------------------------------------
74
+ # Serialisation (override in typed subclasses)
75
+ # ------------------------------------------------------------------
76
+
77
+ def _serialise(self, message: T) -> bytes:
78
+ """
79
+ Serialise a message to bytes for Kafka.
80
+
81
+ Base implementation: ``json.dumps(message).encode("utf-8")``.
82
+ Subclasses override for custom serialisation (protobuf, Avro, etc.).
83
+ """
84
+ return json.dumps(message).encode("utf-8")
85
+
86
+ # ------------------------------------------------------------------
87
+ # Lifecycle
88
+ # ------------------------------------------------------------------
89
+
90
+ async def start(self) -> None:
91
+ """
92
+ Start the underlying ``AIOKafkaProducer``.
93
+
94
+ Must be called before ``publish()`` or ``publish_batch()``.
95
+
96
+ Raises:
97
+ AdapterConnectionError: Broker is unreachable.
98
+ AdapterConfigurationError: Invalid bootstrap servers or settings.
99
+ AdapterTimeoutError: Start exceeded ``connection_timeout``.
100
+ """
101
+ kwargs: dict[str, Any] = {
102
+ "bootstrap_servers": self._settings.kafka_bootstrap_servers,
103
+ "request_timeout_ms": self._settings.kafka_request_timeout_ms,
104
+ "security_protocol": self._settings.kafka_security_protocol,
105
+ }
106
+ if self._settings.kafka_sasl_mechanism:
107
+ kwargs["sasl_mechanism"] = self._settings.kafka_sasl_mechanism
108
+ kwargs["sasl_plain_username"] = self._settings.kafka_sasl_username
109
+ kwargs["sasl_plain_password"] = self._settings.kafka_sasl_password
110
+
111
+ try:
112
+ async with asyncio.timeout(self._settings.connection_timeout):
113
+ self._producer = AIOKafkaProducer(**kwargs)
114
+ await self._producer.start()
115
+ except asyncio.TimeoutError as exc:
116
+ self._producer = None
117
+ raise AdapterTimeoutError(
118
+ f"Kafka producer start timed out after {self._settings.connection_timeout}s",
119
+ adapter=self._settings.adapter_name,
120
+ operation="start",
121
+ cause=exc,
122
+ ) from exc
123
+ except aiokafka.errors.KafkaConnectionError as exc:
124
+ self._producer = None
125
+ raise AdapterConnectionError(
126
+ f"Kafka producer cannot connect to {self._settings.kafka_bootstrap_servers!r}: {exc}",
127
+ adapter=self._settings.adapter_name,
128
+ operation="start",
129
+ cause=exc,
130
+ ) from exc
131
+ except aiokafka.errors.KafkaError as exc:
132
+ self._producer = None
133
+ raise AdapterConfigurationError(
134
+ f"Kafka producer configuration error: {exc}",
135
+ adapter=self._settings.adapter_name,
136
+ operation="start",
137
+ ) from exc
138
+
139
+ async def close(self) -> None:
140
+ """
141
+ Stop the producer and release resources.
142
+
143
+ Idempotent — safe to call multiple times. Never raises.
144
+ """
145
+ if self._producer is not None:
146
+ try:
147
+ await self._producer.stop()
148
+ except Exception as exc: # noqa: BLE001
149
+ _logger.error("KafkaProducer close error (ignored): %s", exc)
150
+ finally:
151
+ self._producer = None
152
+
153
+ # ------------------------------------------------------------------
154
+ # BaseProducer[T] interface
155
+ # ------------------------------------------------------------------
156
+
157
+ async def publish(self, message: T) -> None:
158
+ """
159
+ Publish a single message to the configured topic.
160
+
161
+ Serialises the message to JSON bytes and waits for broker
162
+ acknowledgement via ``send_and_wait``.
163
+
164
+ Args:
165
+ message: The message to publish.
166
+
167
+ Raises:
168
+ RuntimeError: Producer not started — call ``start()`` first.
169
+ AdapterQueryError: Publish failed (broker error).
170
+ AdapterTimeoutError: Publish exceeded ``operation_timeout``.
171
+ """
172
+ if self._producer is None:
173
+ raise RuntimeError(
174
+ "KafkaProducer not started. Call await producer.start() first."
175
+ )
176
+ value = self._serialise(message)
177
+ try:
178
+ async with asyncio.timeout(self._settings.operation_timeout):
179
+ await self._producer.send_and_wait(
180
+ self._settings.kafka_topic, value
181
+ )
182
+ except asyncio.TimeoutError as exc:
183
+ raise AdapterTimeoutError(
184
+ f"publish exceeded {self._settings.operation_timeout}s operation_timeout",
185
+ adapter=self._settings.adapter_name,
186
+ operation="publish",
187
+ cause=exc,
188
+ ) from exc
189
+ except aiokafka.errors.KafkaError as exc:
190
+ raise AdapterQueryError(
191
+ f"publish to topic {self._settings.kafka_topic!r} failed: {exc}",
192
+ adapter=self._settings.adapter_name,
193
+ operation="publish",
194
+ cause=exc,
195
+ ) from exc
196
+
197
+ async def publish_batch(self, messages: list[T]) -> None:
198
+ """
199
+ Publish multiple messages to the configured topic.
200
+
201
+ Sends each message individually, then flushes to ensure all
202
+ are delivered. Raises on the first failure — messages already
203
+ sent are not rolled back.
204
+
205
+ Args:
206
+ messages: List of messages to publish.
207
+
208
+ Raises:
209
+ RuntimeError: Producer not started.
210
+ AdapterQueryError: Batch publish failed.
211
+ AdapterTimeoutError: Exceeded ``operation_timeout``.
212
+ """
213
+ if self._producer is None:
214
+ raise RuntimeError(
215
+ "KafkaProducer not started. Call await producer.start() first."
216
+ )
217
+ try:
218
+ async with asyncio.timeout(self._settings.operation_timeout):
219
+ for message in messages:
220
+ value = self._serialise(message)
221
+ await self._producer.send_and_wait(
222
+ self._settings.kafka_topic, value
223
+ )
224
+ await self._producer.flush()
225
+ except asyncio.TimeoutError as exc:
226
+ raise AdapterTimeoutError(
227
+ f"publish_batch exceeded {self._settings.operation_timeout}s operation_timeout",
228
+ adapter=self._settings.adapter_name,
229
+ operation="publish_batch",
230
+ cause=exc,
231
+ ) from exc
232
+ except aiokafka.errors.KafkaError as exc:
233
+ raise AdapterQueryError(
234
+ f"publish_batch to topic {self._settings.kafka_topic!r} failed: {exc}",
235
+ adapter=self._settings.adapter_name,
236
+ operation="publish_batch",
237
+ cause=exc,
238
+ ) from exc
@@ -0,0 +1,150 @@
1
+ Metadata-Version: 2.4
2
+ Name: openframe-adapters-queue-kafka
3
+ Version: 1.1.0
4
+ Summary: OpenFrame Microservice Suite - Apache Kafka queue adapter.
5
+ Project-URL: Homepage, https://github.com/Furious-Meteors/openframe-adapters
6
+ Project-URL: Documentation, https://furious-meteors.github.io/openframe-adapters/
7
+ Project-URL: Repository, https://github.com/Furious-Meteors/openframe-adapters
8
+ Project-URL: Changelog, https://github.com/Furious-Meteors/openframe-adapters/blob/production/.github/CHANGELOG.md
9
+ Project-URL: Bug Tracker, https://github.com/Furious-Meteors/openframe-adapters/issues
10
+ Author-email: Furious Meteors Engineering <engineering@furiousmeteors.dev>
11
+ Maintainer-email: Furious Meteors Engineering <engineering@furiousmeteors.dev>
12
+ License: MIT
13
+ Keywords: hexagonal,kafka,messaging,microservice,openframe,queue
14
+ Classifier: Development Status :: 5 - Production/Stable
15
+ Classifier: Intended Audience :: Developers
16
+ Classifier: License :: OSI Approved :: MIT License
17
+ Classifier: Operating System :: OS Independent
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
+ Classifier: Topic :: System :: Distributed Computing
24
+ Classifier: Typing :: Typed
25
+ Requires-Python: >=3.11
26
+ Requires-Dist: aiokafka>=0.11
27
+ Requires-Dist: openframe-core<3,>=2.0
28
+ Provides-Extra: dev
29
+ Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
30
+ Requires-Dist: pytest-mock>=3.14; extra == 'dev'
31
+ Requires-Dist: pytest>=8.0; extra == 'dev'
32
+ Description-Content-Type: text/markdown
33
+
34
+ # openframe-adapters-queue-kafka
35
+
36
+ Apache Kafka queue adapter for the **OpenFrame Microservice Development Suite**.
37
+
38
+ Part of the [`openframe-adapters`](https://github.com/Furious-Meteors/openframe-adapters) monorepo.
39
+
40
+ ---
41
+
42
+ ## What it provides
43
+
44
+ | Symbol | Purpose |
45
+ |---|---|
46
+ | `KafkaSettings` | Pydantic-settings subclass — reads all config from env vars |
47
+ | `KafkaProducer[T]` | Generic async message producer — `BaseProducer[T]` |
48
+ | `KafkaConsumer[T]` | Generic async message consumer — `BaseConsumer[T]` |
49
+ | `KafkaPlugin` | `OpenFramePlugin` — structured lifecycle via `PluginRegistry` |
50
+
51
+ ## Installation
52
+
53
+ ```bash
54
+ # Via meta-package (recommended)
55
+ pip install "openframe-adapters[kafka]"
56
+
57
+ # Or directly
58
+ pip install openframe-adapters-queue-kafka
59
+ ```
60
+
61
+ ## Quick start
62
+
63
+ ```python
64
+ from openframe.adapters.queue.kafka import KafkaSettings, KafkaProducer, KafkaConsumer
65
+
66
+ settings = KafkaSettings(kafka_bootstrap_servers="localhost:9092")
67
+
68
+ # Produce
69
+ producer = KafkaProducer(settings)
70
+ await producer.start()
71
+ await producer.publish({"event": "item.created", "id": "abc"})
72
+ await producer.publish_batch([{"event": "x"}, {"event": "y"}])
73
+ await producer.close()
74
+
75
+ # Consume
76
+ consumer = KafkaConsumer(settings)
77
+
78
+ async def handle(event: dict) -> None:
79
+ print(f"Received: {event}")
80
+
81
+ await consumer.subscribe(handle) # runs until consumer.close() called
82
+ ```
83
+
84
+ ## Configuration
85
+
86
+ | Env var | Default | Description |
87
+ |---|---|---|
88
+ | `KAFKA_BOOTSTRAP_SERVERS` | **required** | `host:port,host:port` |
89
+ | `KAFKA_TOPIC` | `"openframe"` | Default topic for producer and consumer |
90
+ | `KAFKA_GROUP_ID` | `"openframe-group"` | Consumer group ID |
91
+ | `KAFKA_AUTO_OFFSET_RESET` | `"earliest"` | `"earliest"` or `"latest"` |
92
+ | `KAFKA_MAX_POLL_RECORDS` | `10` | Max messages per poll |
93
+ | `KAFKA_SESSION_TIMEOUT_MS` | `30000` | Consumer session timeout |
94
+ | `KAFKA_REQUEST_TIMEOUT_MS` | `30000` | Broker request timeout |
95
+ | `KAFKA_SECURITY_PROTOCOL` | `"PLAINTEXT"` | `"PLAINTEXT"`, `"SSL"`, `"SASL_PLAINTEXT"` |
96
+ | `KAFKA_SASL_MECHANISM` | `""` | `"PLAIN"`, `"SCRAM-SHA-256"`, etc. |
97
+ | `KAFKA_SASL_USERNAME` | `""` | SASL username |
98
+ | `KAFKA_SASL_PASSWORD` | `""` | SASL password |
99
+
100
+ ## Typed domain objects
101
+
102
+ ```python
103
+ from openframe.adapters.queue.kafka import KafkaProducer, KafkaConsumer, KafkaSettings
104
+ from dataclasses import dataclass, asdict
105
+
106
+ @dataclass
107
+ class OrderEvent:
108
+ order_id: str
109
+ event_type: str
110
+
111
+ class OrderProducer(KafkaProducer[OrderEvent]):
112
+ def _serialise(self, message: OrderEvent) -> bytes:
113
+ import json
114
+ return json.dumps(asdict(message)).encode("utf-8")
115
+
116
+ class OrderConsumer(KafkaConsumer[OrderEvent]):
117
+ def _deserialise(self, raw: bytes) -> OrderEvent:
118
+ import json
119
+ return OrderEvent(**json.loads(raw.decode("utf-8")))
120
+ ```
121
+
122
+ ## Plugin lifecycle (optional)
123
+
124
+ ```python
125
+ from openframe.core.plugins import PluginRegistry
126
+ from openframe.adapters.queue.kafka import KafkaPlugin, KafkaSettings
127
+
128
+ registry = PluginRegistry()
129
+ registry.register(KafkaPlugin(KafkaSettings()))
130
+ await registry.initialize_all()
131
+
132
+ plugin = registry.get("queue")
133
+ producer = plugin.get_producer()
134
+ await producer.publish({"event": "item.created"})
135
+
136
+ consumer = plugin.make_consumer()
137
+ await consumer.subscribe(handler)
138
+ ```
139
+
140
+ ## Consumer acknowledgement semantics
141
+
142
+ | Outcome | Behaviour |
143
+ |---|---|
144
+ | Handler returns | `ack()` called → offset committed → message consumed |
145
+ | Handler raises | `nack()` called → no commit → message redelivered |
146
+ | `consumer.close()` | polling loop exits → consumer stopped cleanly |
147
+
148
+ ## License
149
+
150
+ MIT — © Furious Meteors Engineering
@@ -0,0 +1,11 @@
1
+ openframe/__init__.py,sha256=ED6jHcYiuYpr_0vjGz0zx2lrrmJT9sDJCzIljoDfmlM,65
2
+ openframe/adapters/__init__.py,sha256=ED6jHcYiuYpr_0vjGz0zx2lrrmJT9sDJCzIljoDfmlM,65
3
+ openframe/adapters/queue/__init__.py,sha256=ED6jHcYiuYpr_0vjGz0zx2lrrmJT9sDJCzIljoDfmlM,65
4
+ openframe/adapters/queue/kafka/__init__.py,sha256=SvuSQQ767qsrEi9vuK1B2oERt5UYjfa-Mn1x6nvOFSI,1275
5
+ openframe/adapters/queue/kafka/config.py,sha256=AbjKdT5us_vcjPop8f7hrvMRg8FLivt3Tdo78PkiZPI,1859
6
+ openframe/adapters/queue/kafka/consumer.py,sha256=CzD0j39mnIECaI4FzYLEoDRy43APRnrWEkW4U7u5Xg8,7834
7
+ openframe/adapters/queue/kafka/plugin.py,sha256=6InEjWSpqWecVPd3Ef1bszfKXVMg8-iKEirSFz0xPMI,5302
8
+ openframe/adapters/queue/kafka/producer.py,sha256=rpoHs1Gw2lqgzS_CUW6hhvTVTS_wpqP27dPKFupL2ew,8798
9
+ openframe_adapters_queue_kafka-1.1.0.dist-info/METADATA,sha256=WYtSUYjsj9quT2-bbh7HYWPTkcjfD6sOG7E6SiLeSP8,5241
10
+ openframe_adapters_queue_kafka-1.1.0.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87
11
+ openframe_adapters_queue_kafka-1.1.0.dist-info/RECORD,,
@@ -0,0 +1,4 @@
1
+ Wheel-Version: 1.0
2
+ Generator: hatchling 1.30.1
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any