potik 0.1.0__py3-none-any.whl

potik/__init__.py

@@ -0,0 +1,54 @@
+"""Typed handlers for Kafka topics.
+
+Declare each `Topic` with the types and serdes of its keys and values, register handlers on a `Router`, and build a core `Runtime` from the routers with `build_runtime()`. The package also re-exports the core names that services use.
+"""
+
+from potik.build import build_runtime
+from potik.core.config import Config
+from potik.core.errors import InvalidOutputError, PermanentError, PotikError
+from potik.core.offsets import OffsetContext, OffsetPolicy, resume_or_earliest, resume_or_fail, resume_or_latest
+from potik.core.producer import Producer
+from potik.core.replies import PendingReply, ReplyListener
+from potik.core.runtime import Runtime
+from potik.core.types import (
+    ConsumerRecord,
+    Delivery,
+    ProducerRecord,
+    RuntimeStatus,
+)
+from potik.handler import Context, DecodeError, HandlerResult, Header, Key
+from potik.middleware.topics import TopicCheckError
+from potik.router import RegistrationError, Router
+from potik.topic import EncodeError, OutputTopic, Topic
+
+__all__ = [
+    "Config",
+    "ConsumerRecord",
+    "Context",
+    "DecodeError",
+    "Delivery",
+    "EncodeError",
+    "HandlerResult",
+    "Header",
+    "InvalidOutputError",
+    "Key",
+    "OffsetContext",
+    "OffsetPolicy",
+    "OutputTopic",
+    "PendingReply",
+    "PermanentError",
+    "PotikError",
+    "Producer",
+    "ProducerRecord",
+    "RegistrationError",
+    "ReplyListener",
+    "Router",
+    "Runtime",
+    "RuntimeStatus",
+    "Topic",
+    "TopicCheckError",
+    "build_runtime",
+    "resume_or_earliest",
+    "resume_or_fail",
+    "resume_or_latest",
+]

potik/build.py

@@ -0,0 +1,124 @@
+"""Build a core `Runtime` from routers of typed handlers."""
+
+from collections.abc import Awaitable, Iterable, Sequence
+from typing import cast
+
+from potik.core.config import Config
+from potik.core.middleware import BaseMiddleware, _chain, _middleware_tuple
+from potik.core.runtime import Runtime
+from potik.core.types import ConsumerRecord, ProducerRecord, RecordHandler
+from potik.handler import _NO_CONTEXT
+from potik.middleware.concurrency import ConcurrencyMiddleware, _validate_limit
+from potik.middleware.exception import ExceptionMiddleware, Rule
+from potik.middleware.topics import TopicCheckMiddleware, _declared_topics
+from potik.router import RegistrationError, Router, _callable_name, _Registration
+
+__all__ = [
+    "build_runtime",
+]
+
+
+def build_runtime(
+    config: Config,
+    routers: Iterable[Router],
+    *,
+    context: object = _NO_CONTEXT,
+    middleware: Iterable[BaseMiddleware] = (),
+    exception_rules: Iterable[Rule[Exception]] = (),
+    max_concurrency: int | None = None,
+    check_topics: bool = True,
+) -> Runtime:
+    """Build a `Runtime` that runs the routers' handlers.
+
+    Open the resources that handlers need first, e.g. a database pool or a `Producer`, then build the runtime with them and run it. A runtime runs once, so another run needs a new runtime with new middleware instances.
+
+    Each record passes through `middleware`, then the exception rules, then the concurrency limits, then its handler.
+
+    Args:
+        config: The runtime limits and client options.
+        routers: The routers whose handlers to run. The runtime copies their handlers, so later registrations have no effect.
+        context: What `Context()` parameters take, e.g. a dataclass of the service's clients.
+        middleware: Middleware instances, outermost first (see `BaseMiddleware`). Pass exception rules as `exception_rules`, not as an `ExceptionMiddleware`.
+        exception_rules: Rules that retry or recover from handler exceptions, tried in order (see `ExceptionMiddleware`). They run inside `middleware`, so e.g. tracing and metrics see one call per record, including its retries. An exception that no rule handles stops the record's partition, or fails the run (see `Config.stop_failed_partitions`). To apply a rule to some topics only, use `on(..., topics=...)`.
+        max_concurrency: The maximum number of handler calls that run at once, across all partitions and topics, e.g. the size of a database pool, or None for no limit. Like each handler's own `max_concurrency`, it limits only handler calls, so retry backoff holds no slot.
+        check_topics: Whether `run()` first checks that the topics exist, with a `TopicCheckMiddleware` before `middleware`. It checks the handled topics and the output topics of the handlers, routers, middleware, and exception rules. A missing topic fails the run with `TopicCheckError`. Turn it off e.g. to start before the topics are created.
+
+    Returns:
+        The runtime. Call its `run()` to run it.
+
+    Raises:
+        RegistrationError: If the routers have no handlers, two routers handle the same topic, a handler takes `Context()` and there is no `context`, or `middleware` includes an `ExceptionMiddleware`.
+        TypeError: If a router is not a `Router`, a middleware is not a `BaseMiddleware` instance, or an exception rule is a type or is not callable.
+        ValueError: If `max_concurrency` is not a positive integer or None.
+
+    Example:
+        ```python
+        async def main() -> None:
+            async with create_pool() as db:
+                await potik.build_runtime(
+                    config,
+                    [router],
+                    context=db,
+                    middleware=[TracingMiddleware()],
+                    exception_rules=[
+                        on(TimeoutError, retry(max_attempts=5)),
+                        on(Exception, dead_letter(dead_letters)),
+                    ],
+                ).run()
+        ```
+    """
+    _validate_limit(max_concurrency, "max_concurrency")
+    stack = _middleware_tuple(middleware)
+    for item in stack:
+        if isinstance(item, ExceptionMiddleware):
+            raise RegistrationError("pass exception rules as build_runtime(exception_rules=...)")
+    if rules := tuple(exception_rules):
+        stack = (*stack, ExceptionMiddleware(*rules))
+
+    registrations: dict[str, _Registration] = {}
+    outputs = [topic for item in stack for topic in _declared_topics(item)]
+    for router in routers:
+        if not isinstance(router, Router):
+            raise TypeError(f"routers must be Router instances, got {type(router).__name__}")
+        for topic, registration in router.registrations.items():
+            if (existing := registrations.get(topic)) is not None:
+                raise RegistrationError(
+                    f"topic {topic!r} is handled by both {_callable_name(existing.handler)} "
+                    f"and {_callable_name(registration.handler)}"
+                )
+            registrations[topic] = registration
+            outputs += registration.output_topics
+        outputs += router.output_topics
+    if not registrations:
+        raise RegistrationError("a runtime needs at least one handler")
+
+    per_topic = {
+        topic: limit for topic, registration in registrations.items() if (limit := registration.max_concurrency)
+    }
+    limits = (
+        (ConcurrencyMiddleware(max_concurrency=max_concurrency, per_topic=per_topic),)
+        if max_concurrency or per_topic
+        else ()
+    )
+    handlers = {
+        topic: cast(RecordHandler, _chain(limits, "handle", _bind(registration, context)))
+        for topic, registration in registrations.items()
+    }
+
+    def handle(record: ConsumerRecord) -> Awaitable[Sequence[ProducerRecord]]:
+        return handlers[cast(str, record.topic())](record)
+
+    topics = tuple(registrations)
+    if check_topics:
+        stack = (TopicCheckMiddleware((*topics, *outputs)), *stack)
+    key_concurrency = {
+        topic: limit for topic, registration in registrations.items() if (limit := registration.key_concurrency)
+    }
+    return Runtime(config, topics, handle, middleware=stack, key_concurrency=key_concurrency)
+
+
+def _bind(registration: _Registration, context: object) -> RecordHandler:
+    try:
+        return registration.bind(context)
+    except LookupError as exc:
+        raise RegistrationError(f"handler {_callable_name(registration.handler)} {exc}") from exc

potik/core/__init__.py

@@ -0,0 +1,59 @@
+"""The core runtime, a minimal wrapper around `confluent-kafka`.
+
+It works without the typed layer: a handler takes a raw `ConsumerRecord` and returns a list or tuple of `ProducerRecord`s.
+"""
+
+from potik.core.config import Config
+from potik.core.errors import InvalidOffsetError, InvalidOutputError, PartitionsLostError, PermanentError, PotikError
+from potik.core.middleware import BaseMiddleware, ProducerScope, RunScope
+from potik.core.offsets import (
+    OffsetContext,
+    OffsetPolicy,
+    resume_or_earliest,
+    resume_or_fail,
+    resume_or_latest,
+)
+from potik.core.producer import Producer
+from potik.core.replies import PendingReply, ReplyListener
+from potik.core.runtime import Runtime
+from potik.core.types import (
+    ConsumerRecord,
+    Delivery,
+    Headers,
+    PartitionStatus,
+    ProducerRecord,
+    RecordHandler,
+    RecordSender,
+    RuntimeStatus,
+    Transaction,
+)
+
+__all__ = [
+    "BaseMiddleware",
+    "Config",
+    "ConsumerRecord",
+    "Delivery",
+    "Headers",
+    "InvalidOffsetError",
+    "InvalidOutputError",
+    "OffsetContext",
+    "OffsetPolicy",
+    "PartitionStatus",
+    "PartitionsLostError",
+    "PendingReply",
+    "PermanentError",
+    "PotikError",
+    "Producer",
+    "ProducerRecord",
+    "ProducerScope",
+    "RecordHandler",
+    "RecordSender",
+    "ReplyListener",
+    "RunScope",
+    "Runtime",
+    "RuntimeStatus",
+    "Transaction",
+    "resume_or_earliest",
+    "resume_or_fail",
+    "resume_or_latest",
+]

potik/core/committer.py

@@ -0,0 +1,208 @@
+"""Commit handled records' outputs and offsets in transactions, through one transactional producer."""
+
+import logging
+import time
+from collections.abc import Mapping, Sequence
+from functools import partial
+from types import MappingProxyType
+
+import anyio
+from confluent_kafka import KafkaError, KafkaException, Message, Producer, TopicPartition
+
+from potik.core.config import Config
+from potik.core.consumer import ConsumerThread
+from potik.core.errors import PotikError
+from potik.core.middleware import BaseMiddleware, _call_once
+from potik.core.types import ProducerRecord, Transaction
+
+__all__: list[str] = []
+
+logger = logging.getLogger(__name__)
+
+_COMMIT_ATTEMPTS = 3
+# How often an idle producer serves its callbacks (errors, statistics, logs). Committing serves them otherwise.
+_IDLE_POLL = 1.0
+_STALE_EPOCH = frozenset({KafkaError.ILLEGAL_GENERATION, KafkaError.STALE_MEMBER_EPOCH})
+# Reported for the outputs of an aborted transaction, e.g. one repeated for a stale epoch.
+_PURGED = frozenset({KafkaError._PURGE_QUEUE, KafkaError._PURGE_INFLIGHT})
+
+
+class CommitterStoppedError(PotikError):
+    """The run stopped, so handled records can no longer commit."""
+
+
+class Committer:
+    """Commits completed records of all partitions through one transactional producer.
+
+    Each transaction atomically produces a batch's outputs and commits its inputs' next offsets with the consumer's group metadata (KIP-447). With the metadata, the coordinator rejects commits from a fenced or stale group member. The next batch accumulates while a transaction is in flight. `commit_batch_size` caps it and backpressures the handlers.
+
+    A transaction rejected for a stale group epoch is repeated. Any other failed transaction fails the run. Its offsets stay uncommitted, and its inputs replay on the next run.
+
+    Each transaction runs through the `commit` stage of `middleware`.
+    """
+
+    def __init__(self, config: Config, consumer: ConsumerThread, middleware: Sequence[BaseMiddleware] = ()) -> None:
+        self._middleware = middleware
+        self._delivery_error: KafkaError | None = None
+        self._producer = Producer(config._producer_config(self._on_failed_delivery))
+        self._consumer = consumer
+        self._init_timeout = config.transaction_timeout_ms / 1000
+        self._interval = config.commit_interval_ms / 1000
+        self._capacity = config.commit_batch_size
+        # Producer calls block in C, so they run on a worker thread. They are joined, never abandoned, when the awaiting task is cancelled.
+        self._thread = anyio.CapacityLimiter(1)
+        self._scope = anyio.CancelScope()
+        self._offsets: dict[tuple[str, int], int] = {}  # (topic, partition) -> next offset
+        self._outputs: list[ProducerRecord] = []
+        self._records = 0
+        self._has_work = anyio.Event()  # the batch is non-empty
+        self._commit_requested = anyio.Event()  # the batch is full, or a flush waits for it
+        self._batch_taken = anyio.Event()  # the batch was taken by a transaction
+        self._batch_committed = anyio.Event()  # a transaction committed
+        self._batches_taken = 0  # batches taken
+        self._batches_committed = 0  # transactions committed
+        self._next_start = 0.0
+        self._pending_since: float | None = None  # monotonic time the batch became non-empty
+        self._committing_since: float | None = None
+        self._stopped: BaseException | None = None
+
+    async def start(self) -> None:
+        await anyio.to_thread.run_sync(self._producer.init_transactions, self._init_timeout, limiter=self._thread)
+
+    @property
+    def full(self) -> bool:
+        return self._records >= self._capacity
+
+    async def wait_space(self) -> None:
+        while self._records >= self._capacity:
+            self._check()
+            await self._batch_taken.wait()
+
+    @property
+    def oldest_uncommitted_at(self) -> float | None:
+        """The monotonic time when the oldest completed but uncommitted record was added."""
+        return self._committing_since if self._committing_since is not None else self._pending_since
+
+    def add(self, partition: tuple[str, int], next_offset: int, outputs: Sequence[ProducerRecord]) -> None:
+        if not self._offsets:
+            self._pending_since = time.monotonic()
+        self._offsets[partition] = next_offset
+        if outputs:
+            self._outputs.extend(outputs)
+        self._records += 1
+        self._has_work.set()
+        if self._records >= self._capacity:
+            self._commit_requested.set()
+
+    async def flush(self) -> None:
+        """Commit everything added so far, without waiting for the commit interval."""
+        target = self._batches_taken
+        if self._offsets:
+            target += 1
+            self._commit_requested.set()
+        while self._batches_committed < target:
+            self._check()
+            await self._batch_committed.wait()
+
+    def stop(self, reason: BaseException) -> None:
+        """Commit nothing more, and fail current and future waiters."""
+        if self._stopped is None:
+            self._stopped = reason
+        self._scope.cancel()
+        self._batch_taken.set()
+        self._batch_committed.set()
+
+    def _check(self) -> None:
+        if self._stopped is not None:
+            raise CommitterStoppedError("the run stopped, so handled records stay uncommitted") from self._stopped
+
+    async def run(self) -> None:
+        with self._scope:
+            while True:
+                with anyio.move_on_after(_IDLE_POLL):
+                    await self._has_work.wait()
+                if not self._has_work.is_set():
+                    await anyio.to_thread.run_sync(self._producer.poll, 0, limiter=self._thread)
+                    continue
+                delay = self._next_start - anyio.current_time()
+                if delay > 0:
+                    with anyio.move_on_after(delay):
+                        await self._commit_requested.wait()
+                await self._commit()
+
+    async def _commit(self) -> None:
+        offsets = [TopicPartition(topic, partition, offset) for (topic, partition), offset in self._offsets.items()]
+        transaction = Transaction(
+            # Read-only: commit middleware must not commit outputs without their inputs' offsets.
+            offsets=MappingProxyType(self._offsets),
+            outputs=self._outputs,
+        )
+        self._offsets, self._outputs, self._records = {}, [], 0
+        self._committing_since, self._pending_since = self._pending_since, None
+        self._has_work, self._commit_requested = anyio.Event(), anyio.Event()
+        self._batch_taken.set()
+        self._batch_taken = anyio.Event()
+        self._batches_taken += 1
+        self._next_start = anyio.current_time() + self._interval
+        await _call_once(
+            self._middleware, "commit", partial(self._commit_transaction, transaction.offsets, offsets), transaction
+        )
+        self._batches_committed += 1
+        self._committing_since = None
+        self._batch_committed.set()
+        self._batch_committed = anyio.Event()
+
+    async def _commit_transaction(
+        self, committing: Mapping[tuple[str, int], int], offsets: list[TopicPartition], transaction: Transaction
+    ) -> None:
+        if transaction.offsets is not committing:
+            raise RuntimeError("commit middleware must not change a transaction's offsets")
+        outputs = transaction.outputs
+        for attempt in range(1, _COMMIT_ATTEMPTS + 1):
+            # Read the group metadata right before the transaction. The batch holds only owned partitions' offsets, because a revocation commits its partitions' work before it completes. The coordinator rejects a stale epoch, including one bumped by an unrelated assignment change (KIP-848) while this transaction was in preparation. _transact aborts such a transaction, so repeating it with the new epoch is safe. The coordinator checks only the epoch, never which partitions the member owns. A repeat is safe only because a fenced member keeps its stale epoch: librdkafka rejoins with a new one only after the poller serves the lost partitions, which stops the run. During a drain, nothing polls, so the member stays fenced and the coordinator rejects every repeat (tests/integration/test_fencing.py).
+            metadata: bytes = await self._consumer.call(lambda consumer: consumer.consumer_group_metadata())
+            try:
+                await anyio.to_thread.run_sync(self._transact, outputs, offsets, metadata, limiter=self._thread)
+                break
+            except KafkaException as exc:
+                if attempt == _COMMIT_ATTEMPTS or exc.args[0].code() not in _STALE_EPOCH:
+                    raise
+                logger.info("Retrying a transaction that the broker rejected during a rebalance: %s", exc)
+                await anyio.sleep(0.05 * attempt)
+
+    def _transact(self, outputs: Sequence[ProducerRecord], offsets: list[TopicPartition], metadata: bytes) -> None:
+        producer = self._producer
+        # A failed commit notes the first failed delivery of its own attempt, not of an attempt aborted before it.
+        self._delivery_error = None
+        producer.begin_transaction()
+        try:
+            produce = producer.produce
+            for topic, key, value, headers, partition in outputs:
+                try:
+                    produce(topic, value, key, partition, headers=headers)
+                except BufferError:
+                    # The client queue is full. Deliver the queued outputs first. It returns within message.timeout.ms, which librdkafka keeps at or below transaction.timeout.ms.
+                    producer.flush()
+                    produce(topic, value, key, partition, headers=headers)
+            # Transactional API timeouts default to the remaining transaction timeout.
+            producer.send_offsets_to_transaction(offsets, metadata)
+        except BaseException:
+            producer.abort_transaction()
+            raise
+        try:
+            producer.commit_transaction()
+        except KafkaException as exc:
+            # Noted first, so that the note survives an abort that raises. Its error then chains this one.
+            if self._delivery_error is not None:
+                exc.add_note(f"first failed delivery: {self._delivery_error}")
+            # Abort only a commit known to have failed. Leave an unknown outcome to the coordinator, and never retry or abort it.
+            if exc.args[0].txn_requires_abort():
+                producer.abort_transaction()
+            raise
+
+    def _on_failed_delivery(self, error: KafkaError, message: Message) -> None:
+        if self._delivery_error is None and error.code() not in _PURGED:
+            self._delivery_error = error
+
+    async def close(self) -> None:
+        await anyio.to_thread.run_sync(self._producer.close, limiter=self._thread)

potik/core/config.py

@@ -0,0 +1,187 @@
+"""The runtime's limits and client options, and the client settings that the runtime owns."""
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field
+from types import MappingProxyType
+from typing import Any
+from uuid import uuid4
+
+from confluent_kafka import KafkaError, Message
+
+from potik.core.offsets import OffsetPolicy, resume_or_latest
+
+__all__ = [
+    "Config",
+]
+
+# librdkafka prefetches up to 100k records per partition, 64MB in total, in responses of up to 1MB per partition, then backs off fetching for a second. Prefetch less and refill sooner, because the runtime buffers records too.
+_CONSUMER_DEFAULTS = {
+    "queued.min.messages": 2000,
+    "queued.max.messages.kbytes": 16384,
+    "max.partition.fetch.bytes": 131072,
+    "fetch.max.bytes": 8388608,
+    "fetch.queue.backoff.ms": 20,
+}
+# librdkafka backs off reconnecting for up to 10s. After an outage, a client may then stay disconnected long enough to time out the transaction in flight. Reconnect within a second of the broker's return, like the Java clients.
+# librdkafka's producer asks the broker to create a missing topic, which a broker with Kafka's defaults does with one partition. Fail the send instead.
+_CLIENT_DEFAULTS = {"reconnect.backoff.max.ms": 1000, "allow.auto.create.topics": False}
+
+# Client options the runtime's correctness depends on.
+_CONSUMER_OWNED = frozenset(
+    {
+        "bootstrap.servers",
+        "group.id",
+        "group.protocol",
+        "enable.auto.commit",
+        "enable.auto.offset.store",
+        "enable.partition.eof",
+        "isolation.level",
+        "auto.offset.reset",
+        "max.poll.interval.ms",
+    }
+)
+# Client options that any producer's guarantees and delivery reports depend on.
+_PRODUCER_OWNED = frozenset(
+    {
+        "bootstrap.servers",
+        "transactional.id",
+        "enable.idempotence",
+        "acks",
+        "request.required.acks",
+        "delivery.report.only.error",
+        "on_delivery",
+        "dr_cb",
+        "dr_msg_cb",
+    }
+)
+_TRANSACTIONAL_OWNED = _PRODUCER_OWNED | {"transaction.timeout.ms", "metadata.recovery.strategy"}
+
+
+@dataclass(frozen=True, kw_only=True)
+class Config:
+    """Runtime limits and librdkafka client options.
+
+    The runtime owns the client settings that its guarantees depend on, e.g. `isolation.level` and `enable.auto.commit`, and the options cannot override them. Callbacks in the options, e.g. `error_cb`, run on the runtime's client threads, so they must be thread-safe functions, not coroutines.
+
+    The producer's `transactional.id` is `{transactional_id_prefix}-{random}`, new each run. With static membership, i.e. a `group.instance.id` in the options, it derives from the group and instance IDs instead. A restarted instance then aborts the transaction that its crashed run left open, so `read_committed` readers do not wait `transaction_timeout_ms` for it. A second live process with the same instance ID fences the first one's producer.
+
+    Args:
+        bootstrap_servers: The cluster's bootstrap servers.
+        group_id: The consumer group.
+        offset_policy: Picks the start offset of each newly assigned partition (see `OffsetPolicy`). `resume_or_latest`, the default, `resume_or_earliest`, and `resume_or_fail` resume the committed offset, and differ only for a partition without one.
+        max_buffered_bytes: The memory of the fetched records that have not gone to a transaction yet, across all partitions, i.e. those that wait for their handler, run, or wait for an earlier record of their partition (see the `key_concurrency` of `Runtime`). A record counts as its key and value plus 350 bytes for its objects and a typical `traceparent` header, so a UUID key and 1KB of JSON count as about 1.4KB. With consumer lag, the buffers stay full, so this is steady memory, not a peak. Each partition gets an equal share, at least 16KB, so the total can exceed it with more than `max_buffered_bytes / 16384` partitions. Handlers that do I/O need only a few buffered records each. Records arrive one record batch of a partition at a time, and a share smaller than the batches of the topics' producers makes partitions pause and fetch records again, which slows handlers that take about a millisecond or less. It does not count librdkafka's prefetch queue, which potik's defaults limit to about 24MB, handler outputs, or handled records that wait for their transaction, at most two `commit_batch_size` batches. A fetch of records larger than the previous fetch's can exceed a partition's share, by at most the prefetch queue, so the budget is a target, not a hard limit.
+        stop_failed_partitions: Whether a failure of one partition stops it, instead of failing the run. A stopped partition handles nothing past the failed record, which stays uncommitted, while other partitions keep running. The record runs again once the partition is assigned again, e.g. after a restart. A failure is a handler exception that no exception rule handles, or a fetch error, e.g. a committed offset that retention deleted.
+        commit_batch_size: The maximum number of handled records per transaction.
+        commit_interval_ms: The minimum time between the starts of two transactions, unless the batch fills first. A transaction also waits for the previous one to finish, so batches grow with throughput.
+        transaction_timeout_ms: The producer's `transaction.timeout.ms`, after which the broker aborts an open transaction. At least 1000.
+        rebalance_timeout_ms: The longest that the runtime may take to handle an assignment or a revocation, including the revoked partitions' grace period and final commit. Exceeding it fails the run. Must be less than `max_poll_interval_ms`.
+        revocation_grace_ms: How long in-flight handlers of revoked partitions may run before the runtime cancels them. Their records run again on the partitions' next owner. The default, 0, cancels them at once. Must be less than `rebalance_timeout_ms`.
+        shutdown_grace_ms: How long in-flight handlers may run on shutdown before the runtime cancels them. The default, 0, cancels them at once. Plus `rebalance_timeout_ms`, it limits the whole shutdown, including the final commit, and must be less than `max_poll_interval_ms`.
+        max_poll_interval_ms: The consumer's `max.poll.interval.ms`. The runtime keeps polling while handlers run, so handlers may take longer than this.
+        transactional_id_prefix: The start of the producer's `transactional.id`. Give each app its own, so that its ACL on the prefixed `TransactionalId` resource covers no other app's IDs.
+        options: librdkafka options of both clients, e.g. for the connection, TLS, SASL, OAuth, `client.id`, or logging.
+        consumer_options: librdkafka options of the consumer, e.g. `group.remote.assignor`, `group.instance.id`, `client.rack`, or fetch sizes.
+        producer_options: librdkafka options of the transactional producer, e.g. `compression.type` or `linger.ms`.
+
+    Raises:
+        ValueError: If a limit is out of range, or options override a setting that the runtime owns.
+    """
+
+    bootstrap_servers: str
+    group_id: str
+    offset_policy: OffsetPolicy = resume_or_latest
+    max_buffered_bytes: int = 16 * 1024 * 1024
+    stop_failed_partitions: bool = True
+    commit_batch_size: int = 1000
+    commit_interval_ms: int = 100
+    transaction_timeout_ms: int = 10_000
+    rebalance_timeout_ms: int = 60_000
+    revocation_grace_ms: int = 0
+    shutdown_grace_ms: int = 0
+    max_poll_interval_ms: int = 300_000
+    transactional_id_prefix: str = "potik"
+    options: Mapping[str, Any] = field(default_factory=dict)
+    consumer_options: Mapping[str, Any] = field(default_factory=dict)
+    producer_options: Mapping[str, Any] = field(default_factory=dict)
+
+    def __post_init__(self) -> None:
+        if not self.bootstrap_servers or not self.group_id or not self.transactional_id_prefix:
+            raise ValueError("bootstrap_servers, group_id, and transactional_id_prefix must not be empty")
+        if not callable(self.offset_policy):
+            raise TypeError("offset_policy must be callable")
+        for name in (
+            "max_buffered_bytes",
+            "commit_batch_size",
+            "transaction_timeout_ms",
+            "rebalance_timeout_ms",
+            "max_poll_interval_ms",
+        ):
+            value = getattr(self, name)
+            if type(value) is not int or value < 1:
+                raise ValueError(f"{name} must be a positive integer, got {value!r}")
+        for name in ("commit_interval_ms", "revocation_grace_ms", "shutdown_grace_ms"):
+            value = getattr(self, name)
+            if type(value) is not int or value < 0:
+                raise ValueError(f"{name} must be a non-negative integer, got {value!r}")
+        if self.transaction_timeout_ms < 1000:
+            raise ValueError("transaction_timeout_ms must be at least 1000 (librdkafka minimum)")
+        if self.rebalance_timeout_ms >= self.max_poll_interval_ms:
+            raise ValueError("rebalance_timeout_ms must be less than max_poll_interval_ms")
+        if self.revocation_grace_ms >= self.rebalance_timeout_ms:
+            raise ValueError("revocation_grace_ms must be less than rebalance_timeout_ms")
+        # Polling stops during a shutdown. Handlers get shutdown_grace_ms to finish, and cancelled ones get up to rebalance_timeout_ms more. If that exceeds max_poll_interval_ms, the group fences the final commit.
+        if self.shutdown_grace_ms + self.rebalance_timeout_ms >= self.max_poll_interval_ms:
+            raise ValueError("shutdown_grace_ms + rebalance_timeout_ms must be less than max_poll_interval_ms")
+        for name, owned in (
+            ("options", _CONSUMER_OWNED | _TRANSACTIONAL_OWNED),
+            ("consumer_options", _CONSUMER_OWNED),
+            ("producer_options", _TRANSACTIONAL_OWNED),
+        ):
+            options = getattr(self, name)
+            if conflicts := owned.intersection(options):
+                raise ValueError(f"{name} cannot include settings that the runtime manages: {sorted(conflicts)}")
+            object.__setattr__(self, name, MappingProxyType(dict(options)))
+
+    def client_config(self) -> dict[str, Any]:
+        """Return librdkafka options to connect a client of your own to the cluster, e.g. an `AdminClient`."""
+        return _client_config(self.bootstrap_servers, self.options)
+
+    def _consumer_config(self) -> dict[str, Any]:
+        return {
+            **_CONSUMER_DEFAULTS,
+            **self.client_config(),
+            **self.consumer_options,
+            "group.id": self.group_id,
+            "group.protocol": "consumer",  # KIP-848 broker-driven incremental rebalances
+            "enable.auto.commit": False,
+            "enable.auto.offset.store": False,
+            "enable.partition.eof": False,
+            "isolation.level": "read_committed",
+            # Out-of-range positions fail instead of silently skipping records.
+            "auto.offset.reset": "error",
+            "max.poll.interval.ms": self.max_poll_interval_ms,
+        }
+
+    def _producer_config(self, on_failed_delivery: Callable[[KafkaError, Message], None]) -> dict[str, Any]:
+        instance = {**self.options, **self.consumer_options}.get("group.instance.id")
+        # The group's length makes the ID unambiguous. Without it, group "a-b" with instance "c" and group "a" with instance "b-c" would fence each other.
+        suffix = f"{len(self.group_id)}-{self.group_id}-{instance}" if instance else uuid4()
+        return {
+            **_idempotent({**self.client_config(), **self.producer_options}),
+            "transactional.id": f"{self.transactional_id_prefix}-{suffix}",
+            "transaction.timeout.ms": self.transaction_timeout_ms,
+            # Re-bootstrapping while all brokers are down drops the connection that a pending transactional call waits on. That call then blocks forever, ignoring its timeout (librdkafka 2.16.0).
+            "metadata.recovery.strategy": "none",
+            "delivery.report.only.error": True,
+            "on_delivery": on_failed_delivery,
+        }
+
+
+def _client_config(bootstrap_servers: str, options: Mapping[str, Any]) -> dict[str, Any]:
+    """Return the options of a client to `bootstrap_servers`: potik's defaults, overridden by `options`."""
+    return {**_CLIENT_DEFAULTS, **options, "bootstrap.servers": bootstrap_servers}
+
+
+def _idempotent(options: Mapping[str, Any]) -> dict[str, Any]:
+    """Return a producer's `options`, made idempotent and waiting for all in-sync replicas."""
+    return {**options, "enable.idempotence": True, "acks": "all"}