ddkafka 0.0.1__tar.gz

ddkafka-0.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Artem Davydov
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

ddkafka-0.0.1/PKG-INFO

@@ -0,0 +1,169 @@
+Metadata-Version: 2.4
+Name: ddkafka
+Version: 0.0.1
+Summary: Kafka helpers: batch consumer, lock-guarded consumer maker and msgpack producer on top of aiokafka
+Keywords: python,kafka,aiokafka,consumer,producer,ddkafka
+Author: davyddd
+License-Expression: MIT
+License-File: LICENSE
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Typing :: Typed
+Requires-Dist: aiokafka>=0.12,<0.15
+Requires-Dist: msgpack>=1.0,<2
+Requires-Dist: ddutils>=0.2.3,<0.3.0
+Requires-Dist: pydantic>=2.0,<3
+Requires-Python: >=3.11, <3.15
+Project-URL: Repository, https://github.com/davyddd/ddkafka
+Description-Content-Type: text/markdown
+
+# ddkafka
+
+[![pypi](https://img.shields.io/pypi/v/ddkafka.svg)](https://pypi.python.org/pypi/ddkafka)
+[![downloads](https://static.pepy.tech/badge/ddkafka/month)](https://pepy.tech/project/ddkafka)
+[![versions](https://img.shields.io/pypi/pyversions/ddkafka.svg)](https://github.com/davyddd/ddkafka)
+[![codecov](https://codecov.io/gh/davyddd/ddkafka/branch/main/graph/badge.svg)](https://app.codecov.io/github/davyddd/ddkafka)
+[![license](https://img.shields.io/github/license/davyddd/ddkafka.svg)](https://github.com/davyddd/ddkafka/blob/main/LICENSE)
+
+Kafka helpers: batch consumer, lock-guarded consumer maker and msgpack producer on top of aiokafka
+
+## Installation
+
+Install the library using pip:
+```bash
+pip install ddkafka
+```
+
+## Development
+
+The project runs entirely in Docker. Requires Docker and [Fabric](https://www.fabfile.org/) on the host:
+
+```bash
+fab build      # build the dev image
+fab tests      # run pytest
+fab linters    # run ruff (with --fix), ty and complexipy
+fab shell      # IPython inside the container
+fab bash       # bash inside the container
+```
+
+This project was generated from [dd-lib-stub](https://github.com/davyddd/dd-lib-stub);
+run `copier update` to pull in template updates.
+
+## Producer
+
+`BaseKafkaProducerRepository` sends events serialized with msgpack from `model_dump(mode='json')`, keyed by
+`idempotent_key`. One `AIOKafkaProducer` is created lazily per subclass and shared by all its instances.
+
+```python
+from pydantic import BaseModel
+
+from ddkafka.config import ProducerConfig
+from ddkafka.producer import BaseKafkaProducerRepository
+
+
+class ProfileEvent(BaseModel):
+    profile_id: int
+    action: str
+
+    @property
+    def idempotent_key(self) -> str | None:
+        return f'{self.profile_id}:{self.action}'
+
+
+class EventProducerRepository(BaseKafkaProducerRepository):
+    bootstrap_servers = ['kafka:9092']
+    config = ProducerConfig(linger_ms=100)
+
+
+event_producer_repository_impl = EventProducerRepository()
+
+await event_producer_repository_impl.create('profile_events', ProfileEvent(profile_id=1, action='open_app'))
+await event_producer_repository_impl.bulk_create('profile_events', events)
+```
+
+Any object with `idempotent_key` and `model_dump` satisfies the `Producible` protocol; pydantic models only need the
+property.
+
+`create` and `bulk_create` return once the record is in the producer's buffer; aiokafka sends buffered records in
+batches (`linger_ms`, `max_batch_size`), and a record the broker rejects is reported with an ERROR log entry. The
+buffer only reaches Kafka while the process runs, so call `stop()` on shutdown (FastAPI lifespan, dramatiq
+`after_worker_shutdown`), otherwise up to `linger_ms` worth of records is lost on SIGTERM:
+
+```python
+await EventProducerRepository.stop()
+```
+
+## Consumer
+
+`BaseKafkaConsumerRepository[Domain]` reads one partition in batches. Payloads are msgpack dictionaries turned into
+`Domain` with `model_validate`. `get_batches()` polls until the partition runs dry or a poll comes back thinner than
+`batch_size * min_batch_fill_ratio`; `commit()` acknowledges everything yielded so far.
+
+```python
+from ddkafka.consumer import BaseKafkaConsumerRepository
+
+
+class ProfileEventConsumerRepository(BaseKafkaConsumerRepository[ProfileEvent]):
+    bootstrap_servers = ['kafka:9092']
+    topic = 'profile_events'
+    group_id = 'analytics_context'
+    batch_size = 100
+
+
+repo = ProfileEventConsumerRepository(partition=0)
+await repo.start()
+try:
+    async for batch in repo.get_batches():
+        await process(batch)
+        await repo.commit()
+finally:
+    await repo.stop()
+```
+
+`get_batches(timestamp_threshold_minutes=5)` yields only messages older than five minutes, for consumers that
+deliberately lag behind the producer.
+
+Tunables are class attributes: `batch_size`, `poll_timeout_ms`, `min_batch_fill_ratio` and `config`
+(`ConsumerConfig`, passed to `AIOKafkaConsumer` as keyword arguments). Missing or mistyped attributes fail at
+class definition.
+
+A record that cannot be deserialized (a schema the consumer does not know, a payload written by something else)
+raises `MessageDeserializationError` with the topic, partition and offset, and the partition stays at that offset
+until the group's offset is moved past it. A consumer whose stream can afford to lose such a record sets
+`skip_undeserializable = True`: the record is logged with its offset and dropped, the rest of the batch goes on.
+
+## Consumer maker
+
+`KafkaConsumerRepositoryMaker` wraps a consumer in a distributed lock, so one partition is processed by one
+worker at a time across the whole deployment. The lock is anything satisfying `Lockable` (`acquire`, `release`,
+`extend`, `timeout`), for example `RedisLock` from [ddredis](https://github.com/davyddd/ddredis). Entering acquires
+the lock and starts the consumer; every yielded batch is followed by a checkpoint (lock heartbeat + offset commit);
+leaving commits, stops and releases. A lock that is already held raises from `acquire`, a consumer that fails to
+start raises `ConsumerStartError` after releasing the lock.
+
+```python
+from ddredis.lock import RedisLock
+
+from ddkafka.consumer_maker import KafkaConsumerRepositoryMaker
+
+async with KafkaConsumerRepositoryMaker(
+    consumer_class=ProfileEventConsumerRepository,
+    partition=0,
+    lock_class=RedisLock,
+    lock_kwargs={'redis_client': redis_client, 'timeout': 300},
+) as maker:
+    async for batch in maker.get_batches():
+        await process(batch)
+```
+
+Every batch is committed as a whole after the caller is done with it; a batch the caller fails on is not committed
+and is read again next time, so processing must tolerate repeated records (`idempotent_key`). `batch_size` bounds
+how much is reprocessed. `maker.checkpoint()` commits the current batch, so call it only once the batch is processed.
+
+With `stream=True` the consumer does not return when the topic runs dry: it keeps the lock alive and polls
+again after `stream_round_pause`, so a cron that starts it acts as a supervisor and messages are picked up as
+they land. The pace must fit the lock: `stream_round_pause` plus the consumer's poll timeout may not exceed half
+of the lock timeout, otherwise the maker refuses at construction.

ddkafka-0.0.1/README.md

@@ -0,0 +1,147 @@
+# ddkafka
+
+[![pypi](https://img.shields.io/pypi/v/ddkafka.svg)](https://pypi.python.org/pypi/ddkafka)
+[![downloads](https://static.pepy.tech/badge/ddkafka/month)](https://pepy.tech/project/ddkafka)
+[![versions](https://img.shields.io/pypi/pyversions/ddkafka.svg)](https://github.com/davyddd/ddkafka)
+[![codecov](https://codecov.io/gh/davyddd/ddkafka/branch/main/graph/badge.svg)](https://app.codecov.io/github/davyddd/ddkafka)
+[![license](https://img.shields.io/github/license/davyddd/ddkafka.svg)](https://github.com/davyddd/ddkafka/blob/main/LICENSE)
+
+Kafka helpers: batch consumer, lock-guarded consumer maker and msgpack producer on top of aiokafka
+
+## Installation
+
+Install the library using pip:
+```bash
+pip install ddkafka
+```
+
+## Development
+
+The project runs entirely in Docker. Requires Docker and [Fabric](https://www.fabfile.org/) on the host:
+
+```bash
+fab build      # build the dev image
+fab tests      # run pytest
+fab linters    # run ruff (with --fix), ty and complexipy
+fab shell      # IPython inside the container
+fab bash       # bash inside the container
+```
+
+This project was generated from [dd-lib-stub](https://github.com/davyddd/dd-lib-stub);
+run `copier update` to pull in template updates.
+
+## Producer
+
+`BaseKafkaProducerRepository` sends events serialized with msgpack from `model_dump(mode='json')`, keyed by
+`idempotent_key`. One `AIOKafkaProducer` is created lazily per subclass and shared by all its instances.
+
+```python
+from pydantic import BaseModel
+
+from ddkafka.config import ProducerConfig
+from ddkafka.producer import BaseKafkaProducerRepository
+
+
+class ProfileEvent(BaseModel):
+    profile_id: int
+    action: str
+
+    @property
+    def idempotent_key(self) -> str | None:
+        return f'{self.profile_id}:{self.action}'
+
+
+class EventProducerRepository(BaseKafkaProducerRepository):
+    bootstrap_servers = ['kafka:9092']
+    config = ProducerConfig(linger_ms=100)
+
+
+event_producer_repository_impl = EventProducerRepository()
+
+await event_producer_repository_impl.create('profile_events', ProfileEvent(profile_id=1, action='open_app'))
+await event_producer_repository_impl.bulk_create('profile_events', events)
+```
+
+Any object with `idempotent_key` and `model_dump` satisfies the `Producible` protocol; pydantic models only need the
+property.
+
+`create` and `bulk_create` return once the record is in the producer's buffer; aiokafka sends buffered records in
+batches (`linger_ms`, `max_batch_size`), and a record the broker rejects is reported with an ERROR log entry. The
+buffer only reaches Kafka while the process runs, so call `stop()` on shutdown (FastAPI lifespan, dramatiq
+`after_worker_shutdown`), otherwise up to `linger_ms` worth of records is lost on SIGTERM:
+
+```python
+await EventProducerRepository.stop()
+```
+
+## Consumer
+
+`BaseKafkaConsumerRepository[Domain]` reads one partition in batches. Payloads are msgpack dictionaries turned into
+`Domain` with `model_validate`. `get_batches()` polls until the partition runs dry or a poll comes back thinner than
+`batch_size * min_batch_fill_ratio`; `commit()` acknowledges everything yielded so far.
+
+```python
+from ddkafka.consumer import BaseKafkaConsumerRepository
+
+
+class ProfileEventConsumerRepository(BaseKafkaConsumerRepository[ProfileEvent]):
+    bootstrap_servers = ['kafka:9092']
+    topic = 'profile_events'
+    group_id = 'analytics_context'
+    batch_size = 100
+
+
+repo = ProfileEventConsumerRepository(partition=0)
+await repo.start()
+try:
+    async for batch in repo.get_batches():
+        await process(batch)
+        await repo.commit()
+finally:
+    await repo.stop()
+```
+
+`get_batches(timestamp_threshold_minutes=5)` yields only messages older than five minutes, for consumers that
+deliberately lag behind the producer.
+
+Tunables are class attributes: `batch_size`, `poll_timeout_ms`, `min_batch_fill_ratio` and `config`
+(`ConsumerConfig`, passed to `AIOKafkaConsumer` as keyword arguments). Missing or mistyped attributes fail at
+class definition.
+
+A record that cannot be deserialized (a schema the consumer does not know, a payload written by something else)
+raises `MessageDeserializationError` with the topic, partition and offset, and the partition stays at that offset
+until the group's offset is moved past it. A consumer whose stream can afford to lose such a record sets
+`skip_undeserializable = True`: the record is logged with its offset and dropped, the rest of the batch goes on.
+
+## Consumer maker
+
+`KafkaConsumerRepositoryMaker` wraps a consumer in a distributed lock, so one partition is processed by one
+worker at a time across the whole deployment. The lock is anything satisfying `Lockable` (`acquire`, `release`,
+`extend`, `timeout`), for example `RedisLock` from [ddredis](https://github.com/davyddd/ddredis). Entering acquires
+the lock and starts the consumer; every yielded batch is followed by a checkpoint (lock heartbeat + offset commit);
+leaving commits, stops and releases. A lock that is already held raises from `acquire`, a consumer that fails to
+start raises `ConsumerStartError` after releasing the lock.
+
+```python
+from ddredis.lock import RedisLock
+
+from ddkafka.consumer_maker import KafkaConsumerRepositoryMaker
+
+async with KafkaConsumerRepositoryMaker(
+    consumer_class=ProfileEventConsumerRepository,
+    partition=0,
+    lock_class=RedisLock,
+    lock_kwargs={'redis_client': redis_client, 'timeout': 300},
+) as maker:
+    async for batch in maker.get_batches():
+        await process(batch)
+```
+
+Every batch is committed as a whole after the caller is done with it; a batch the caller fails on is not committed
+and is read again next time, so processing must tolerate repeated records (`idempotent_key`). `batch_size` bounds
+how much is reprocessed. `maker.checkpoint()` commits the current batch, so call it only once the batch is processed.
+
+With `stream=True` the consumer does not return when the topic runs dry: it keeps the lock alive and polls
+again after `stream_round_pause`, so a cron that starts it acts as a supervisor and messages are picked up as
+they land. The pace must fit the lock: `stream_round_pause` plus the consumer's poll timeout may not exceed half
+of the lock timeout, otherwise the maker refuses at construction.

ddkafka-0.0.1/ddkafka/__init__.py

@@ -0,0 +1,3 @@
+from . import config, consumer, consumer_maker, producer
+
+__all__ = ('config', 'consumer', 'consumer_maker', 'producer')

ddkafka-0.0.1/ddkafka/config.py

@@ -0,0 +1,149 @@
+from typing import Literal, Self
+
+from pydantic import BaseModel, ConfigDict, Field, model_validator
+
+
+class ProducerConfig(BaseModel):
+    """Keyword arguments passed to `aiokafka.AIOKafkaProducer` as they are (`model_dump()`).
+
+    Any other `AIOKafkaProducer` keyword argument (`compression_type`, `max_request_size`, ...) can be
+    passed as well and is forwarded untouched.
+
+    The defaults assume a cluster set up for durability (`default.replication.factor=3`,
+    `min.insync.replicas=2`): `acks='all'` is the client half of that guarantee, the rest is batching
+    tuned for a steady event stream rather than single sends.
+    """
+
+    model_config = ConfigDict(frozen=True, extra='allow')
+
+    acks: Literal[0, 1, 'all'] = Field(
+        default='all',
+        description=(
+            "How many brokers must confirm a record before `send` resolves. `'all'` waits for every in-sync "
+            "replica, which is the only setting where the cluster's `min.insync.replicas` applies: with "
+            'replication factor 3 and min ISR 2 a confirmed record survives the loss of one broker. `1` is '
+            'confirmed by the leader alone and is lost if the leader dies before a follower copies it; `0` '
+            'does not wait at all. Replication and min ISR are cluster settings chosen by devops; overriding '
+            '`acks` silently changes the delivery guarantee they set up, so do it knowingly. Note that '
+            "`'all'` needs the topic to actually have at least `min.insync.replicas` replicas: a topic "
+            'created with one replica rejects every write with `NotEnoughReplicas`.'
+        ),
+    )
+    enable_idempotence: bool = Field(
+        default=False,
+        description=(
+            'Exactly one copy of each record even when a send is retried after a broker failure. Requires '
+            "`acks='all'` and costs a sequence number per partition. Off by default: records are keyed with "
+            '`idempotent_key`, so a duplicate is cheap to drop on the consumer side, and consumers are '
+            'at-least-once anyway.'
+        ),
+    )
+    linger_ms: int = Field(
+        default=50,
+        description=(
+            'How long the producer waits for more records before sending a batch. 0 sends each record at '
+            'once; a small delay lets the records of one request share a batch, which is what makes '
+            '`bulk_create` cheap.'
+        ),
+    )
+    max_batch_size: int = Field(
+        default=32_000,
+        description='Bytes per partition batch. A batch is sent when it reaches this size or `linger_ms` passes.',
+    )
+    retry_backoff_ms: int = Field(
+        default=500,
+        description='Pause before retrying a failed request (leader change, transient broker error).',
+    )
+    metadata_max_age_ms: int = Field(
+        default=30_000,
+        description=(
+            'How often the producer refreshes cluster metadata even without errors, so a new partition or a '
+            'moved leader is picked up within this time.'
+        ),
+    )
+    request_timeout_ms: int = Field(
+        default=5_000,
+        description=(
+            "How long a produce request may wait for the broker's response before it fails. With `acks='all'` "
+            "this includes the followers catching up, so keep it above the broker's `replica.lag.time.max.ms`."
+        ),
+    )
+
+    @model_validator(mode='after')
+    def _idempotence_requires_acks_all(self) -> Self:
+        if self.enable_idempotence and self.acks != 'all':
+            raise ValueError("`enable_idempotence=True` requires `acks='all'`")
+        return self
+
+
+class ConsumerConfig(BaseModel):
+    """Keyword arguments passed to `aiokafka.AIOKafkaConsumer` as they are (`model_dump()`).
+
+    Any other `AIOKafkaConsumer` keyword argument (`max_poll_records`, `isolation_level`, ...) can be
+    passed as well and is forwarded untouched.
+
+    The defaults make a batch consumer: offsets are committed by the repository after a batch is
+    processed (at-least-once) and fetches wait for a meaningful amount of data.
+
+    The repository assigns its partition explicitly (`assign`, not `subscribe`), so the consumer never
+    joins the group: `session_timeout_ms`, `heartbeat_interval_ms` and the other group membership
+    settings have no effect here and are left out. They can still be passed if a consumer subscribes.
+    """
+
+    model_config = ConfigDict(frozen=True, extra='allow')
+
+    auto_offset_reset: Literal['earliest', 'latest', 'none'] = Field(
+        default='earliest',
+        description=(
+            'Where a group starts when it has no committed offset for a partition (first run, or the '
+            "committed offset was deleted by retention): `'earliest'` reads the whole retained history, "
+            "`'latest'` only new records, `'none'` raises and leaves the decision to the caller."
+        ),
+    )
+    # Only False is accepted. Auto commit acknowledges the consumer's position, the offset right after the
+    # last record `getmany` returned, every `auto_commit_interval_ms`, regardless of what the application
+    # has done with those records. That breaks everything the repository relies on: `commit()` after a
+    # processed batch (at-least-once) becomes meaningless, records filtered out by
+    # `timestamp_threshold_minutes` get committed past and are never read again, and a partition that
+    # must stay on an undeserializable record is silently moved past it.
+    enable_auto_commit: Literal[False] = Field(
+        default=False,
+        description=(
+            'Must stay False: `BaseKafkaConsumerRepository.commit` acknowledges offsets after a batch is '
+            'processed. Auto commit would acknowledge records as soon as they are fetched, and a crash in the '
+            'middle of a batch would lose them.'
+        ),
+    )
+    request_timeout_ms: int = Field(
+        default=25_000,
+        description='How long a request to the broker may wait for a response.',
+    )
+    retry_backoff_ms: int = Field(
+        default=1_000,
+        description='Pause before retrying a failed request (coordinator moved, transient broker error).',
+    )
+    fetch_min_bytes: int = Field(
+        default=120_000,
+        description=(
+            'The broker holds a fetch until this many bytes are available or `fetch_max_wait_ms` passes. '
+            'A high value turns a trickle of small records into fewer, fuller batches.'
+        ),
+    )
+    fetch_max_bytes: int = Field(
+        default=125_000_000,
+        description='Upper bound on one fetch response across all partitions the consumer reads.',
+    )
+    fetch_max_wait_ms: int = Field(
+        default=1_000,
+        description='How long the broker may hold a fetch waiting for `fetch_min_bytes`.',
+    )
+    max_partition_fetch_bytes: int = Field(
+        default=25_000_000,
+        description=(
+            'Upper bound on the data returned for one partition in one fetch. Must be at least the largest '
+            'record in the topic, otherwise the consumer is stuck on it.'
+        ),
+    )
+
+
+__all__ = ('ConsumerConfig', 'ProducerConfig')

ddkafka-0.0.1/ddkafka/consumer.py

@@ -0,0 +1,232 @@
+import time
+from abc import ABC
+from collections.abc import AsyncIterator, Iterator
+from logging import getLogger
+from typing import Any, ClassVar, Generic, Protocol, Self, TypeVar
+
+import msgpack
+from aiokafka import AIOKafkaConsumer, TopicPartition
+from ddutils.class_helpers import get_generic_base_argument, require_class_attribute
+
+from ddkafka.config import ConsumerConfig
+
+logger = getLogger(__name__)
+
+
+class Deserializable(Protocol):
+    @classmethod
+    def model_validate(cls, data: dict) -> Self: ...
+
+
+DomainT = TypeVar('DomainT', bound=Deserializable)
+
+
+class MessageDeserializationError(Exception):
+    """A record that cannot be turned into the domain type; names the record so the offset can be skipped."""
+
+    def __init__(self, topic: str, partition: int, offset: int):
+        self.topic = topic
+        self.partition = partition
+        self.offset = offset
+        super().__init__(f'Cannot deserialize the message at {topic}[{partition}] offset {offset}')
+
+
+class BaseKafkaConsumerRepository(ABC, Generic[DomainT]):
+    """
+    Base async Kafka consumer repository: one instance reads one partition of `topic` in batches.
+
+    Messages are msgpack payloads turned into the domain type with `model_validate`. `get_batches`
+    polls until the partition runs dry or a poll comes back too thin (`min_batch_fill_ratio`),
+    and `commit` acknowledges every batch yielded so far, including the one being processed.
+
+    A record that cannot be deserialized (a schema the consumer does not know, a foreign payload)
+    raises `MessageDeserializationError` with its offset, and the partition stays at that offset
+    until someone moves it. A consumer whose stream can afford to lose such a record sets
+    `skip_undeserializable = True`: the record is logged with its offset and dropped instead.
+
+    Class Attributes:
+        bootstrap_servers: Kafka servers address
+        topic: Topic name
+        group_id: Consumer group ID
+        batch_size: Max records per poll
+        poll_timeout_ms: Timeout for poll operation in milliseconds
+        min_batch_fill_ratio: Minimum batch fill ratio to continue (0.0-1.0)
+        skip_undeserializable: Drop records that cannot be deserialized instead of raising
+        config: Consumer configuration
+
+    Example:
+        class ProfileEventConsumerRepository(BaseKafkaConsumerRepository[ProfileEvent]):
+            bootstrap_servers = ['kafka:9092']
+            topic = 'profile_events'
+            group_id = 'analytics_context'
+            batch_size = 100
+    """
+
+    bootstrap_servers: ClassVar[list[str]]
+    topic: ClassVar[str]
+    group_id: ClassVar[str]
+    batch_size: ClassVar[int] = 1_000
+    poll_timeout_ms: ClassVar[int] = 10_000
+    min_batch_fill_ratio: ClassVar[float] = 0.1
+    skip_undeserializable: ClassVar[bool] = False
+    config: ClassVar[ConsumerConfig] = ConsumerConfig()
+
+    _domain_class: ClassVar[type[Deserializable]]
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+
+        domain_class = get_generic_base_argument(cls, BaseKafkaConsumerRepository)
+        if domain_class is None or isinstance(domain_class, TypeVar):
+            raise TypeError(
+                f'{cls.__name__} must specify domain type: class {cls.__name__}(BaseKafkaConsumerRepository[YourDomain])'
+            )
+        cls._domain_class = domain_class
+
+        require_class_attribute(cls, 'bootstrap_servers', list)
+        require_class_attribute(cls, 'topic', str)
+        require_class_attribute(cls, 'group_id', str)
+        if require_class_attribute(cls, 'batch_size', int) < 1:
+            raise ValueError(f'`batch_size` class attribute of {cls.__name__} must be positive')
+        if require_class_attribute(cls, 'poll_timeout_ms', int) < 0:
+            raise ValueError(f'`poll_timeout_ms` class attribute of {cls.__name__} must not be negative')
+        if not (0 <= require_class_attribute(cls, 'min_batch_fill_ratio', float) <= 1):
+            raise ValueError(f'`min_batch_fill_ratio` class attribute of {cls.__name__} must be between 0 and 1')
+        require_class_attribute(cls, 'skip_undeserializable', bool)
+        require_class_attribute(cls, 'config', ConsumerConfig)
+
+    def __init__(self, partition: int):
+        self._partition = partition
+        self._topic_partition = TopicPartition(self.topic, partition)
+        self._processed_offsets: dict[TopicPartition, int] = {}
+        self._create_consumer()
+
+    def _deserialize(self, records: list[Any]) -> Iterator[DomainT]:
+        # Done here rather than through aiokafka's `value_deserializer`, where a failure surfaces
+        # from `getmany` without the record it happened on.
+        for record in records:
+            try:
+                yield self._domain_class.model_validate(msgpack.loads(record.value))  # type: ignore[misc]
+            except Exception as e:
+                if not self.skip_undeserializable:
+                    raise MessageDeserializationError(self.topic, self._partition, record.offset) from e
+                logger.error(
+                    {
+                        'message': 'KAFKA_CONSUMER: Skipped a message that cannot be deserialized',
+                        'topic': self.topic,
+                        'partition': self._partition,
+                        'offset': record.offset,
+                        'error': repr(e),
+                    }
+                )
+
+    def _create_consumer(self) -> None:
+        logger.info(
+            {
+                'message': 'KAFKA_CONSUMER: Creating consumer',
+                'topic': self.topic,
+                'partition': self._partition,
+                'group_id': self.group_id,
+            }
+        )
+
+        self._consumer = AIOKafkaConsumer(
+            bootstrap_servers=self.bootstrap_servers,
+            group_id=self.group_id,
+            **self.config.model_dump(),
+        )
+        self._consumer.assign([self._topic_partition])
+
+    async def start(self) -> None:
+        await self._consumer.start()
+
+        logger.info(
+            {'message': 'KAFKA_CONSUMER: Started', 'topic': self.topic, 'partition': self._partition, 'group_id': self.group_id}
+        )
+
+    async def stop(self) -> None:
+        await self._consumer.stop()
+        logger.info(
+            {'message': 'KAFKA_CONSUMER: Stopped', 'topic': self.topic, 'partition': self._partition, 'group_id': self.group_id}
+        )
+
+    async def _get_offset_limit(self, timestamp_threshold_minutes: int) -> int | None:
+        """The first offset younger than the threshold, or None when the partition has no such message."""
+        threshold_ms = int(time.time() * 1000) - (timestamp_threshold_minutes * 60 * 1000)
+        offsets_for_times = await self._consumer.offsets_for_times({self._topic_partition: threshold_ms})
+        offset_and_timestamp = offsets_for_times.get(self._topic_partition)
+        return offset_and_timestamp.offset if offset_and_timestamp else None
+
+    async def _poll(self, offset_limit: int | None) -> list[Any]:
+        """One `getmany` for the partition; with `offset_limit` only the records before it."""
+        records_by_partition = await self._consumer.getmany(
+            self._topic_partition, timeout_ms=self.poll_timeout_ms, max_records=self.batch_size
+        )
+        records = records_by_partition.get(self._topic_partition, [])
+
+        logger.info(
+            {
+                'message': 'KAFKA_CONSUMER: Retrieved batch',
+                'topic': self.topic,
+                'partition': self._partition,
+                'records_count': len(records),
+            }
+        )
+
+        if offset_limit is not None:
+            records = [record for record in records if record.offset < offset_limit]
+        return records
+
+    async def get_batches(self, timestamp_threshold_minutes: int | None = None) -> AsyncIterator[tuple[DomainT, ...]]:
+        """
+        Returns an async iterator over batches of domain entities.
+
+        Args:
+            timestamp_threshold_minutes: Only messages OLDER than this threshold (in minutes) are included.
+                                         Defaults to None (no filtering).
+        """
+        offset_limit: int | None = None
+        if timestamp_threshold_minutes is not None:
+            offset_limit = await self._get_offset_limit(timestamp_threshold_minutes)
+
+        while True:
+            records = await self._poll(offset_limit)
+            if not records:
+                break
+
+            values = tuple(self._deserialize(records))
+            records_count, last_offset = len(records), records[-1].offset
+            # The raw records are not needed past this point; dropping them before the batch is
+            # handed out frees the msgpack payloads while the caller works on the domain objects.
+            del records
+
+            # Recorded before the batch is handed out, so `commit` right after processing it
+            # acknowledges this batch and not the previous one. The caller is responsible for not
+            # committing a batch it failed on (the maker skips the commit on an exceptional exit).
+            self._processed_offsets[self._topic_partition] = last_offset + 1
+
+            if values:  # a batch of skipped records only is nothing to hand out
+                yield values
+
+            # A poll that came back mostly empty means the partition is nearly drained: finish
+            # after this batch instead of waiting `poll_timeout_ms` for a handful more.
+            if records_count < self.batch_size * self.min_batch_fill_ratio:
+                break
+
+    async def commit(self) -> None:
+        if self._processed_offsets:
+            await self._consumer.commit(self._processed_offsets)
+
+        logger.info(
+            {
+                'message': 'KAFKA_CONSUMER: Committed offsets',
+                'topic': self.topic,
+                'partition': self._partition,
+                'offsets': str(self._processed_offsets),
+            }
+        )
+
+        self._processed_offsets.clear()
+
+
+__all__ = ('BaseKafkaConsumerRepository', 'Deserializable', 'MessageDeserializationError')

ddkafka-0.0.1/ddkafka/consumer_maker.py

@@ -0,0 +1,177 @@
+import asyncio
+from collections.abc import AsyncIterator
+from datetime import timedelta
+from typing import Any, Generic, Protocol, Self, TypeVar
+
+from ddkafka.consumer import BaseKafkaConsumerRepository, Deserializable
+
+
+class Lockable(Protocol):
+    # How long one `extend` is worth. A streaming consumer has to beat faster than this, so the
+    # contract has to say it out loud rather than leave it to whoever wires the lock up.
+    timeout: float
+
+    async def acquire(self) -> None: ...
+
+    async def release(self) -> None: ...
+
+    async def extend(self) -> None: ...
+
+
+class ConsumerStartError(Exception): ...
+
+
+# How much of the lock's timeout a streaming consumer is allowed to spend between two beats.
+# The rest is headroom for the one thing the check cannot see — how long the caller takes over
+# a batch, which is bounded by nothing here.
+MAX_BEAT_SHARE_OF_LOCK_TIMEOUT = 0.5
+
+DomainT = TypeVar('DomainT', bound=Deserializable)
+
+
+class KafkaConsumerRepositoryMaker(Generic[DomainT]):
+    """
+    Context manager for a Kafka consumer guarded by a distributed lock.
+
+    The lock (anything satisfying `Lockable`, e.g. `ddredis.lock.RedisLock`) is keyed by group,
+    topic and partition, so only one consumer per partition runs across all workers. Entering
+    acquires the lock and starts the consumer; every batch yielded by `get_batches` is followed
+    by a checkpoint (lock heartbeat + offset commit); leaving commits, stops and releases. An exit
+    caused by an exception does not commit, so the batch being processed is read again next time.
+
+    `stream` decides what happens when the topic runs dry. Without it the consumer drains what
+    is waiting and returns, which is what a periodic sync wants. With it the consumer stays on
+    its partition indefinitely, and the cron that starts it becomes a supervisor rather than a
+    schedule: every tick tries to start one, the lock turns all but the first away, and a tick
+    only does anything once the running consumer has died. Recovery from a death that leaves
+    the lock held (a killed worker) takes the lock's own `timeout`, so a streaming consumer
+    wants that timeout measured in tens of seconds, not minutes.
+
+    `stream_round_pause` is how long it waits between rounds — the topic's chance to refill,
+    and the ceiling on how long a message can sit once it has arrived. A consumer whose topic
+    trickles wants it longer, one whose latency is the point wants it short. It is bounded from
+    above by the lock: pause plus poll timeout is the longest the lock can go unextended, and a
+    pace that does not fit is refused at construction.
+
+    Example:
+        async with KafkaConsumerRepositoryMaker(
+            consumer_class=EventConsumerRepository,
+            partition=0,
+            lock_class=RedisLock,
+            lock_kwargs={'redis_client': redis_client, 'timeout': 300}
+        ) as maker:
+            async for batch in maker.get_batches():
+                process(batch)
+    """
+
+    def __init__(
+        self,
+        consumer_class: type[BaseKafkaConsumerRepository[DomainT]],
+        partition: int,
+        lock_class: type[Lockable],
+        lock_kwargs: dict[str, Any],
+        stream: bool = False,
+        stream_round_pause: timedelta = timedelta(seconds=1),
+    ):
+        lock_key = self.build_lock_key(consumer_class, partition)
+        self.lock = lock_class(key=lock_key, **lock_kwargs)  # type: ignore[call-arg]
+        self._consumer_class = consumer_class
+        self._partition = partition
+        self._stream = stream
+        self._stream_round_pause = stream_round_pause
+
+        if stream:
+            self._check_beat_fits_lock()
+
+    def _check_beat_fits_lock(self) -> None:
+        """Refuse a pace that would let the lock die under a consumer that is still running.
+
+        A streaming consumer beats once per round, and a round is the pause plus a poll that may
+        wait its whole timeout out. Let that exceed the lock and the lock expires between beats:
+        the next tick then starts a second consumer on the same partition, both read it and both
+        commit, and nothing in the running system says so.
+        """
+        poll_timeout = self._consumer_class.poll_timeout_ms / 1000
+        beat_ceiling = self._stream_round_pause.total_seconds() + poll_timeout
+        allowed = self.lock.timeout * MAX_BEAT_SHARE_OF_LOCK_TIMEOUT
+        if beat_ceiling > allowed:
+            raise ValueError(
+                f'{self._consumer_class.__name__} streams with up to {beat_ceiling:g}s between lock '
+                f'heartbeats (stream_round_pause {self._stream_round_pause.total_seconds():g}s + poll '
+                f'timeout {poll_timeout:g}s), which does not fit the '
+                f'{MAX_BEAT_SHARE_OF_LOCK_TIMEOUT:g} share of the lock timeout it is allowed '
+                f'({allowed:g}s of {self.lock.timeout:g}s). Shorten the pause or raise the lock timeout.'
+            )
+
+    @staticmethod
+    def build_lock_key(consumer_class: type[BaseKafkaConsumerRepository[Any]], partition: int) -> str:
+        return f'{consumer_class.group_id}_{consumer_class.topic}_partition_{partition}'
+
+    async def get_batches(self) -> AsyncIterator[tuple[DomainT, ...]]:
+        while True:
+            async for batch in self.consumer_repo.get_batches():
+                yield batch
+                await self.checkpoint()
+
+            if not self._stream:
+                return
+
+            # The topic went quiet. Returning here would hand the partition back and leave the
+            # next message to whatever starts a consumer again — a cron tick, and with it a
+            # whole stage of the send path's latency. Instead the lock is kept alive and the
+            # consumer asked once more: its own poll timeout is the wait, so an idle partition
+            # costs one call every few seconds and a message is taken up as it lands.
+            #
+            # A checkpoint rather than a bare `extend`, so the heartbeat has one call site: a
+            # round that found nothing yields no batch, so without this an idle partition would
+            # have no heartbeat at all and the lock would expire under a running consumer.
+            await self.checkpoint()
+
+            # The pause lets records accumulate into fuller batches and is the floor under a
+            # poll that returns instantly, which would otherwise spin.
+            await asyncio.sleep(self._stream_round_pause.total_seconds())
+
+    async def checkpoint(self) -> None:
+        """
+        Extends the lock and commits every batch yielded so far, including the one being processed.
+
+        `get_batches` calls it after the caller is done with a batch. Calling it yourself from
+        inside the loop commits the current batch before it is processed: if the processing then
+        fails, those records are gone.
+        """
+        await self.lock.extend()
+        await self.consumer_repo.commit()
+
+    async def __aenter__(self) -> Self:
+        await self.lock.acquire()
+
+        # Anything that fails after the lock is taken has to give it back, or the partition stays
+        # locked until the lock's timeout: the constructor (aiokafka needs a running loop) as much
+        # as `start` (no broker).
+        try:
+            self.consumer_repo = self._consumer_class(partition=self._partition)
+            await self.consumer_repo.start()
+        except Exception as e:  # noqa: BLE001
+            await self._close()
+            raise ConsumerStartError('Failed to start consumer') from e
+
+        return self
+
+    async def _close(self) -> None:
+        try:
+            if hasattr(self, 'consumer_repo'):
+                await self.consumer_repo.stop()
+        finally:
+            await self.lock.release()
+
+    async def __aexit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:
+        # The offsets of a yielded batch are already recorded, so an exit caused by an error in
+        # the caller must not commit: the batch it failed on stays in the topic for the next run.
+        try:
+            if exc_type is None:
+                await self.consumer_repo.commit()
+        finally:
+            await self._close()
+
+
+__all__ = ('ConsumerStartError', 'KafkaConsumerRepositoryMaker', 'Lockable')

ddkafka-0.0.1/ddkafka/producer.py

@@ -0,0 +1,137 @@
+import asyncio
+from abc import ABC
+from asyncio import Future
+from functools import partial
+from logging import getLogger
+from typing import Any, ClassVar, Protocol
+
+import msgpack
+from aiokafka import AIOKafkaProducer
+from ddutils.class_helpers import require_class_attribute
+
+from ddkafka.config import ProducerConfig
+
+logger = getLogger(__name__)
+
+
+class Producible(Protocol):
+    @property
+    def idempotent_key(self) -> str | None: ...
+
+    def model_dump(self, *args: Any, **kwargs: Any) -> dict: ...
+
+
+def serialize_value(event: Producible) -> bytes:
+    return msgpack.dumps(event.model_dump(mode='json'))
+
+
+def serialize_key(key: Any) -> bytes | None:
+    # aiokafka hands the key to the serializer even when it is None, and a key of b'None' would
+    # route every keyless event to one partition.
+    return None if key is None else str(key).encode('utf-8')
+
+
+def _log_delivery_failure(topic: str, key: Any, future: Future) -> None:
+    # `send` resolves when the record is buffered; the broker's answer lands in this future later.
+    # Nobody awaits it, so without this callback a rejected record (`NotEnoughReplicas`, a timeout)
+    # would only show up as "Future exception was never retrieved" from asyncio.
+    if future.cancelled():
+        return
+    error = future.exception()
+    if error is not None:
+        logger.error({'message': 'KAFKA_PRODUCER: Delivery failed', 'topic': topic, 'key': key, 'error': repr(error)})
+
+
+class BaseKafkaProducerRepository(ABC):
+    """
+    Base async Kafka producer.
+
+    One `AIOKafkaProducer` is created lazily per subclass, once per process, and shared by all its
+    instances, so a producer repository can be instantiated freely. Events are serialized with msgpack from
+    `model_dump(mode='json')` and keyed by `idempotent_key`.
+
+    `create` and `bulk_create` return once the record is in the producer's buffer; aiokafka sends
+    buffered records in batches (`linger_ms`, `max_batch_size`). A record the broker rejects is
+    reported with an ERROR log entry, since nothing awaits its delivery.
+
+    The buffer only reaches Kafka while the process runs: call `stop()` on application shutdown,
+    otherwise up to `linger_ms` worth of records is lost on SIGTERM.
+
+    Class Attributes:
+        bootstrap_servers: Kafka servers address
+        config: Producer configuration
+
+    Example:
+        class EventProducerRepository(BaseKafkaProducerRepository):
+            bootstrap_servers = ['kafka:9092']
+            config = ProducerConfig(linger_ms=100)
+
+        producer_repository_impl = EventProducerRepository()
+        await producer_repository_impl.bulk_create('topic', events)
+        ...
+        await EventProducerRepository.stop()  # on shutdown
+    """
+
+    bootstrap_servers: ClassVar[list[str]]
+    config: ClassVar[ProducerConfig] = ProducerConfig()
+
+    _producer: ClassVar[AIOKafkaProducer | None] = None
+    _producer_lock: ClassVar[asyncio.Lock]
+    _stopped: ClassVar[bool] = False
+
+    def __init_subclass__(cls, **kwargs: Any) -> None:
+        super().__init_subclass__(**kwargs)
+        require_class_attribute(cls, 'bootstrap_servers', list)
+        require_class_attribute(cls, 'config', ProducerConfig)
+        cls._producer_lock = asyncio.Lock()
+
+    @classmethod
+    async def _get_producer(cls) -> AIOKafkaProducer:
+        if cls._producer is not None:
+            return cls._producer
+
+        # The lock makes the first concurrent sends start one producer instead of one each.
+        async with cls._producer_lock:
+            if cls._stopped:
+                raise RuntimeError(f'{cls.__name__} producer was stopped')
+            if cls._producer is None:
+                producer = AIOKafkaProducer(
+                    bootstrap_servers=cls.bootstrap_servers,
+                    value_serializer=serialize_value,
+                    key_serializer=serialize_key,
+                    **cls.config.model_dump(),
+                )
+                await producer.start()
+                cls._producer = producer
+            return cls._producer
+
+    @classmethod
+    async def stop(cls) -> None:
+        """
+        Sends what is still buffered and closes the producer. Call it on application shutdown.
+
+        A producer is created once per process: after `stop` the repository refuses to send.
+        """
+        async with cls._producer_lock:
+            cls._stopped = True
+            if cls._producer is not None:
+                await cls._producer.stop()
+                cls._producer = None
+
+    @classmethod
+    async def _send(cls, producer: AIOKafkaProducer, topic: str, event: Producible) -> None:
+        key = event.idempotent_key
+        future = await producer.send(topic, key=key, value=event)
+        future.add_done_callback(partial(_log_delivery_failure, topic, key))
+
+    async def create(self, topic: str, event: Producible) -> None:
+        producer = await self._get_producer()
+        await self._send(producer, topic, event)
+
+    async def bulk_create(self, topic: str, events: list[Producible]) -> None:
+        producer = await self._get_producer()
+        for event in events:
+            await self._send(producer, topic, event)
+
+
+__all__ = ('BaseKafkaProducerRepository', 'Producible', 'serialize_key', 'serialize_value')

ddkafka-0.0.1/pyproject.toml

@@ -0,0 +1,59 @@
+[project]
+name = "ddkafka"
+description = "Kafka helpers: batch consumer, lock-guarded consumer maker and msgpack producer on top of aiokafka"
+version = "0.0.1"
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+requires-python = ">=3.11,<3.15"
+keywords = [
+    "python",
+    "kafka",
+    "aiokafka",
+    "consumer",
+    "producer",
+    "ddkafka",
+]
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Typing :: Typed",
+]
+dependencies = [
+    "aiokafka>=0.12,<0.15",
+    "msgpack>=1.0,<2",
+    "ddutils>=0.2.3,<0.3.0",
+    "pydantic>=2.0,<3",
+]
+
+[[project.authors]]
+name = "davyddd"
+
+[project.urls]
+Repository = "https://github.com/davyddd/ddkafka"
+
+[dependency-groups]
+dev = [
+    "ipdb==0.13.13",
+    "ipython==9.16.1",
+]
+test = [
+    "pytest==9.0.3",
+    "pytest-cov==5.0.0",
+    "parameterized==0.9.0",
+]
+linter = [
+    "ruff==0.15.10",
+    "ty==0.0.15",
+    "complexipy==8.0.1",
+]
+
+[tool.uv.build-backend]
+module-root = ""
+
+[build-system]
+requires = ["uv_build>=0.12.0,<0.13.0"]
+build-backend = "uv_build"

ddkafka-0.0.1/pyproject.toml.orig

@@ -0,0 +1,51 @@
+[project]
+name = "ddkafka"
+description = "Kafka helpers: batch consumer, lock-guarded consumer maker and msgpack producer on top of aiokafka"
+version = "0.0.1"
+authors = [{ name = "davyddd" }]
+license = "MIT"
+license-files = ["LICENSE"]
+readme = "README.md"
+requires-python = ">=3.11,<3.15"
+keywords = ["python", "kafka", "aiokafka", "consumer", "producer", "ddkafka"]
+# uv_build does not derive these from requires-python (poetry did); PyPI and the pyversions badge read them
+classifiers = [
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
+    "Typing :: Typed",
+]
+dependencies = [
+    "aiokafka>=0.12,<0.15",
+    "msgpack>=1.0,<2",
+    "ddutils>=0.2.3,<0.3.0",
+    "pydantic>=2.0,<3",
+]
+
+[project.urls]
+Repository = "https://github.com/davyddd/ddkafka"
+
+[dependency-groups]
+dev = [
+    "ipdb==0.13.13",
+    "ipython==9.16.1",
+]
+test = [
+    "pytest==9.0.3",
+    "pytest-cov==5.0.0",
+    "parameterized==0.9.0",
+]
+linter = [
+    "ruff==0.15.10",
+    "ty==0.0.15",
+    "complexipy==8.0.1",
+]
+
+[tool.uv.build-backend]
+module-root = ""
+
+[build-system]
+requires = ["uv_build>=0.12.0,<0.13.0"]
+build-backend = "uv_build"