kafka-python 3.0.0__py3-none-any.whl

kafka/protocol/consumer/metadata.py

@@ -0,0 +1,30 @@
+from __future__ import annotations
+
+from ..api_data import ApiData
+from kafka.structs import TopicPartition
+
+
+ConsumerProtocolType = 'consumer'
+
+
+class ConsumerProtocolSubscription(ApiData): pass
+class ConsumerProtocolAssignment(ApiData):
+
+    # Compatibility with old manual protocol definition
+    @property
+    def assignment(self) -> list:
+        return self.assigned_partitions
+
+    @assignment.setter
+    def assignment(self, value: list) -> None:
+        self.assigned_partitions = value
+
+    def partitions(self) -> list[TopicPartition]:
+        return [TopicPartition(topic, partition)
+                for topic, partitions in self.assigned_partitions
+                for partition in partitions]
+
+
+__all__ = [
+    'ConsumerProtocolSubscription', 'ConsumerProtocolAssignment', 'ConsumerProtocolType',
+]

kafka/protocol/consumer/metadata.pyi

@@ -0,0 +1,89 @@
+# Generated by generate_stubs.py (Python 3.14)
+import uuid
+from typing import Any, Self
+
+from kafka.protocol.api_data import ApiData
+from kafka.protocol.data_container import DataContainer
+
+__all__ = ['ConsumerProtocolSubscription', 'ConsumerProtocolAssignment', 'ConsumerProtocolType']
+
+class ConsumerProtocolSubscription(ApiData):
+    class TopicPartition(DataContainer):
+        topic: str
+        partitions: list[int]
+        def __init__(
+            self,
+            *args: Any,
+            topic: str = ...,
+            partitions: list[int] = ...,
+            version: int | None = None,
+            **kwargs: Any,
+        ) -> None: ...
+        @property
+        def version(self) -> int | None: ...
+        def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+    topics: list[str]
+    user_data: bytes | ApiData | None
+    owned_partitions: list[TopicPartition]
+    generation_id: int
+    rack_id: str | None
+    def __init__(
+        self,
+        *args: Any,
+        topics: list[str] = ...,
+        user_data: bytes | ApiData | None = ...,
+        owned_partitions: list[TopicPartition] = ...,
+        generation_id: int = ...,
+        rack_id: str | None = ...,
+        version: int | None = None,
+        **kwargs: Any,
+    ) -> None: ...
+    @property
+    def version(self) -> int | None: ...
+    def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+    name: str
+    type: str
+    valid_versions: tuple[int, int]
+    min_version: int
+    max_version: int
+
+class ConsumerProtocolAssignment(ApiData):
+    class TopicPartition(DataContainer):
+        topic: str
+        partitions: list[int]
+        def __init__(
+            self,
+            *args: Any,
+            topic: str = ...,
+            partitions: list[int] = ...,
+            version: int | None = None,
+            **kwargs: Any,
+        ) -> None: ...
+        @property
+        def version(self) -> int | None: ...
+        def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+    assigned_partitions: list[TopicPartition]
+    user_data: bytes | ApiData | None
+    def __init__(
+        self,
+        *args: Any,
+        assigned_partitions: list[TopicPartition] = ...,
+        user_data: bytes | ApiData | None = ...,
+        version: int | None = None,
+        **kwargs: Any,
+    ) -> None: ...
+    @property
+    def version(self) -> int | None: ...
+    def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+    name: str
+    type: str
+    valid_versions: tuple[int, int]
+    min_version: int
+    max_version: int
+    @property
+    def assignment(self) -> list: ...
+    def partitions(self) -> list[TopicPartition]: ...
+
+ConsumerProtocolType: str

kafka/protocol/consumer/offsets.py

@@ -0,0 +1,75 @@
+from enum import IntEnum
+
+from ..api_message import ApiMessage
+from kafka.util import EnumHelper
+
+
+UNKNOWN_OFFSET = -1
+
+class OffsetResetStrategy:
+    LATEST = -1
+    EARLIEST = -2
+    NONE = 0
+
+
+class IsolationLevel(EnumHelper, IntEnum):
+    READ_UNCOMMITTED = 0
+    READ_COMMITTED = 1
+
+
+class OffsetSpec(EnumHelper, IntEnum):
+    # Any >= 0:         # earliest offset whose timestamp is greater than or equal to the given timestamp and the timestamp of that record.
+    LATEST = -1         # offset of the next message that will be appended to the log and a timestamp of -1
+    EARLIEST = -2       # first offset on the partition, including remote-storage, and a timestamp of -1
+    MAX_TIMESTAMP = -3  # offset and timestamp corresponding to the record with the highest timestamp on the partition. (KIP-734)
+    EARLIEST_LOCAL = -4 # first offset on the local partition of the leader broker, excluding remote-storage, and a timestamp of -1 (KIP-405)
+    LATEST_TIERED = -5  # the latest offset of the partition in remote storage (KIP-1005)
+
+
+class OffsetTimestamp(int):
+    """Millisecond-timestamp spec for partition offset lookup.
+
+    Wraps an int so it can be distinguished from a bare offset. Use with
+    :meth:`KafkaAdminClient.reset_group_offsets` (and anywhere else a spec
+    may be mixed with explicit offsets) to request "earliest offset whose
+    timestamp is >= N ms".
+    """
+    __slots__ = ()
+
+    def __repr__(self):
+        return f'OffsetTimestamp({int(self)})'
+
+
+class ListOffsetsRequest(ApiMessage):
+    @classmethod
+    def min_version_for_timestamp(cls, ts):
+        ts = OffsetSpec(ts)
+        if ts == OffsetSpec.MAX_TIMESTAMP:
+            return 7
+        elif ts == OffsetSpec.EARLIEST_LOCAL:
+            return 8
+        elif ts == OffsetSpec.LATEST_TIERED:
+            return 9
+        else:
+            return 0
+
+    @classmethod
+    def min_version_for_isolation_level(cls, il):
+        if int(il) > 0:
+            return 2
+        else:
+            return 0
+
+class ListOffsetsResponse(ApiMessage): pass
+
+
+class OffsetForLeaderEpochRequest(ApiMessage): pass
+class OffsetForLeaderEpochResponse(ApiMessage): pass
+
+
+__all__ = [
+    'UNKNOWN_OFFSET', 'OffsetResetStrategy', 'IsolationLevel',
+    'OffsetSpec', 'OffsetTimestamp',
+    'ListOffsetsRequest', 'ListOffsetsResponse',
+    'OffsetForLeaderEpochRequest', 'OffsetForLeaderEpochResponse',
+]

kafka/protocol/consumer/offsets.pyi

@@ -0,0 +1,288 @@
+# Generated by generate_stubs.py (Python 3.14)
+import uuid
+from typing import Any, Self
+
+from enum import IntEnum
+from kafka.protocol.api_message import ApiMessage
+from kafka.protocol.data_container import DataContainer
+
+__all__ = ['UNKNOWN_OFFSET', 'OffsetResetStrategy', 'IsolationLevel', 'OffsetSpec', 'OffsetTimestamp', 'ListOffsetsRequest', 'ListOffsetsResponse', 'OffsetForLeaderEpochRequest', 'OffsetForLeaderEpochResponse']
+
+UNKNOWN_OFFSET: int
+
+class OffsetResetStrategy:
+    LATEST: int
+    EARLIEST: int
+    NONE: int
+
+class IsolationLevel(EnumHelper, IntEnum):
+    READ_UNCOMMITTED: int
+    READ_COMMITTED: int
+
+class OffsetSpec(EnumHelper, IntEnum):
+    LATEST: int
+    EARLIEST: int
+    MAX_TIMESTAMP: int
+    EARLIEST_LOCAL: int
+    LATEST_TIERED: int
+
+class OffsetTimestamp(int):
+    ...
+
+class ListOffsetsRequest(ApiMessage):
+    class ListOffsetsTopic(DataContainer):
+        class ListOffsetsPartition(DataContainer):
+            partition_index: int
+            current_leader_epoch: int
+            timestamp: int
+            max_num_offsets: int
+            def __init__(
+                self,
+                *args: Any,
+                partition_index: int = ...,
+                current_leader_epoch: int = ...,
+                timestamp: int = ...,
+                max_num_offsets: int = ...,
+                version: int | None = None,
+                **kwargs: Any,
+            ) -> None: ...
+            @property
+            def version(self) -> int | None: ...
+            def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+        name: str
+        partitions: list[ListOffsetsPartition]
+        def __init__(
+            self,
+            *args: Any,
+            name: str = ...,
+            partitions: list[ListOffsetsPartition] = ...,
+            version: int | None = None,
+            **kwargs: Any,
+        ) -> None: ...
+        @property
+        def version(self) -> int | None: ...
+        def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+    replica_id: int
+    isolation_level: int
+    topics: list[ListOffsetsTopic]
+    timeout_ms: int
+    def __init__(
+        self,
+        *args: Any,
+        replica_id: int = ...,
+        isolation_level: int = ...,
+        topics: list[ListOffsetsTopic] = ...,
+        timeout_ms: int = ...,
+        version: int | None = None,
+        **kwargs: Any,
+    ) -> None: ...
+    @property
+    def version(self) -> int | None: ...
+    def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+    name: str
+    type: str
+    API_KEY: int
+    API_VERSION: int
+    valid_versions: tuple[int, int]
+    min_version: int
+    max_version: int
+    @property
+    def header(self) -> Any: ...
+    @classmethod
+    def is_request(cls) -> bool: ...
+    def expect_response(self) -> bool: ...
+    def with_header(self, correlation_id: int = 0, client_id: str = "kafka-python") -> None: ...
+    @classmethod
+    def min_version_for_timestamp(cls, ts: Any) -> Any: ...
+    @classmethod
+    def min_version_for_isolation_level(cls, il: Any) -> Any: ...
+
+class ListOffsetsResponse(ApiMessage):
+    class ListOffsetsTopicResponse(DataContainer):
+        class ListOffsetsPartitionResponse(DataContainer):
+            partition_index: int
+            error_code: int
+            old_style_offsets: list[int]
+            timestamp: int
+            offset: int
+            leader_epoch: int
+            def __init__(
+                self,
+                *args: Any,
+                partition_index: int = ...,
+                error_code: int = ...,
+                old_style_offsets: list[int] = ...,
+                timestamp: int = ...,
+                offset: int = ...,
+                leader_epoch: int = ...,
+                version: int | None = None,
+                **kwargs: Any,
+            ) -> None: ...
+            @property
+            def version(self) -> int | None: ...
+            def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+        name: str
+        partitions: list[ListOffsetsPartitionResponse]
+        def __init__(
+            self,
+            *args: Any,
+            name: str = ...,
+            partitions: list[ListOffsetsPartitionResponse] = ...,
+            version: int | None = None,
+            **kwargs: Any,
+        ) -> None: ...
+        @property
+        def version(self) -> int | None: ...
+        def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+    throttle_time_ms: int
+    topics: list[ListOffsetsTopicResponse]
+    def __init__(
+        self,
+        *args: Any,
+        throttle_time_ms: int = ...,
+        topics: list[ListOffsetsTopicResponse] = ...,
+        version: int | None = None,
+        **kwargs: Any,
+    ) -> None: ...
+    @property
+    def version(self) -> int | None: ...
+    def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+    name: str
+    type: str
+    API_KEY: int
+    API_VERSION: int
+    valid_versions: tuple[int, int]
+    min_version: int
+    max_version: int
+    @property
+    def header(self) -> Any: ...
+    @classmethod
+    def is_request(cls) -> bool: ...
+    def expect_response(self) -> bool: ...
+    def with_header(self, correlation_id: int = 0, client_id: str = "kafka-python") -> None: ...
+
+class OffsetForLeaderEpochRequest(ApiMessage):
+    class OffsetForLeaderTopic(DataContainer):
+        class OffsetForLeaderPartition(DataContainer):
+            partition: int
+            current_leader_epoch: int
+            leader_epoch: int
+            def __init__(
+                self,
+                *args: Any,
+                partition: int = ...,
+                current_leader_epoch: int = ...,
+                leader_epoch: int = ...,
+                version: int | None = None,
+                **kwargs: Any,
+            ) -> None: ...
+            @property
+            def version(self) -> int | None: ...
+            def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+        topic: str
+        partitions: list[OffsetForLeaderPartition]
+        def __init__(
+            self,
+            *args: Any,
+            topic: str = ...,
+            partitions: list[OffsetForLeaderPartition] = ...,
+            version: int | None = None,
+            **kwargs: Any,
+        ) -> None: ...
+        @property
+        def version(self) -> int | None: ...
+        def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+    replica_id: int
+    topics: list[OffsetForLeaderTopic]
+    def __init__(
+        self,
+        *args: Any,
+        replica_id: int = ...,
+        topics: list[OffsetForLeaderTopic] = ...,
+        version: int | None = None,
+        **kwargs: Any,
+    ) -> None: ...
+    @property
+    def version(self) -> int | None: ...
+    def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+    name: str
+    type: str
+    API_KEY: int
+    API_VERSION: int
+    valid_versions: tuple[int, int]
+    min_version: int
+    max_version: int
+    @property
+    def header(self) -> Any: ...
+    @classmethod
+    def is_request(cls) -> bool: ...
+    def expect_response(self) -> bool: ...
+    def with_header(self, correlation_id: int = 0, client_id: str = "kafka-python") -> None: ...
+
+class OffsetForLeaderEpochResponse(ApiMessage):
+    class OffsetForLeaderTopicResult(DataContainer):
+        class EpochEndOffset(DataContainer):
+            error_code: int
+            partition: int
+            leader_epoch: int
+            end_offset: int
+            def __init__(
+                self,
+                *args: Any,
+                error_code: int = ...,
+                partition: int = ...,
+                leader_epoch: int = ...,
+                end_offset: int = ...,
+                version: int | None = None,
+                **kwargs: Any,
+            ) -> None: ...
+            @property
+            def version(self) -> int | None: ...
+            def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+        topic: str
+        partitions: list[EpochEndOffset]
+        def __init__(
+            self,
+            *args: Any,
+            topic: str = ...,
+            partitions: list[EpochEndOffset] = ...,
+            version: int | None = None,
+            **kwargs: Any,
+        ) -> None: ...
+        @property
+        def version(self) -> int | None: ...
+        def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+
+    throttle_time_ms: int
+    topics: list[OffsetForLeaderTopicResult]
+    def __init__(
+        self,
+        *args: Any,
+        throttle_time_ms: int = ...,
+        topics: list[OffsetForLeaderTopicResult] = ...,
+        version: int | None = None,
+        **kwargs: Any,
+    ) -> None: ...
+    @property
+    def version(self) -> int | None: ...
+    def to_dict(self, meta: bool = False, json: bool = True) -> dict: ...
+    name: str
+    type: str
+    API_KEY: int
+    API_VERSION: int
+    valid_versions: tuple[int, int]
+    min_version: int
+    max_version: int
+    @property
+    def header(self) -> Any: ...
+    @classmethod
+    def is_request(cls) -> bool: ...
+    def expect_response(self) -> bool: ...
+    def with_header(self, correlation_id: int = 0, client_id: str = "kafka-python") -> None: ...

kafka/protocol/data_container.py

@@ -0,0 +1,166 @@
+from kafka.util import classproperty
+
+
+class SlotsBuilder(type):
+    def __new__(metacls, name, bases, attrs, **kw):
+        if attrs.get('_struct') is not None:
+            attrs['__slots__'] = attrs.get('__slots__', ()) + tuple(attrs['_struct'].fields.keys())
+        return super().__new__(metacls, name, bases, attrs)
+
+
+class DataContainer(metaclass=SlotsBuilder):
+    __slots__ = ('tags', 'unknown_tags', '_version')
+    _struct = None
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        # Generate field data_classes and set as class attrs (by field.type_str)
+        if cls._struct is not None:
+            for field in cls._struct.fields.values():
+                if field.is_struct() or field.is_struct_array():
+                    if not field.has_data_class():
+                        field.set_data_class(type(field.type_str, (DataContainer,), {'_struct': field}))
+                    setattr(cls, field.type_str, field.data_class)
+
+    def __init__(self, *args, version=None, **field_vals):
+        assert self._struct is not None
+        if version is not None and not self._struct.min_version <= version <= self._struct.max_version:
+            raise ValueError(f'Invalid version: {version} (min={self._struct.min_version}, max={self._struct.max_version})')
+        self._version = version
+        # Support positional arg init for convenience
+        if len(args) > 0:
+            field_args = self._struct.untagged_fields(self._version) if self._version is not None else self._struct._fields
+            if len(args) > len(field_args):
+                raise RuntimeError('Unable to init DataContainer with %d positional args: unexpected %d' % (len(args), len(field_args)))
+            field_vals.update({field_args[i].name: arg for i, arg in enumerate(args)})
+            args = ()
+        self.tags = None
+        self.unknown_tags = None
+        for field in self._struct._fields:
+            name = field.name
+            if name in field_vals:
+                if field._tag is not None:
+                    if self.tags is None:
+                        self.tags = set()
+                    self.tags.add(name)
+                setattr(self, name, field_vals.pop(name))
+            # set default values for in-version attributes only
+            # for outbound requests these fields will be encoded and this
+            # optimizes to avoid __getattr__ lookups in the general case
+            elif self._version is not None and field.for_version_q(self._version):
+                setattr(self, name, field.default)
+        # Remaining field_vals are unknown tags or errors
+        for name in list(field_vals):
+            if name.startswith('_'):
+                if self.unknown_tags is None:
+                    self.unknown_tags = {}
+                self.unknown_tags[name] = field_vals.pop(name)
+        if field_vals:
+            raise ValueError('Unrecognized fields for type %s: %s' % (self._struct.name, field_vals))
+
+    def __getattr__(self, name):
+        # Only called when normal slot lookup fails (field not set).
+        # Returns the schema default for fields outside the decoded version.
+        field_map = self._struct.fields
+        if name in field_map:
+            return field_map[name].default
+        raise AttributeError("'%s' object has no attribute '%s'" % (type(self).__name__, name))
+
+    @property
+    def version(self):
+        return self._version
+
+    def encode(self, *args, **kwargs):
+        """Add version= to kwargs, otherwise pass-through to _struct"""
+        return self._struct.encode(self, *args, **kwargs)
+
+    def encode_into(self, out, *args, **kwargs):
+        """Buffer-based encode, pass-through to _struct.encode_into"""
+        self._struct.encode_into(self, out, *args, **kwargs)
+
+    @classmethod
+    def decode(cls, data, **kwargs):
+        """Add version= to kwargs, otherwise pass-through to _struct"""
+        return cls._struct.decode(data, **kwargs)
+
+    @classproperty
+    def fields(cls): # pylint: disable=E0213
+        return cls._struct.fields
+
+    def __repr__(self):
+        if self._version is not None:
+            v_filter = lambda field: field.for_version_q(self._version)
+            key_vals = ['version=%s' % self._version]
+        else:
+            v_filter = lambda field: True
+            key_vals = []
+        for field in filter(v_filter, self._struct._fields):
+            key_vals.append('%s=%s' % (field.name, repr(getattr(self, field.name))))
+        return self.__class__.__name__ + '(' + ', '.join(key_vals) + ')'
+
+    def __eq__(self, other):
+        # For backwards compatibility Data struct is equal to tuple with same field values
+        if isinstance(other, tuple):
+            # TODO: handle fields changes by version?
+            if len(other) < len(self._struct._fields):
+                return False
+            for i, field in enumerate(self._struct._fields):
+                if getattr(self, field.name) != other[i]:
+                    return False
+            if len(other) == len(self._struct._fields):
+                return True
+            elif len(other) == len(self._struct._fields) + 1 and isinstance(other[-1], dict) and other[-1] == {}:
+                # TODO: Handle non-empty tag dicts...
+                return True
+            return False
+        if self.__class__ != other.__class__:
+            return False
+        if self._struct != other._struct:
+            return False
+        for field in self._struct._fields:
+            if getattr(self, field.name) != getattr(other, field.name):
+                return False
+        return True
+
+    def __iter__(self):
+        if self._version is None:
+            raise RuntimeError('DataContainer Iteration not supported without _version')
+        return iter([getattr(self, field.name) for field in self._struct.untagged_fields(self._version)])
+
+    def _to_dict_vals(self, meta=False, json=True):
+        if meta:
+            yield ('_type', self.__class__.__name__)
+            yield ('_version', self._version)
+            if meta != 'all':
+                meta=False
+        for field in self._struct._fields:
+            if self._version is not None and not field.for_version_q(self._version):
+                continue
+            if field.is_struct():
+                val = getattr(self, field.name)
+                yield (field.name, None if val is None else dict(val._to_dict_vals(meta=meta, json=json)))
+            elif field.is_struct_array():
+                val = getattr(self, field.name)
+                yield (field.name, None if val is None else [dict(v._to_dict_vals(meta=meta, json=json)) for v in val])
+            else:
+                val = getattr(self, field.name)
+                yield (field.name, field.to_json(val) if json else val)
+
+    def to_dict(self, meta=False, json=True):
+        """Use meta=True to include top-level version; meta='all' to include all internal versions
+        json=False to return raw encoding; json=True (default) to convert values to be json-serializable
+        """
+        return dict(self._to_dict_vals(meta=meta, json=json))
+
+    def __getitem__(self, key):
+        if self._version is None:
+            raise RuntimeError('DataContainer subscript not supported without _version')
+        elif isinstance(key, int):
+            field = self._struct.untagged_fields(self._version)[key]
+            return getattr(self, field.name)
+        elif isinstance(key, slice):
+            fields = self._struct.untagged_fields(self._version)
+            start, stop, step = key.indices(len(fields))
+            return [getattr(self, fields[i].name) for i in range(start, stop, step)]
+        else:
+            raise TypeError('DataContainer subscript supports int or slices only: %s' % type(key).__name__)

kafka/protocol/frame.py

@@ -0,0 +1,30 @@
+class KafkaBytes(bytearray):
+    def __init__(self, size):
+        super().__init__(size)
+        self._idx = 0
+
+    def read(self, nbytes=None):
+        if nbytes is None:
+            nbytes = len(self) - self._idx
+        start = self._idx
+        self._idx += nbytes
+        if self._idx > len(self):
+            self._idx = len(self)
+        return bytes(self[start:self._idx])
+
+    def write(self, data):
+        start = self._idx
+        self._idx += len(data)
+        self[start:self._idx] = data
+
+    def seek(self, idx):
+        self._idx = idx
+
+    def tell(self):
+        return self._idx
+
+    def __str__(self):
+        return 'KafkaBytes(%d)' % len(self)
+
+    def __repr__(self):
+        return str(self)