kafka-python 3.0.0__py3-none-any.whl

kafka/protocol/old/api.py

@@ -0,0 +1,205 @@
+from abc import ABC, abstractmethod, abstractproperty
+from io import BytesIO
+import weakref
+
+from .struct import Struct
+from .types import Int16, Int32, String, Schema, Array, TaggedFields
+
+
+class ResponseClassRegistry:
+    _response_class_registry = {}
+
+    @classmethod
+    def register_response_class(cls, response_class):
+        cls._response_class_registry[(response_class.API_KEY, response_class.API_VERSION)] = response_class
+
+    @classmethod
+    def get_response_class(cls, header):
+        key = (header.api_key, header.api_version)
+        if key in cls._response_class_registry:
+            return cls._response_class_registry[key]
+
+
+class RequestHeader(ResponseClassRegistry, Struct):
+    SCHEMA = Schema(
+        ('api_key', Int16),
+        ('api_version', Int16),
+        ('correlation_id', Int32),
+        ('client_id', String('utf-8'))
+    )
+
+    def get_response_class(self):
+        return ResponseClassRegistry.get_response_class(self)
+
+
+class RequestHeaderV2(ResponseClassRegistry, Struct):
+    # Flexible response / request headers end in field buffer
+    SCHEMA = Schema(
+        ('api_key', Int16),
+        ('api_version', Int16),
+        ('correlation_id', Int32),
+        ('client_id', String('utf-8')),
+        ('tags', TaggedFields),
+    )
+
+    def get_response_class(self):
+        key = (self.api_key, self.api_version) # pylint: disable=E1101
+        if key in ResponseClassRegistry._response_class_registry:
+            return ResponseClassRegistry._response_class_registry[key]
+
+
+class ResponseHeader(Struct):
+    SCHEMA = Schema(
+        ('correlation_id', Int32),
+    )
+
+
+class ResponseHeaderV2(Struct):
+    SCHEMA = Schema(
+        ('correlation_id', Int32),
+        ('tags', TaggedFields),
+    )
+
+
+class RequestResponse(Struct, ABC):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._header = None
+
+    @abstractproperty
+    def API_KEY(self):
+        """Integer identifier for api request"""
+        pass
+
+    @abstractproperty
+    def API_VERSION(self):
+        """Integer of api request version"""
+        pass
+
+    def to_object(self):
+        return _to_object(self.SCHEMA, self)
+
+    @classmethod
+    @abstractmethod
+    def is_request(cls):
+        pass
+
+    @property
+    def header(self):
+        return self._header
+
+    def with_header(self, correlation_id=0, client_id='kafka-python'):
+        if self.is_request():
+            kwargs = {
+                'api_key': self.API_KEY,
+                'api_version': self.API_VERSION,
+                'correlation_id': correlation_id,
+                'client_id': client_id,
+            }
+        else:
+            kwargs = {
+                'correlation_id': correlation_id,
+            }
+        self._header = self.header_class()(**kwargs)
+
+    def encode(self, header=False, framed=False):
+        data = super().encode()
+        if not framed and not header:
+            return data
+        bits = [data]
+        if header:
+            bits.insert(0, self.header.encode())
+        if framed:
+            bits.insert(0, Int32.encode(sum(map(len, bits))))
+        return b''.join(bits)
+
+    @classmethod
+    @abstractmethod
+    def header_class(cls):
+        pass
+
+    @classmethod
+    def parse_header(cls, read_buffer):
+        return cls.header_class().decode(read_buffer)
+
+    @classmethod
+    def decode(cls, data, header=False, framed=False):
+        if not framed and not header:
+            return super().decode(data)
+        if isinstance(data, bytes):
+            data = BytesIO(data)
+        if framed:
+            size = Int32.decode(data)
+        if header:
+            hdr = cls.parse_header(data)
+        else:
+            hdr = None
+        ret = super().decode(data)
+        if hdr is not None:
+            ret._header = hdr
+        return ret
+
+    def __eq__(self, other):
+        return self._header == other._header and super().__eq__(other)
+
+
+class Request(RequestResponse):
+    FLEXIBLE_VERSION = False
+
+    @classmethod
+    def is_request(cls):
+        return True
+
+    def expect_response(self):
+        """Override this method if an api request does not always generate a response"""
+        return True
+
+    @classmethod
+    def header_class(cls):
+        if cls.FLEXIBLE_VERSION:
+            return RequestHeaderV2
+        else:
+            return RequestHeader
+
+
+class Response(RequestResponse):
+    FLEXIBLE_VERSION = False
+
+    def __init_subclass__(cls, **kwargs):
+        super().__init_subclass__(**kwargs)
+        ResponseClassRegistry.register_response_class(weakref.proxy(cls))
+
+    @classmethod
+    def is_request(cls):
+        return False
+
+    @classmethod
+    def header_class(cls):
+        if cls.FLEXIBLE_VERSION:
+            return ResponseHeaderV2
+        else:
+            return ResponseHeader
+
+
+def _to_object(schema, data):
+    obj = {}
+    for idx, (name, _type) in enumerate(zip(schema.names, schema.fields)):
+        if isinstance(data, Struct):
+            val = data.get_item(name)
+        else:
+            val = data[idx]
+
+        if isinstance(_type, Schema):
+            obj[name] = _to_object(_type, val)
+        elif isinstance(_type, Array):
+            if isinstance(_type.array_of, (Array, Schema)):
+                obj[name] = [
+                    _to_object(_type.array_of, x)
+                    for x in val
+                ]
+            else:
+                obj[name] = val
+        else:
+            obj[name] = val
+
+    return obj

kafka/protocol/old/api_versions.py

@@ -0,0 +1,133 @@
+from io import BytesIO
+
+from .api import Request, Response
+from .types import Array, CompactArray, CompactString, Int16, Int32, Schema, TaggedFields
+
+
+class BaseApiVersionsResponse(Response):
+    API_KEY = 18
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_keys', Array(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16)))
+    )
+    ALIASES = {'api_versions': 'api_keys'}
+
+    @classmethod
+    def decode(cls, data, header=False, framed=False):
+        if isinstance(data, bytes):
+            data = BytesIO(data)
+        # Check error_code, decode as v0 if any error
+        curr = data.tell()
+        if framed:
+            nbytes = Int32.decode(data)
+        if header:
+            cls.parse_header(data) # Note: non-flexible header
+        err = Int16.decode(data)
+        data.seek(curr)
+        if err != 0:
+            return ApiVersionsResponse_v0.decode(data, header=header, framed=framed)
+        return super().decode(data, header=header, framed=framed)
+
+
+class ApiVersionsResponse_v0(Response):
+    API_KEY = 18
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_keys', Array(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16)))
+    )
+    ALIASES = {'api_versions': 'api_keys'}
+
+
+class ApiVersionsResponse_v1(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_keys', Array(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16))),
+        ('throttle_time_ms', Int32)
+    )
+
+
+class ApiVersionsResponse_v2(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 2
+    SCHEMA = ApiVersionsResponse_v1.SCHEMA
+
+
+class ApiVersionsResponse_v3(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('api_keys', CompactArray(
+            ('api_key', Int16),
+            ('min_version', Int16),
+            ('max_version', Int16),
+            ('tags', TaggedFields))),
+        ('throttle_time_ms', Int32),
+        ('tags', TaggedFields)
+    )
+    # Note: ApiVersions Response does not send FLEXIBLE_VERSION header!
+
+
+class ApiVersionsResponse_v4(BaseApiVersionsResponse):
+    API_KEY = 18
+    API_VERSION = 4
+    SCHEMA = ApiVersionsResponse_v3.SCHEMA
+
+
+class ApiVersionsRequest_v0(Request):
+    API_KEY = 18
+    API_VERSION = 0
+    SCHEMA = Schema()
+
+
+class ApiVersionsRequest_v1(Request):
+    API_KEY = 18
+    API_VERSION = 1
+    SCHEMA = ApiVersionsRequest_v0.SCHEMA
+
+
+class ApiVersionsRequest_v2(Request):
+    API_KEY = 18
+    API_VERSION = 2
+    SCHEMA = ApiVersionsRequest_v1.SCHEMA
+
+
+class ApiVersionsRequest_v3(Request):
+    API_KEY = 18
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('client_software_name', CompactString('utf-8')),
+        ('client_software_version', CompactString('utf-8')),
+        ('tags', TaggedFields)
+    )
+    FLEXIBLE_VERSION = True
+
+
+class ApiVersionsRequest_v4(Request):
+    API_KEY = 18
+    API_VERSION = 4
+    SCHEMA = ApiVersionsRequest_v3.SCHEMA
+    FLEXIBLE_VERSION = True
+
+
+ApiVersionsRequest = [
+    ApiVersionsRequest_v0, ApiVersionsRequest_v1, ApiVersionsRequest_v2,
+    ApiVersionsRequest_v3, ApiVersionsRequest_v4,
+]
+ApiVersionsResponse = [
+    ApiVersionsResponse_v0, ApiVersionsResponse_v1, ApiVersionsResponse_v2,
+    ApiVersionsResponse_v3, ApiVersionsResponse_v4,
+]

kafka/protocol/old/commit.py

@@ -0,0 +1,355 @@
+from .api import Request, Response
+from .types import Array, Int16, Int32, Int64, Schema, String
+
+
+class OffsetCommitResponse_v0(Response):
+    API_KEY = 8
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('error_code', Int16)))))
+    )
+
+
+class OffsetCommitResponse_v1(Response):
+    API_KEY = 8
+    API_VERSION = 1
+    SCHEMA = OffsetCommitResponse_v0.SCHEMA
+
+
+class OffsetCommitResponse_v2(Response):
+    API_KEY = 8
+    API_VERSION = 2
+    SCHEMA = OffsetCommitResponse_v1.SCHEMA
+
+
+class OffsetCommitResponse_v3(Response):
+    API_KEY = 8
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('error_code', Int16)))))
+    )
+
+
+class OffsetCommitResponse_v4(Response):
+    API_KEY = 8
+    API_VERSION = 4
+    SCHEMA = OffsetCommitResponse_v3.SCHEMA
+
+
+class OffsetCommitResponse_v5(Response):
+    API_KEY = 8
+    API_VERSION = 5
+    SCHEMA = OffsetCommitResponse_v4.SCHEMA
+
+
+class OffsetCommitResponse_v6(Response):
+    API_KEY = 8
+    API_VERSION = 6
+    SCHEMA = OffsetCommitResponse_v5.SCHEMA
+
+
+class OffsetCommitResponse_v7(Response):
+    API_KEY = 8
+    API_VERSION = 7
+    SCHEMA = OffsetCommitResponse_v6.SCHEMA
+
+
+class OffsetCommitRequest_v0(Request):
+    API_KEY = 8
+    API_VERSION = 0  # Zookeeper-backed storage
+    SCHEMA = Schema(
+        ('group_id', String('utf-8')),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_metadata', String('utf-8'))))))
+    )
+    ALIASES = {
+        'consumer_group': 'group_id',
+    }
+
+
+class OffsetCommitRequest_v1(Request):
+    API_KEY = 8
+    API_VERSION = 1  # Kafka-backed storage
+    SCHEMA = Schema(
+        ('group_id', String('utf-8')),
+        ('generation_id_or_member_epoch', Int32),
+        ('member_id', String('utf-8')),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('commit_timestamp', Int64),
+                ('committed_metadata', String('utf-8'))))))
+    )
+    ALIASES = {
+        'consumer_group': 'group_id',
+        'consumer_group_generation_id': 'generation_id_or_member_epoch',
+        'consumer_id': 'member_id',
+    }
+
+
+class OffsetCommitRequest_v2(Request):
+    API_KEY = 8
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('group_id', String('utf-8')),
+        ('generation_id_or_member_epoch', Int32),
+        ('member_id', String('utf-8')),
+        ('retention_time_ms', Int64), # added retention_time_ms, dropped timestamp
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_metadata', String('utf-8'))))))
+    )
+    DEFAULT_RETENTION_TIME = -1
+    ALIASES = {
+        'consumer_group': 'group_id',
+        'consumer_group_generation_id': 'generation_id_or_member_epoch',
+        'consumer_id': 'member_id',
+        'retention_time': 'retention_time_ms',
+    }
+
+
+class OffsetCommitRequest_v3(Request):
+    API_KEY = 8
+    API_VERSION = 3
+    SCHEMA = OffsetCommitRequest_v2.SCHEMA
+    DEFAULT_RETENTION_TIME = -1
+    ALIASES = OffsetCommitRequest_v2.ALIASES
+
+
+class OffsetCommitRequest_v4(Request):
+    API_KEY = 8
+    API_VERSION = 4
+    SCHEMA = OffsetCommitRequest_v3.SCHEMA
+    DEFAULT_RETENTION_TIME = -1
+    ALIASES = OffsetCommitRequest_v3.ALIASES
+
+
+class OffsetCommitRequest_v5(Request):
+    API_KEY = 8
+    API_VERSION = 5 # drops retention_time
+    SCHEMA = Schema(
+        ('group_id', String('utf-8')),
+        ('generation_id_or_member_epoch', Int32),
+        ('member_id', String('utf-8')),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_metadata', String('utf-8'))))))
+    )
+    ALIASES = {
+        'consumer_group': 'group_id',
+        'consumer_group_generation_id': 'generation_id_or_member_epoch',
+        'consumer_id': 'member_id',
+    }
+
+
+class OffsetCommitRequest_v6(Request):
+    API_KEY = 8
+    API_VERSION = 6
+    SCHEMA = Schema(
+        ('group_id', String('utf-8')),
+        ('generation_id_or_member_epoch', Int32),
+        ('member_id', String('utf-8')),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_leader_epoch', Int32), # added for fencing / kip-320. default -1
+                ('committed_metadata', String('utf-8'))))))
+    )
+    ALIASES = OffsetCommitRequest_v5.ALIASES
+
+
+class OffsetCommitRequest_v7(Request):
+    API_KEY = 8
+    API_VERSION = 7
+    SCHEMA = Schema(
+        ('group_id', String('utf-8')),
+        ('generation_id_or_member_epoch', Int32),
+        ('member_id', String('utf-8')),
+        ('group_instance_id', String('utf-8')), # added for static membership / kip-345
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_leader_epoch', Int32),
+                ('committed_metadata', String('utf-8'))))))
+    )
+    ALIASES = {
+        'consumer_group': 'group_id',
+        'generation_id': 'generation_id_or_member_epoch',
+        'consumer_id': 'member_id',
+    }
+
+
+OffsetCommitRequest = [
+    OffsetCommitRequest_v0, OffsetCommitRequest_v1, OffsetCommitRequest_v2,
+    OffsetCommitRequest_v3, OffsetCommitRequest_v4, OffsetCommitRequest_v5,
+    OffsetCommitRequest_v6, OffsetCommitRequest_v7,
+]
+OffsetCommitResponse = [
+    OffsetCommitResponse_v0, OffsetCommitResponse_v1, OffsetCommitResponse_v2,
+    OffsetCommitResponse_v3, OffsetCommitResponse_v4, OffsetCommitResponse_v5,
+    OffsetCommitResponse_v6, OffsetCommitResponse_v7,
+]
+
+
+class OffsetFetchResponse_v0(Response):
+    API_KEY = 9
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16)))))
+    )
+
+
+class OffsetFetchResponse_v1(Response):
+    API_KEY = 9
+    API_VERSION = 1
+    SCHEMA = OffsetFetchResponse_v0.SCHEMA
+
+
+class OffsetFetchResponse_v2(Response):
+    # Added in KIP-88
+    API_KEY = 9
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16))))),
+        ('error_code', Int16)
+    )
+
+
+class OffsetFetchResponse_v3(Response):
+    API_KEY = 9
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16))))),
+        ('error_code', Int16)
+    )
+
+
+class OffsetFetchResponse_v4(Response):
+    API_KEY = 9
+    API_VERSION = 4
+    SCHEMA = OffsetFetchResponse_v3.SCHEMA
+
+
+class OffsetFetchResponse_v5(Response):
+    API_KEY = 9
+    API_VERSION = 5
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_leader_epoch', Int32),
+                ('metadata', String('utf-8')),
+                ('error_code', Int16))))),
+        ('error_code', Int16)
+    )
+
+
+class OffsetFetchRequest_v0(Request):
+    API_KEY = 9
+    API_VERSION = 0  # zookeeper-backed storage
+    SCHEMA = Schema(
+        ('group_id', String('utf-8')),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partition_indexes', Array(Int32))))
+    )
+    ALIASES = {
+        'consumer_group': 'group_id',
+    }
+
+
+class OffsetFetchRequest_v1(Request):
+    API_KEY = 9
+    API_VERSION = 1  # kafka-backed storage
+    SCHEMA = OffsetFetchRequest_v0.SCHEMA
+    ALIASES = OffsetFetchRequest_v0.ALIASES
+
+
+class OffsetFetchRequest_v2(Request):
+    # KIP-88: Allows passing null topics to return offsets for all partitions
+    # that the consumer group has a stored offset for, even if no consumer in
+    # the group is currently consuming that partition.
+    API_KEY = 9
+    API_VERSION = 2
+    SCHEMA = OffsetFetchRequest_v1.SCHEMA
+    ALIASES = OffsetFetchRequest_v1.ALIASES
+
+
+class OffsetFetchRequest_v3(Request):
+    API_KEY = 9
+    API_VERSION = 3
+    SCHEMA = OffsetFetchRequest_v2.SCHEMA
+    ALIASES = OffsetFetchRequest_v2.ALIASES
+
+
+class OffsetFetchRequest_v4(Request):
+    API_KEY = 9
+    API_VERSION = 4
+    SCHEMA = OffsetFetchRequest_v3.SCHEMA
+    ALIASES = OffsetFetchRequest_v3.ALIASES
+
+
+class OffsetFetchRequest_v5(Request):
+    API_KEY = 9
+    API_VERSION = 5
+    SCHEMA = OffsetFetchRequest_v4.SCHEMA
+    ALIASES = OffsetFetchRequest_v4.ALIASES
+
+
+OffsetFetchRequest = [
+    OffsetFetchRequest_v0, OffsetFetchRequest_v1,
+    OffsetFetchRequest_v2, OffsetFetchRequest_v3,
+    OffsetFetchRequest_v4, OffsetFetchRequest_v5,
+]
+OffsetFetchResponse = [
+    OffsetFetchResponse_v0, OffsetFetchResponse_v1,
+    OffsetFetchResponse_v2, OffsetFetchResponse_v3,
+    OffsetFetchResponse_v4, OffsetFetchResponse_v5,
+]