kafka-python 3.0.0__py3-none-any.whl

kafka/protocol/old/metadata.py

@@ -0,0 +1,290 @@
+from .api import Request, Response
+from .types import Array, Boolean, Int16, Int32, Schema, String, BitField
+
+
+class MetadataResponse_v0(Response):
+    API_KEY = 3
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32))),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition_index', Int32),
+                ('leader_id', Int32),
+                ('replica_nodes', Array(Int32)),
+                ('isr_nodes', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v1(Response):
+    API_KEY = 3
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('name', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition_index', Int32),
+                ('leader_id', Int32),
+                ('replica_nodes', Array(Int32)),
+                ('isr_nodes', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v2(Response):
+    API_KEY = 3
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),  # <-- Added cluster_id field in v2
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('name', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition_index', Int32),
+                ('leader_id', Int32),
+                ('replica_nodes', Array(Int32)),
+                ('isr_nodes', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v3(Response):
+    API_KEY = 3
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('name', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition_index', Int32),
+                ('leader_id', Int32),
+                ('replica_nodes', Array(Int32)),
+                ('isr_nodes', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v4(Response):
+    API_KEY = 3
+    API_VERSION = 4
+    SCHEMA = MetadataResponse_v3.SCHEMA
+
+
+class MetadataResponse_v5(Response):
+    API_KEY = 3
+    API_VERSION = 5
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('name', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition_index', Int32),
+                ('leader_id', Int32),
+                ('replica_nodes', Array(Int32)),
+                ('isr_nodes', Array(Int32)),
+                ('offline_replicas', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v6(Response):
+    """Metadata Request/Response v6 is the same as v5,
+    but on quota violation, brokers send out responses before throttling."""
+    API_KEY = 3
+    API_VERSION = 6
+    SCHEMA = MetadataResponse_v5.SCHEMA
+
+
+class MetadataResponse_v7(Response):
+    """v7 adds per-partition leader_epoch field"""
+    API_KEY = 3
+    API_VERSION = 7
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('name', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition_index', Int32),
+                ('leader_id', Int32),
+                ('leader_epoch', Int32),
+                ('replica_nodes', Array(Int32)),
+                ('isr_nodes', Array(Int32)),
+                ('offline_replicas', Array(Int32))))))
+    )
+
+
+class MetadataResponse_v8(Response):
+    """v8 adds authorized_operations fields"""
+    API_KEY = 3
+    API_VERSION = 8
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('brokers', Array(
+            ('node_id', Int32),
+            ('host', String('utf-8')),
+            ('port', Int32),
+            ('rack', String('utf-8')))),
+        ('cluster_id', String('utf-8')),
+        ('controller_id', Int32),
+        ('topics', Array(
+            ('error_code', Int16),
+            ('name', String('utf-8')),
+            ('is_internal', Boolean),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition_index', Int32),
+                ('leader_id', Int32),
+                ('leader_epoch', Int32),
+                ('replica_nodes', Array(Int32)),
+                ('isr_nodes', Array(Int32)),
+                ('offline_replicas', Array(Int32)))),
+            ('authorized_operations', BitField))),
+        ('authorized_operations', BitField)
+    )
+
+
+class MetadataRequest_v0(Request):
+    API_KEY = 3
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(String('utf-8')))
+    )
+    ALL_TOPICS = [] # Empty Array (len 0) for topics returns all topics
+    NO_TOPICS = [] # v0 does not support a 'no topics' request, so we'll just ask for ALL
+
+
+class MetadataRequest_v1(Request):
+    API_KEY = 3
+    API_VERSION = 1
+    SCHEMA = MetadataRequest_v0.SCHEMA
+    ALL_TOPICS = None # Null Array (len -1) for topics returns all topics
+    NO_TOPICS = [] # Empty array (len 0) for topics returns no topics
+
+
+class MetadataRequest_v2(Request):
+    API_KEY = 3
+    API_VERSION = 2
+    SCHEMA = MetadataRequest_v1.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v3(Request):
+    API_KEY = 3
+    API_VERSION = 3
+    SCHEMA = MetadataRequest_v1.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v4(Request):
+    API_KEY = 3
+    API_VERSION = 4
+    SCHEMA = Schema(
+        ('topics', Array(String('utf-8'))),
+        ('allow_auto_topic_creation', Boolean)
+    )
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v5(Request):
+    """
+    The v5 metadata request is the same as v4.
+    An additional field for offline_replicas has been added to the v5 metadata response
+    """
+    API_KEY = 3
+    API_VERSION = 5
+    SCHEMA = MetadataRequest_v4.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v6(Request):
+    API_KEY = 3
+    API_VERSION = 6
+    SCHEMA = MetadataRequest_v5.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v7(Request):
+    API_KEY = 3
+    API_VERSION = 7
+    SCHEMA = MetadataRequest_v6.SCHEMA
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+class MetadataRequest_v8(Request):
+    API_KEY = 3
+    API_VERSION = 8
+    SCHEMA = Schema(
+        ('topics', Array(String('utf-8'))),
+        ('allow_auto_topic_creation', Boolean),
+        ('include_cluster_authorized_operations', Boolean),
+        ('include_topic_authorized_operations', Boolean)
+    )
+    ALL_TOPICS = None
+    NO_TOPICS = []
+
+
+MetadataRequest = [
+    MetadataRequest_v0, MetadataRequest_v1, MetadataRequest_v2,
+    MetadataRequest_v3, MetadataRequest_v4, MetadataRequest_v5,
+    MetadataRequest_v6, MetadataRequest_v7, MetadataRequest_v8,
+]
+MetadataResponse = [
+    MetadataResponse_v0, MetadataResponse_v1, MetadataResponse_v2,
+    MetadataResponse_v3, MetadataResponse_v4, MetadataResponse_v5,
+    MetadataResponse_v6, MetadataResponse_v7, MetadataResponse_v8,
+]

kafka/protocol/old/offset_for_leader_epoch.py

@@ -0,0 +1,133 @@
+from .api import Request, Response
+from .types import Array, CompactArray, CompactString, Int16, Int32, Int64, Schema, String, TaggedFields
+
+
+class OffsetForLeaderEpochResponse_v0(Response):
+    API_KEY = 23
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('end_offset', Int64))))))
+
+
+class OffsetForLeaderEpochResponse_v1(Response):
+    API_KEY = 23
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader_epoch', Int32),
+                ('end_offset', Int64))))))
+
+
+class OffsetForLeaderEpochResponse_v2(Response):
+    API_KEY = 23
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader_epoch', Int32),
+                ('end_offset', Int64))))))
+
+
+class OffsetForLeaderEpochResponse_v3(Response):
+    API_KEY = 23
+    API_VERSION = 3
+    SCHEMA = OffsetForLeaderEpochResponse_v2.SCHEMA
+
+
+class OffsetForLeaderEpochResponse_v4(Response):
+    API_KEY = 23
+    API_VERSION = 4
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', CompactArray(
+            ('topic', CompactString('utf-8')),
+            ('partitions', CompactArray(
+                ('error_code', Int16),
+                ('partition', Int32),
+                ('leader_epoch', Int32),
+                ('end_offset', Int64),
+                ('tags', TaggedFields))),
+            ('tags', TaggedFields))),
+        ('tags', TaggedFields))
+
+
+class OffsetForLeaderEpochRequest_v0(Request):
+    API_KEY = 23
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('leader_epoch', Int32))))))
+
+
+class OffsetForLeaderEpochRequest_v1(Request):
+    API_KEY = 23
+    API_VERSION = 1
+    SCHEMA = OffsetForLeaderEpochRequest_v0.SCHEMA
+
+
+class OffsetForLeaderEpochRequest_v2(Request):
+    API_KEY = 23
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('leader_epoch', Int32))))))
+
+
+class OffsetForLeaderEpochRequest_v3(Request):
+    API_KEY = 23
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('topics', Array(
+            ('topic', String('utf-8')),
+            ('partitions', Array(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('leader_epoch', Int32))))))
+
+
+class OffsetForLeaderEpochRequest_v4(Request):
+    API_KEY = 23
+    API_VERSION = 4
+    SCHEMA = Schema(
+        ('replica_id', Int32),
+        ('topics', CompactArray(
+            ('topic', CompactString('utf-8')),
+            ('partitions', CompactArray(
+                ('partition', Int32),
+                ('current_leader_epoch', Int32),
+                ('leader_epoch', Int32),
+                ('tags', TaggedFields))),
+            ('tags', TaggedFields))),
+        ('tags', TaggedFields))
+
+OffsetForLeaderEpochRequest = [
+    OffsetForLeaderEpochRequest_v0, OffsetForLeaderEpochRequest_v1,
+    OffsetForLeaderEpochRequest_v2, OffsetForLeaderEpochRequest_v3,
+    OffsetForLeaderEpochRequest_v4,
+]
+OffsetForLeaderEpochResponse = [
+    OffsetForLeaderEpochResponse_v0, OffsetForLeaderEpochResponse_v1,
+    OffsetForLeaderEpochResponse_v2, OffsetForLeaderEpochResponse_v3,
+    OffsetForLeaderEpochResponse_v4,
+]

kafka/protocol/old/produce.py

@@ -0,0 +1,247 @@
+from .api import Request, Response
+from .types import Int16, Int32, Int64, String, Array, Schema, Bytes
+
+
+class ProduceResponse_v0(Response):
+    API_KEY = 0
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('responses', Array(
+            ('name', String('utf-8')),
+            ('partition_responses', Array(
+                ('index', Int32),
+                ('error_code', Int16),
+                ('base_offset', Int64)))))
+    )
+    ALIASES = {
+        'topics': 'responses',
+    }
+
+
+class ProduceResponse_v1(Response):
+    API_KEY = 0
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('responses', Array(
+            ('name', String('utf-8')),
+            ('partition_responses', Array(
+                ('index', Int32),
+                ('error_code', Int16),
+                ('base_offset', Int64))))),
+        ('throttle_time_ms', Int32)
+    )
+    ALIASES = ProduceResponse_v0.ALIASES
+
+
+class ProduceResponse_v2(Response):
+    API_KEY = 0
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('responses', Array(
+            ('name', String('utf-8')),
+            ('partition_responses', Array(
+                ('index', Int32),
+                ('error_code', Int16),
+                ('base_offset', Int64),
+                ('log_append_time_ms', Int64))))),
+        ('throttle_time_ms', Int32)
+    )
+    ALIASES = ProduceResponse_v1.ALIASES
+
+
+class ProduceResponse_v3(Response):
+    # Adds support for message format v2
+    API_KEY = 0
+    API_VERSION = 3
+    SCHEMA = ProduceResponse_v2.SCHEMA
+    ALIASES = ProduceResponse_v2.ALIASES
+
+
+class ProduceResponse_v4(Response):
+    """
+    The version number is bumped up to indicate that the client supports KafkaStorageException.
+    The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 3
+    """
+    API_KEY = 0
+    API_VERSION = 4
+    SCHEMA = ProduceResponse_v3.SCHEMA
+    ALIASES = ProduceResponse_v3.ALIASES
+
+
+class ProduceResponse_v5(Response):
+    API_KEY = 0
+    API_VERSION = 5
+    SCHEMA = Schema(
+        ('responses', Array(
+            ('name', String('utf-8')),
+            ('partition_responses', Array(
+                ('index', Int32),
+                ('error_code', Int16),
+                ('base_offset', Int64),
+                ('log_append_time_ms', Int64),
+                ('log_start_offset', Int64))))),
+        ('throttle_time_ms', Int32)
+    )
+    ALIASES = ProduceResponse_v4.ALIASES
+
+
+class ProduceResponse_v6(Response):
+    """
+    The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
+    """
+    API_KEY = 0
+    API_VERSION = 6
+    SCHEMA = ProduceResponse_v5.SCHEMA
+    ALIASES = ProduceResponse_v5.ALIASES
+
+
+class ProduceResponse_v7(Response):
+    """
+    V7 bumped up to indicate ZStandard capability. (see KIP-110)
+    """
+    API_KEY = 0
+    API_VERSION = 7
+    SCHEMA = ProduceResponse_v6.SCHEMA
+    ALIASES = ProduceResponse_v6.ALIASES
+
+
+class ProduceResponse_v8(Response):
+    """
+    V8 bumped up to add two new fields record_errors offset list and error_message
+    (See KIP-467)
+    """
+    API_KEY = 0
+    API_VERSION = 8
+    SCHEMA = Schema(
+        ('responses', Array(
+            ('name', String('utf-8')),
+            ('partition_responses', Array(
+                ('index', Int32),
+                ('error_code', Int16),
+                ('base_offset', Int64),
+                ('log_append_time_ms', Int64),
+                ('log_start_offset', Int64),
+                ('record_errors', (Array(
+                    ('batch_index', Int32),
+                    ('batch_index_error_message', String('utf-8'))
+                 ))),
+                ('error_message', String('utf-8')))
+             ))),
+        ('throttle_time_ms', Int32)
+    )
+    ALIASES = ProduceResponse_v7.ALIASES
+
+
+class ProduceRequest(Request):
+    API_KEY = 0
+
+    def expect_response(self):
+        if self.acks == 0: # pylint: disable=no-member
+            return False
+        return True
+
+
+class ProduceRequest_v0(ProduceRequest):
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('acks', Int16),
+        ('timeout_ms', Int32),
+        ('topic_data', Array(
+            ('name', String('utf-8')),
+            ('partition_data', Array(
+                ('index', Int32),
+                ('records', Bytes)))))
+    )
+    ALIASES = {
+        'required_acks': 'acks',
+        'timeout': 'timeout_ms',
+        'topics': 'topic_data',
+    }
+
+
+class ProduceRequest_v1(ProduceRequest):
+    API_VERSION = 1
+    SCHEMA = ProduceRequest_v0.SCHEMA
+    ALIASES = ProduceRequest_v0.ALIASES
+
+
+class ProduceRequest_v2(ProduceRequest):
+    API_VERSION = 2
+    SCHEMA = ProduceRequest_v1.SCHEMA
+    ALIASES = ProduceRequest_v1.ALIASES
+
+
+class ProduceRequest_v3(ProduceRequest):
+    # Adds support for message format v2
+    API_VERSION = 3
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('acks', Int16),
+        ('timeout_ms', Int32),
+        ('topic_data', Array(
+            ('name', String('utf-8')),
+            ('partition_data', Array(
+                ('index', Int32),
+                ('records', Bytes)))))
+    )
+    ALIASES = ProduceRequest_v2.ALIASES
+
+
+class ProduceRequest_v4(ProduceRequest):
+    """
+    The version number is bumped up to indicate that the client supports KafkaStorageException.
+    The KafkaStorageException will be translated to NotLeaderForPartitionException in the response if version <= 3
+    """
+    API_VERSION = 4
+    SCHEMA = ProduceRequest_v3.SCHEMA
+    ALIASES = ProduceRequest_v3.ALIASES
+
+
+class ProduceRequest_v5(ProduceRequest):
+    """
+    Same as v4. The version number is bumped since the v5 response includes an additional
+    partition level field: the log_start_offset.
+    """
+    API_VERSION = 5
+    SCHEMA = ProduceRequest_v4.SCHEMA
+    ALIASES = ProduceRequest_v4.ALIASES
+
+
+class ProduceRequest_v6(ProduceRequest):
+    """
+    The version number is bumped to indicate that on quota violation brokers send out responses before throttling.
+    """
+    API_VERSION = 6
+    SCHEMA = ProduceRequest_v5.SCHEMA
+    ALIASES = ProduceRequest_v5.ALIASES
+
+
+class ProduceRequest_v7(ProduceRequest):
+    """
+    V7 bumped up to indicate ZStandard capability. (see KIP-110)
+    """
+    API_VERSION = 7
+    SCHEMA = ProduceRequest_v6.SCHEMA
+    ALIASES = ProduceRequest_v6.ALIASES
+
+
+class ProduceRequest_v8(ProduceRequest):
+    """
+    V8 bumped up to add two new fields record_errors offset list and error_message to PartitionResponse
+    (See KIP-467)
+    """
+    API_VERSION = 8
+    SCHEMA = ProduceRequest_v7.SCHEMA
+    ALIASES = ProduceRequest_v7.ALIASES
+
+
+ProduceRequest = [
+    ProduceRequest_v0, ProduceRequest_v1, ProduceRequest_v2,
+    ProduceRequest_v3, ProduceRequest_v4, ProduceRequest_v5,
+    ProduceRequest_v6, ProduceRequest_v7, ProduceRequest_v8,
+]
+ProduceResponse = [
+    ProduceResponse_v0, ProduceResponse_v1, ProduceResponse_v2,
+    ProduceResponse_v3, ProduceResponse_v4, ProduceResponse_v5,
+    ProduceResponse_v6, ProduceResponse_v7, ProduceResponse_v8,
+]

kafka/protocol/old/sasl_authenticate.py

@@ -0,0 +1,38 @@
+from .api import Request, Response
+from .types import Bytes, Int16, Int64, Schema, String
+
+
+class SaslAuthenticateResponse_v0(Response):
+    API_KEY = 36
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('error_message', String('utf-8')),
+        ('auth_bytes', Bytes))
+
+
+class SaslAuthenticateResponse_v1(Response):
+    API_KEY = 36
+    API_VERSION = 1
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('error_message', String('utf-8')),
+        ('auth_bytes', Bytes),
+        ('session_lifetime_ms', Int64))
+
+
+class SaslAuthenticateRequest_v0(Request):
+    API_KEY = 36
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('auth_bytes', Bytes))
+
+
+class SaslAuthenticateRequest_v1(Request):
+    API_KEY = 36
+    API_VERSION = 1
+    SCHEMA = SaslAuthenticateRequest_v0.SCHEMA
+
+
+SaslAuthenticateRequest = [SaslAuthenticateRequest_v0, SaslAuthenticateRequest_v1]
+SaslAuthenticateResponse = [SaslAuthenticateResponse_v0, SaslAuthenticateResponse_v1]