kafka-python 3.0.0__py3-none-any.whl

kafka/protocol/old/sasl_handshake.py

@@ -0,0 +1,39 @@
+from .api import Request, Response
+from .types import Array, Int16, Schema, String
+
+
+class SaslHandshakeResponse_v0(Response):
+    API_KEY = 17
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('error_code', Int16),
+        ('mechanisms', Array(String('utf-8')))
+    )
+    ALIASES = {
+        'enabled_mechanisms': 'mechanisms',
+    }
+
+
+class SaslHandshakeResponse_v1(Response):
+    API_KEY = 17
+    API_VERSION = 1
+    SCHEMA = SaslHandshakeResponse_v0.SCHEMA
+    ALIASES = SaslHandshakeResponse_v0.ALIASES
+
+
+class SaslHandshakeRequest_v0(Request):
+    API_KEY = 17
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('mechanism', String('utf-8'))
+    )
+
+
+class SaslHandshakeRequest_v1(Request):
+    API_KEY = 17
+    API_VERSION = 1
+    SCHEMA = SaslHandshakeRequest_v0.SCHEMA
+
+
+SaslHandshakeRequest = [SaslHandshakeRequest_v0, SaslHandshakeRequest_v1]
+SaslHandshakeResponse = [SaslHandshakeResponse_v0, SaslHandshakeResponse_v1]

kafka/protocol/old/struct.py

@@ -0,0 +1,87 @@
+from abc import ABC, abstractproperty
+from io import BytesIO
+
+from .abstract import AbstractType
+from .types import Schema, TaggedFields
+
+from kafka.util import WeakMethod
+
+
+class Struct(ABC):
+
+    @abstractproperty
+    def SCHEMA(self):
+        """An instance of Schema() representing the structure"""
+        pass
+
+    ALIASES = {} # for compatibility with new protocol defs from json
+    def __getattr__(self, name):
+        if name in self.ALIASES:
+            return getattr(self, self.ALIASES[name])
+        raise AttributeError(f"'{type(self).__name__}' object has no attribute '{name}'")
+
+    def __setattr__(self, name, value):
+        if name in self.ALIASES:
+            name = self.ALIASES[name]
+        return super().__setattr__(name, value)
+
+    def __init__(self, *args, **kwargs):
+        if self.SCHEMA.has_tagged_fields():
+            # Dont require TaggedFields value in *args
+            if len(args) == len(self.SCHEMA) - 1:
+                args = (*args, {})
+            elif len(args) == len(self.SCHEMA) and args[-1] is None:
+                args = (*args[:-1], {})
+        if len(args) == len(self.SCHEMA):
+            for i, name in enumerate(self.SCHEMA.names):
+                setattr(self, name, args[i])
+        elif len(args) > 0:
+            raise ValueError('Args must be empty or mirror schema')
+        else:
+            if self.SCHEMA.has_tagged_fields():
+                if kwargs.get('tags') is None:
+                    kwargs['tags'] = {}
+            for name in self.ALIASES:
+                if name in kwargs:
+                    kwargs[self.ALIASES[name]] = kwargs.pop(name)
+            for name in self.SCHEMA.names:
+                setattr(self, name, kwargs.pop(name, None))
+            if kwargs:
+                raise ValueError('Keyword(s) not in schema %s: %s'
+                                 % (list(self.SCHEMA.names),
+                                    ', '.join(kwargs.keys())))
+
+    def encode(self):
+        return self.SCHEMA.encode(
+            [getattr(self, name) for name in self.SCHEMA.names]
+        )
+
+    @classmethod
+    def decode(cls, data):
+        if isinstance(data, bytes):
+            data = BytesIO(data)
+        return cls(*cls.SCHEMA.decode(data))
+
+    def get_item(self, name):
+        if name not in self.SCHEMA.names:
+            raise KeyError("%s is not in the schema" % name)
+        return getattr(self, name)
+
+    def __repr__(self):
+        key_vals = []
+        for name, field in zip(self.SCHEMA.names, self.SCHEMA.fields):
+            key_vals.append('%s=%s' % (name, field.repr(getattr(self, name))))
+        return self.__class__.__name__ + '(' + ', '.join(key_vals) + ')'
+
+    def __hash__(self):
+        return hash(self.encode())
+
+    def __eq__(self, other):
+        if not isinstance(other, Struct):
+            return False
+        if self.SCHEMA != other.SCHEMA:
+            return False
+        for attr in self.SCHEMA.names:
+            if getattr(self, attr) != getattr(other, attr):
+                return False
+        return True

kafka/protocol/old/txn_offset_commit.py

@@ -0,0 +1,73 @@
+from .api import Request, Response
+from .types import Array, Int16, Int32, Int64, Schema, String
+
+
+class TxnOffsetCommitResponse_v0(Response):
+    API_KEY = 28
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('throttle_time_ms', Int32),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('error_code', Int16))))))
+
+
+class TxnOffsetCommitResponse_v1(Response):
+    API_KEY = 28
+    API_VERSION = 1
+    SCHEMA = TxnOffsetCommitResponse_v0.SCHEMA
+
+
+class TxnOffsetCommitResponse_v2(Response):
+    API_KEY = 28
+    API_VERSION = 2
+    SCHEMA = TxnOffsetCommitResponse_v1.SCHEMA
+
+
+class TxnOffsetCommitRequest_v0(Request):
+    API_KEY = 28
+    API_VERSION = 0
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('group_id', String('utf-8')),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_metadata', String('utf-8')))))))
+
+
+class TxnOffsetCommitRequest_v1(Request):
+    API_KEY = 28
+    API_VERSION = 1
+    SCHEMA = TxnOffsetCommitRequest_v0.SCHEMA
+
+
+class TxnOffsetCommitRequest_v2(Request):
+    API_KEY = 28
+    API_VERSION = 2
+    SCHEMA = Schema(
+        ('transactional_id', String('utf-8')),
+        ('group_id', String('utf-8')),
+        ('producer_id', Int64),
+        ('producer_epoch', Int16),
+        ('topics', Array(
+            ('name', String('utf-8')),
+            ('partitions', Array(
+                ('partition_index', Int32),
+                ('committed_offset', Int64),
+                ('committed_leader_epoch', Int32),
+                ('committed_metadata', String('utf-8')))))))
+
+
+TxnOffsetCommitRequest = [
+    TxnOffsetCommitRequest_v0, TxnOffsetCommitRequest_v1, TxnOffsetCommitRequest_v2, 
+]
+TxnOffsetCommitResponse = [
+    TxnOffsetCommitResponse_v0, TxnOffsetCommitResponse_v1, TxnOffsetCommitResponse_v2, 
+]

kafka/protocol/old/types.py

@@ -0,0 +1,440 @@
+import struct
+from struct import error
+import uuid
+
+from .abstract import AbstractType
+
+
+def _pack(f, value):
+    try:
+        return f(value)
+    except error as e:
+        try:
+            fmt = f.__self__.format
+        except AttributeError:
+            fmt = 'unknown'
+        raise ValueError("Error encountered when attempting to convert value: "
+                        "{!r} to struct format: '{}', hit error: {}"
+                        .format(value, fmt, e))
+
+
+def _unpack(f, data):
+    try:
+        (value,) = f(data)
+        return value
+    except error as e:
+        try:
+            fmt = f.__self__.format
+        except AttributeError:
+            fmt = 'unknown'
+        raise ValueError("Error encountered when attempting to convert value: "
+                        "{!r} to struct format: '{}', hit error: {}"
+                        .format(data, fmt, e))
+
+
+class Int8(AbstractType):
+    _pack = struct.Struct('>b').pack
+    _unpack = struct.Struct('>b').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(1))
+
+
+class Int16(AbstractType):
+    _pack = struct.Struct('>h').pack
+    _unpack = struct.Struct('>h').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(2))
+
+
+class Int32(AbstractType):
+    _pack = struct.Struct('>i').pack
+    _unpack = struct.Struct('>i').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(4))
+
+
+class Int64(AbstractType):
+    _pack = struct.Struct('>q').pack
+    _unpack = struct.Struct('>q').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(8))
+
+
+class Float64(AbstractType):
+    _pack = struct.Struct('>d').pack
+    _unpack = struct.Struct('>d').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(8))
+
+
+class UUID(AbstractType):
+    ZERO_UUID = uuid.UUID(int=0)
+
+    @classmethod
+    def encode(cls, value):
+        if value is None:
+            value = cls.ZERO_UUID
+        if isinstance(value, uuid.UUID):
+            return value.bytes
+        return uuid.UUID(value).bytes
+
+    @classmethod
+    def decode(cls, data):
+        val = uuid.UUID(bytes=data.read(16))
+        if val == cls.ZERO_UUID:
+            return None
+        return val
+
+
+class String(AbstractType):
+    def __init__(self, encoding='utf-8'):
+        self.encoding = encoding
+
+    def encode(self, value):
+        if value is None:
+            return Int16.encode(-1)
+        value = str(value).encode(self.encoding)
+        return Int16.encode(len(value)) + value
+
+    def decode(self, data):
+        length = Int16.decode(data)
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding string')
+        return value.decode(self.encoding)
+
+
+class Bytes(AbstractType):
+    @classmethod
+    def encode(cls, value):
+        if value is None:
+            return Int32.encode(-1)
+        elif not isinstance(value, bytes):
+            value = value.encode()
+        return Int32.encode(len(value)) + value
+
+    @classmethod
+    def decode(cls, data):
+        length = Int32.decode(data)
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding Bytes')
+        return value
+
+    @classmethod
+    def repr(cls, value):
+        return repr(value[:100] + b'...' if value is not None and len(value) > 100 else value)
+
+
+class Boolean(AbstractType):
+    _pack = struct.Struct('>?').pack
+    _unpack = struct.Struct('>?').unpack
+
+    @classmethod
+    def encode(cls, value):
+        return _pack(cls._pack, value)
+
+    @classmethod
+    def decode(cls, data):
+        return _unpack(cls._unpack, data.read(1))
+
+
+class Schema(AbstractType):
+    def __init__(self, *fields):
+        if fields:
+            self.names, self.fields = zip(*fields)
+        else:
+            self.names, self.fields = (), ()
+
+    def has_tagged_fields(self):
+        return len(self.fields) and self.fields[-1] is TaggedFields
+
+    def encode(self, item):
+        # Add empty tags to item if missing
+        if self.has_tagged_fields() and len(item) == len(self.fields) - 1:
+            item = [*item, {}]
+        if len(item) != len(self.fields):
+            raise ValueError('Item field count does not match Schema')
+        return b''.join([
+            field.encode(item[i])
+            for i, field in enumerate(self.fields)
+        ])
+
+    def decode(self, data):
+        decoded = [field.decode(data) for field in self.fields]
+        # Drop empty tags from tuple decoding
+        if self.has_tagged_fields() and decoded[-1] == {}:
+            decoded.pop()
+        return tuple(decoded)
+
+    def __len__(self):
+        return len(self.fields)
+
+    def repr(self, value):
+        key_vals = []
+        try:
+            for i in range(len(self)):
+                try:
+                    field_val = getattr(value, self.names[i])
+                except AttributeError:
+                    field_val = value[i]
+                key_vals.append('%s=%s' % (self.names[i], self.fields[i].repr(field_val)))
+            return '(' + ', '.join(key_vals) + ')'
+        except Exception:
+            return repr(value)
+
+
+class Array(AbstractType):
+    def __init__(self, *array_of):
+        if len(array_of) > 1:
+            self.array_of = Schema(*array_of)
+        elif len(array_of) == 1 and (isinstance(array_of[0], AbstractType) or
+                                     issubclass(array_of[0], AbstractType)):
+            self.array_of = array_of[0]
+        else:
+            raise ValueError('Array instantiated with no array_of type')
+
+    def encode(self, items):
+        if items is None:
+            return Int32.encode(-1)
+        encoded_items = [self.array_of.encode(item) for item in items]
+        return b''.join(
+            [Int32.encode(len(encoded_items))] +
+            encoded_items
+        )
+
+    def decode(self, data):
+        length = Int32.decode(data)
+        if length == -1:
+            return None
+        return [self.array_of.decode(data) for _ in range(length)]
+
+    def repr(self, list_of_items):
+        if list_of_items is None:
+            return 'NULL'
+        return '[' + ', '.join([self.array_of.repr(item) for item in list_of_items]) + ']'
+
+
+class UnsignedVarInt32(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value = VarInt32.decode(data)
+        return (value << 1) ^ (value >> 31)
+
+    @classmethod
+    def encode(cls, value):
+        return VarInt32.encode((value >> 1) ^ -(value & 1))
+
+
+class VarInt32(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value, i = 0, 0
+        while True:
+            b, = struct.unpack('B', data.read(1))
+            if not (b & 0x80):
+                break
+            value |= (b & 0x7f) << i
+            i += 7
+            if i > 28:
+                raise ValueError('Invalid value {}'.format(value))
+        value |= b << i
+        return (value >> 1) ^ -(value & 1)
+
+    @classmethod
+    def encode(cls, value):
+        # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 31)
+        value &= 0xffffffff
+        ret = b''
+        while (value & 0xffffff80) != 0:
+            b = (value & 0x7f) | 0x80
+            ret += struct.pack('B', b)
+            value >>= 7
+        ret += struct.pack('B', value)
+        return ret
+
+
+class VarInt64(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value, i = 0, 0
+        while True:
+            b, = struct.unpack('B', data.read(1))
+            if not (b & 0x80):
+                break
+            value |= (b & 0x7f) << i
+            i += 7
+            if i > 63:
+                raise ValueError('Invalid value {}'.format(value))
+        value |= b << i
+        return (value >> 1) ^ -(value & 1)
+
+    @classmethod
+    def encode(cls, value):
+        # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 63)
+        value &= 0xffffffffffffffff
+        ret = b''
+        while (value & 0xffffffffffffff80) != 0:
+            b = (value & 0x7f) | 0x80
+            ret += struct.pack('B', b)
+            value >>= 7
+        ret += struct.pack('B', value)
+        return ret
+
+
+class CompactString(String):
+    def decode(self, data):
+        length = UnsignedVarInt32.decode(data) - 1
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding string')
+        return value.decode(self.encoding)
+
+    def encode(self, value):
+        if value is None:
+            return UnsignedVarInt32.encode(0)
+        value = str(value).encode(self.encoding)
+        return UnsignedVarInt32.encode(len(value) + 1) + value
+
+
+class TaggedFields(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        num_fields = UnsignedVarInt32.decode(data)
+        ret = {}
+        if not num_fields:
+            return ret
+        prev_tag = -1
+        for i in range(num_fields):
+            tag = UnsignedVarInt32.decode(data)
+            if tag <= prev_tag:
+                raise ValueError('Invalid or out-of-order tag {}'.format(tag))
+            prev_tag = tag
+            size = UnsignedVarInt32.decode(data)
+            val = data.read(size)
+            ret[tag] = val
+        return ret
+
+    @classmethod
+    def encode(cls, value):
+        if value is None:
+            value = {}
+        ret = UnsignedVarInt32.encode(len(value))
+        for k, v in value.items():
+            # do we allow for other data types ?? It could get complicated really fast
+            assert isinstance(v, bytes), 'Value {} is not a byte array'.format(v)
+            assert isinstance(k, int) and k >= 0, 'Key {} is not a non-negative integer'.format(k)
+            ret += UnsignedVarInt32.encode(k)
+            ret += UnsignedVarInt32.encode(len(v))
+            ret += v
+        return ret
+
+
+class CompactBytes(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        length = UnsignedVarInt32.decode(data) - 1
+        if length < 0:
+            return None
+        value = data.read(length)
+        if len(value) != length:
+            raise ValueError('Buffer underrun decoding Bytes')
+        return value
+
+    @classmethod
+    def encode(cls, value):
+        if value is None:
+            return UnsignedVarInt32.encode(0)
+        else:
+            return UnsignedVarInt32.encode(len(value) + 1) + value
+
+
+class CompactArray(Array):
+    def encode(self, items):
+        if items is None:
+            return UnsignedVarInt32.encode(0)
+        return b''.join(
+            [UnsignedVarInt32.encode(len(items) + 1)] +
+            [self.array_of.encode(item) for item in items]
+        )
+
+    def decode(self, data):
+        length = UnsignedVarInt32.decode(data) - 1
+        if length == -1:
+            return None
+        return [self.array_of.decode(data) for _ in range(length)]
+
+
+class BitField(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        vals = cls.from_32_bit_field(Int32.decode(data))
+        if vals == {31}:
+            vals = None
+        return vals
+
+    @classmethod
+    def encode(cls, vals):
+        if vals is None:
+            vals = {31}
+        # to_32_bit_field returns unsigned val, so we need to
+        # encode >I to avoid crash if/when byte 31 is set
+        # (note that decode as signed still works fine)
+        return struct.Struct('>I').pack(cls.to_32_bit_field(vals))
+
+    @classmethod
+    def to_32_bit_field(cls, vals):
+        value = 0
+        for b in vals:
+            assert 0 <= b < 32
+            value |= 1 << b
+        return value
+
+    @classmethod
+    def from_32_bit_field(cls, value):
+        result = set()
+        count = 0
+        while value != 0:
+            if (value & 1) != 0:
+                result.add(count)
+            count += 1
+            value = (value & 0xFFFFFFFF) >> 1
+        return result