kafka-python 2.2.18__tar.gz → 2.2.20__tar.gz

{kafka_python-2.2.18 → kafka_python-2.2.20}/CHANGES.md

@@ -1,3 +1,22 @@
+# 2.2.20 (Jun 3, 2026)
+
+Fixes
+* kafka.conn: Improve error handling for sasl authenticate mechanisms
+* kafka.net: Validate SASL/SCRAM iterations (#3026)
+* KafkaProtocol: validate network frame size (backport of #3019)
+* Clear `_callbacks`/`_errbacks` list when future `is_done` to avoid reference cycles (#2891)
+
+# 2.2.19 (Apr 9, 2026)
+
+Fixes
+* Fix TaggedFields value encoding; add test coverage (#2725)
+* Fix zstd multi-frame decompression failure (#2717)
+* Fix KeyError in KafkaConsumer.committed() (#2710)
+* Fix VarInt/VarLong encoding; move tests to test/protocol/ (#2706)
+* Fix `Fetcher._fetch_offsets_by_times retry handling` (#2833)
+* Fixes to support integration testing with external KAFKA_URI (#2838)
+* Minor py2 fixes: consumer integration test; done pip install python-snappy
+
 # 2.2.18 (Nov 20, 2025)
 
 Fixes

{kafka_python-2.2.18 → kafka_python-2.2.20}/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: kafka-python
-Version: 2.2.18
+Version: 2.2.20
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/admin/client.py

@@ -87,6 +87,9 @@ class KafkaAdminClient(object):
         max_in_flight_requests_per_connection (int): Requests are pipelined
             to kafka brokers up to this number of maximum requests per
             broker connection. Default: 5.
+        receive_message_max_bytes (int): Maximum allowed network frame size.
+            Used to avoid OOM when decoding malformed network message header.
+            Default: 1000000.
         receive_buffer_bytes (int): The size of the TCP receive buffer
             (SO_RCVBUF) to use when reading data. Default: None (relies on
             system defaults). Java client defaults to 32768.
@@ -163,11 +166,10 @@ class KafkaAdminClient(object):
         'reconnect_backoff_ms': 50,
         'reconnect_backoff_max_ms': 30000,
         'max_in_flight_requests_per_connection': 5,
+        'receive_message_max_bytes': 1000000,
         'receive_buffer_bytes': None,
         'send_buffer_bytes': None,
         'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
-        'sock_chunk_bytes': 4096,  # undocumented experimental option
-        'sock_chunk_buffer_count': 1000,  # undocumented experimental option
         'retry_backoff_ms': 100,
         'metadata_max_age_ms': 300000,
         'security_protocol': 'PLAINTEXT',

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/client_async.py

@@ -89,6 +89,9 @@ class KafkaClient(object):
         max_in_flight_requests_per_connection (int): Requests are pipelined
             to kafka brokers up to this number of maximum requests per
             broker connection. Default: 5.
+        receive_message_max_bytes (int): Maximum allowed network frame size.
+            Used to avoid OOM when decoding malformed network message header.
+            Default: 1000000.
         receive_buffer_bytes (int): The size of the TCP receive buffer
             (SO_RCVBUF) to use when reading data. Default: None (relies on
             system defaults). Java client defaults to 32768.
@@ -186,6 +189,7 @@ class KafkaClient(object):
         'reconnect_backoff_ms': 50,
         'reconnect_backoff_max_ms': 30000,
         'max_in_flight_requests_per_connection': 5,
+        'receive_message_max_bytes': 1000000,
         'receive_buffer_bytes': None,
         'send_buffer_bytes': None,
         'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/codec.py

@@ -327,7 +327,5 @@ def zstd_encode(payload):
 def zstd_decode(payload):
     if not zstd:
         raise NotImplementedError("Zstd codec is not available")
-    try:
-        return zstd.ZstdDecompressor().decompress(payload)
-    except zstd.ZstdError:
-        return zstd.ZstdDecompressor().decompress(payload, max_output_size=ZSTD_MAX_OUTPUT_SIZE)
+    with zstd.ZstdDecompressor().stream_reader(io.BytesIO(payload), read_across_frames=True) as reader:
+        return reader.read()

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/conn.py

@@ -122,6 +122,9 @@ class BrokerConnection(object):
         max_in_flight_requests_per_connection (int): Requests are pipelined
             to kafka brokers up to this number of maximum requests per
             broker connection. Default: 5.
+        receive_message_max_bytes (int): Maximum allowed network frame size.
+            Used to avoid OOM when decoding malformed network message header.
+            Default: 1000000.
         receive_buffer_bytes (int): The size of the TCP receive buffer
             (SO_RCVBUF) to use when reading data. Default: None (relies on
             system defaults). Java client defaults to 32768.
@@ -202,6 +205,7 @@ class BrokerConnection(object):
         'reconnect_backoff_ms': 50,
         'reconnect_backoff_max_ms': 30000,
         'max_in_flight_requests_per_connection': 5,
+        'receive_message_max_bytes': 1000000,
         'receive_buffer_bytes': None,
         'send_buffer_bytes': None,
         'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
@@ -292,7 +296,8 @@ class BrokerConnection(object):
 
         self._protocol = KafkaProtocol(
             client_id=self.config['client_id'],
-            api_version=self.config['api_version'])
+            api_version=self.config['api_version'],
+            max_frame_size=self.config['receive_message_max_bytes'])
         self.state = ConnectionStates.DISCONNECTED
         self._reset_reconnect_backoff()
         self._sock = None
@@ -695,9 +700,12 @@ class BrokerConnection(object):
                     'Kafka broker does not support %s sasl mechanism. Enabled mechanisms are: %s'
                     % (self.config['sasl_mechanism'], response.enabled_mechanisms)))
         else:
-            self._sasl_authenticate(future)
+            try:
+                ret = self._sasl_authenticate()
+                future.success(ret)
+            except Exception as exc:
+                future.failure(exc)
 
-        assert future.is_done, 'SASL future not complete after mechanism processing!'
         if future.failed():
             self.close(error=future.exception)
         else:
@@ -809,24 +817,24 @@ class BrokerConnection(object):
             log.debug('%s: Received %d raw sasl auth bytes from server', self, nbytes)
             return data[4:]
 
-    def _sasl_authenticate(self, future):
+    def _sasl_authenticate(self):
         while not self._sasl_mechanism.is_done():
             send_token = self._sasl_mechanism.auth_bytes()
             self._send_sasl_authenticate(send_token)
             if not self._can_send_recv():
-                return future.failure(Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self))
+                raise Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self)
 
             recv_token = self._recv_sasl_authenticate()
             if recv_token is None:
-                return future.failure(Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self))
+                raise Errors.KafkaConnectionError("%s: Connection failure during Sasl Authenticate" % self)
             else:
                 self._sasl_mechanism.receive(recv_token)
 
         if self._sasl_mechanism.is_authenticated():
             log.info('%s: %s', self, self._sasl_mechanism.auth_details())
-            return future.success(True)
+            return True
         else:
-            return future.failure(Errors.SaslAuthenticationFailedError('Failed to authenticate via SASL %s' % self.config['sasl_mechanism']))
+            raise Errors.SaslAuthenticationFailedError('Failed to authenticate via SASL %s' % self.config['sasl_mechanism'])
 
     def blacked_out(self):
         """

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/consumer/fetcher.py

@@ -250,16 +250,17 @@ class Fetcher(six.Iterator):
                 break
 
             if future.succeeded():
-                fetched_offsets.update(future.value[0])
-                if not future.value[1]:
+                offsets, retry = future.value
+                fetched_offsets.update(offsets)
+                if not retry:
                     return fetched_offsets
 
-                timestamps = {tp: timestamps[tp] for tp in future.value[1]}
+                timestamps = {tp: timestamps[tp] for tp in retry}
 
             elif not future.retriable():
                 raise future.exception  # pylint: disable-msg=raising-bad-type
 
-            if future.exception.invalid_metadata or self._client.cluster.need_update:
+            elif future.exception.invalid_metadata or self._client.cluster.need_update:
                 refresh_future = self._client.cluster.request_update()
                 self._client.poll(future=refresh_future, timeout_ms=timer.timeout_ms)
 

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/consumer/group.py

@@ -163,6 +163,9 @@ class KafkaConsumer(six.Iterator):
             should be set no higher than 1/3 of that value. It can be
             adjusted even lower to control the expected time for normal
             rebalances. Default: 3000
+        receive_message_max_bytes (int): Maximum allowed network frame size.
+            Used to avoid OOM when decoding malformed network message header.
+            Default: 1000000.
         receive_buffer_bytes (int): The size of the TCP receive buffer
             (SO_RCVBUF) to use when reading data. Default: None (relies on
             system defaults). The java client defaults to 32768.
@@ -303,9 +306,8 @@ class KafkaConsumer(six.Iterator):
         'heartbeat_interval_ms': 3000,
         'receive_buffer_bytes': None,
         'send_buffer_bytes': None,
+        'receive_message_max_bytes': 1000000,
         'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
-        'sock_chunk_bytes': 4096,  # undocumented experimental option
-        'sock_chunk_buffer_count': 1000,  # undocumented experimental option
         'consumer_timeout_ms': float('inf'),
         'security_protocol': 'PLAINTEXT',
         'ssl_context': None,

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/coordinator/consumer.py

@@ -443,7 +443,8 @@ class ConsumerCoordinator(BaseCoordinator):
             self._client.poll(future=future, timeout_ms=timer.timeout_ms)
 
             if future.is_done:
-                del self._offset_fetch_futures[future_key]
+                if future_key in self._offset_fetch_futures:
+                    del self._offset_fetch_futures[future_key]
 
                 if future.succeeded():
                     return future.value

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/errors.py

@@ -57,6 +57,10 @@ class CorrelationIdError(KafkaProtocolError):
     retriable = True
 
 
+class InvalidReceiveError(KafkaProtocolError):
+    pass
+
+
 class KafkaTimeoutError(KafkaError):
     retriable = True
 

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/future.py

@@ -37,6 +37,10 @@ class Future(object):
             self.is_done = True
         if self._callbacks:
             self._call_backs('callback', self._callbacks, self.value)
+        # Clearing the lists releases any reference cycle held via stored
+        # bound methods (e.g. FutureProduceResult<->FutureRecordMetadata).
+        self._callbacks = None
+        self._errbacks = None
         return self
 
     def failure(self, e):
@@ -48,6 +52,10 @@ class Future(object):
             self.exception = exception
             self.is_done = True
         self._call_backs('errback', self._errbacks, self.exception)
+        # Clearing the lists releases any reference cycle held via stored
+        # bound methods (e.g. FutureProduceResult<->FutureRecordMetadata).
+        self._callbacks = None
+        self._errbacks = None
         return self
 
     def add_callback(self, f, *args, **kwargs):

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/producer/kafka.py

@@ -261,6 +261,9 @@ class KafkaProducer(object):
             errors. Default: 100.
         request_timeout_ms (int): Client request timeout in milliseconds.
             Default: 30000.
+        receive_message_max_bytes (int): Maximum allowed network frame size.
+            Used to avoid OOM when decoding malformed network message header.
+            Default: 1000000.
         receive_buffer_bytes (int): The size of the TCP receive buffer
             (SO_RCVBUF) to use when reading data. Default: None (relies on
             system defaults). Java client defaults to 32768.
@@ -392,11 +395,10 @@ class KafkaProducer(object):
         'metadata_max_age_ms': 300000,
         'retry_backoff_ms': 100,
         'request_timeout_ms': 30000,
+        'receive_message_max_bytes': 1000000,
         'receive_buffer_bytes': None,
         'send_buffer_bytes': None,
         'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
-        'sock_chunk_bytes': 4096,  # undocumented experimental option
-        'sock_chunk_buffer_count': 1000,  # undocumented experimental option
         'reconnect_backoff_ms': 50,
         'reconnect_backoff_max_ms': 30000,
         'max_in_flight_requests_per_connection': 5,

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/protocol/parser.py

@@ -23,12 +23,15 @@ class KafkaProtocol(object):
         api_version (tuple): Optional tuple to specify api_version to use.
             Currently only used to check for 0.8.2 protocol quirks, but
             may be used for more in the future.
+        max_frame_size (int): Maximum allowed message frame size.
+            Default: 100000000 (100MB).
     """
-    def __init__(self, client_id=None, api_version=None):
+    def __init__(self, client_id=None, api_version=None, max_frame_size=100000000):
         if client_id is None:
             client_id = self._gen_client_id()
         self._client_id = client_id
         self._api_version = api_version
+        self._max_frame_size = max_frame_size
         self._correlation_id = 0
         self._header = KafkaBytes(4)
         self._rbuffer = None
@@ -105,6 +108,7 @@ class KafkaProtocol(object):
                 if self._header.tell() == 4:
                     self._header.seek(0)
                     nbytes = Int32.decode(self._header)
+                    self._validate_frame_size(nbytes)
                     # reset buffer and switch state to receiving payload bytes
                     self._rbuffer = KafkaBytes(nbytes)
                     self._receiving = True
@@ -132,6 +136,10 @@ class KafkaProtocol(object):
                 self._reset_buffer()
         return responses
 
+    def _validate_frame_size(self, nbytes):
+        if nbytes < 0 or nbytes > self._max_frame_size:
+            raise Errors.InvalidReceiveError('Invalid frame length: %d' % nbytes)
+
     def _process_response(self, read_buffer):
         if not self.in_flight_requests:
             raise Errors.CorrelationIdError('No in-flight-request found for server response')

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/protocol/types.py

@@ -213,6 +213,17 @@ class Array(AbstractType):
 
 
 class UnsignedVarInt32(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value = VarInt32.decode(data)
+        return (value << 1) ^ (value >> 31)
+
+    @classmethod
+    def encode(cls, value):
+        return VarInt32.encode((value >> 1) ^ -(value & 1))
+
+
+class VarInt32(AbstractType):
     @classmethod
     def decode(cls, data):
         value, i = 0, 0
@@ -225,10 +236,12 @@ class UnsignedVarInt32(AbstractType):
             if i > 28:
                 raise ValueError('Invalid value {}'.format(value))
         value |= b << i
-        return value
+        return (value >> 1) ^ -(value & 1)
 
     @classmethod
     def encode(cls, value):
+        # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 31)
         value &= 0xffffffff
         ret = b''
         while (value & 0xffffff80) != 0:
@@ -239,25 +252,12 @@ class UnsignedVarInt32(AbstractType):
         return ret
 
 
-class VarInt32(AbstractType):
-    @classmethod
-    def decode(cls, data):
-        value = UnsignedVarInt32.decode(data)
-        return (value >> 1) ^ -(value & 1)
-
-    @classmethod
-    def encode(cls, value):
-        # bring it in line with the java binary repr
-        value &= 0xffffffff
-        return UnsignedVarInt32.encode((value << 1) ^ (value >> 31))
-
-
 class VarInt64(AbstractType):
     @classmethod
     def decode(cls, data):
         value, i = 0, 0
         while True:
-            b = data.read(1)
+            b, = struct.unpack('B', data.read(1))
             if not (b & 0x80):
                 break
             value |= (b & 0x7f) << i
@@ -270,14 +270,14 @@ class VarInt64(AbstractType):
     @classmethod
     def encode(cls, value):
         # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 63)
         value &= 0xffffffffffffffff
-        v = (value << 1) ^ (value >> 63)
         ret = b''
-        while (v & 0xffffffffffffff80) != 0:
+        while (value & 0xffffffffffffff80) != 0:
             b = (value & 0x7f) | 0x80
             ret += struct.pack('B', b)
-            v >>= 7
-        ret += struct.pack('B', v)
+            value >>= 7
+        ret += struct.pack('B', value)
         return ret
 
 
@@ -322,8 +322,9 @@ class TaggedFields(AbstractType):
         for k, v in value.items():
             # do we allow for other data types ?? It could get complicated really fast
             assert isinstance(v, bytes), 'Value {} is not a byte array'.format(v)
-            assert isinstance(k, int) and k > 0, 'Key {} is not a positive integer'.format(k)
+            assert isinstance(k, int) and k >= 0, 'Key {} is not a non-negative integer'.format(k)
             ret += UnsignedVarInt32.encode(k)
+            ret += UnsignedVarInt32.encode(len(v))
             ret += v
         return ret
 

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka/sasl/scram.py

@@ -106,7 +106,12 @@ class ScramClient:
         self.auth_message += b',c=biws,r=' + self.nonce
 
         salt = base64.b64decode(params['s'].encode('utf-8'))
-        iterations = int(params['i'])
+        try:
+            iterations = int(params['i'])
+            if iterations > 1000000:
+                raise ValueError('too many iterations')
+        except (TypeError, ValueError):
+            raise ValueError('Invalid value (not integer or too large) for Iteration count in server-first-message')
         self.create_salted_password(salt, iterations)
 
         self.client_key = self.hmac(self.salted_password, b'Client Key')

kafka_python-2.2.20/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '2.2.20'

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
-Metadata-Version: 2.4
+Metadata-Version: 2.1
 Name: kafka-python
-Version: 2.2.18
+Version: 2.2.20
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.18 → kafka_python-2.2.20}/kafka_python.egg-info/SOURCES.txt

@@ -149,7 +149,6 @@ test/test_package.py
 test/test_partition_movements.py
 test/test_partitioner.py
 test/test_producer.py
-test/test_protocol.py
 test/test_record_accumulator.py
 test/test_sender.py
 test/test_subscription_state.py

{kafka_python-2.2.18 → kafka_python-2.2.20}/test/integration/test_admin_integration.py

@@ -93,7 +93,7 @@ def test_create_describe_delete_acls(kafka_admin_client):
 def test_describe_configs_broker_resource_returns_configs(kafka_admin_client):
     """Tests that describe config returns configs for broker
     """
-    broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId
+    broker_id = kafka_admin_client._client.least_loaded_node()
     configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)])
 
     assert len(configs) == 1
@@ -121,7 +121,7 @@ def test_describe_configs_topic_resource_returns_configs(topic, kafka_admin_clie
 def test_describe_configs_mixed_resources_returns_configs(topic, kafka_admin_client):
     """Tests that describe config returns configs for mixed resource types (topic + broker)
     """
-    broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId
+    broker_id = kafka_admin_client._client.least_loaded_node()
     configs = kafka_admin_client.describe_configs([
         ConfigResource(ConfigResourceType.TOPIC, topic),
         ConfigResource(ConfigResourceType.BROKER, broker_id)])
@@ -146,14 +146,6 @@ def test_describe_configs_invalid_broker_id_raises(kafka_admin_client):
         kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)])
 
 
-@pytest.mark.skipif(env_kafka_version() < (0, 11), reason='Describe consumer group requires broker >=0.11')
-def test_describe_consumer_group_does_not_exist(kafka_admin_client):
-    """Tests that the describe consumer group call fails if the group coordinator is not available
-    """
-    with pytest.raises(CoordinatorNotAvailableError):
-        kafka_admin_client.describe_consumer_groups(['test'])
-
-
 @pytest.mark.skipif(env_kafka_version() < (0, 11), reason='Describe consumer group requires broker >=0.11')
 def test_describe_consumer_group_exists(kafka_admin_client, kafka_consumer_factory, topic):
     """Tests that the describe consumer group call returns valid consumer group information

{kafka_python-2.2.18 → kafka_python-2.2.20}/test/integration/test_consumer_integration.py

@@ -323,4 +323,4 @@ def test_kafka_consumer_position_after_seek_to_end(kafka_consumer_factory, topic
     position = consumer.position(tp, timeout_ms=1000)
 
     # Verify we got the expected position
-    assert position == 10, f"Expected position 10, got {position}"
+    assert position == 10, "Expected position 10, got {}".format(position)

{kafka_python-2.2.18 → kafka_python-2.2.20}/test/integration/test_sasl_integration.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import uuid
 import time
 
@@ -8,6 +9,7 @@ from kafka.admin import NewTopic
 from kafka.protocol.metadata import MetadataRequest_v1
 from test.testutil import assert_message_count, env_kafka_version, random_string, special_to_underscore
 
+pytestmark = pytest.mark.skipif("KAFKA_URI" in os.environ, reason="Testing on external Kafka Broker")
 
 @pytest.fixture(
     params=[

{kafka_python-2.2.18 → kafka_python-2.2.20}/test/test_codec.py

@@ -124,3 +124,14 @@ def test_zstd():
         b1 = random_string(100).encode('utf-8')
         b2 = zstd_decode(zstd_encode(b1))
         assert b1 == b2
+
+
+@pytest.mark.skipif(not has_zstd(), reason="Zstd not available")
+def test_zstd_multi_frame():
+    """Test that zstd_decode handles multiple concatenated zstd frames."""
+    frame1_data = b'some payload data ' * 100
+    frame2_data = b'another frame of data ' * 100
+    # Concatenate two independently compressed zstd frames
+    multi_frame_payload = zstd_encode(frame1_data) + zstd_encode(frame2_data)
+    result = zstd_decode(multi_frame_payload)
+    assert result == frame1_data + frame2_data