kafka-python 2.2.17__tar.gz → 2.2.19__tar.gz

{kafka_python-2.2.17 → kafka_python-2.2.19}/CHANGES.md

@@ -1,3 +1,19 @@
+# 2.2.19 (Apr 9, 2026)
+
+Fixes
+* Fix TaggedFields value encoding; add test coverage (#2725)
+* Fix zstd multi-frame decompression failure (#2717)
+* Fix KeyError in KafkaConsumer.committed() (#2710)
+* Fix VarInt/VarLong encoding; move tests to test/protocol/ (#2706)
+* Fix `Fetcher._fetch_offsets_by_times retry handling` (#2833)
+* Fixes to support integration testing with external KAFKA_URI (#2838)
+* Minor py2 fixes: consumer integration test; done pip install python-snappy
+
+# 2.2.18 (Nov 20, 2025)
+
+Fixes
+* Add ProducerBatch.__lt__ for heapq (#2698)
+
 # 2.2.17 (Nov 20, 2025)
 
 Fixes

{kafka_python-2.2.17 → kafka_python-2.2.19}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.17
+Version: 2.2.19
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.17 → kafka_python-2.2.19}/kafka/codec.py

@@ -327,7 +327,5 @@ def zstd_encode(payload):
 def zstd_decode(payload):
     if not zstd:
         raise NotImplementedError("Zstd codec is not available")
-    try:
-        return zstd.ZstdDecompressor().decompress(payload)
-    except zstd.ZstdError:
-        return zstd.ZstdDecompressor().decompress(payload, max_output_size=ZSTD_MAX_OUTPUT_SIZE)
+    with zstd.ZstdDecompressor().stream_reader(io.BytesIO(payload), read_across_frames=True) as reader:
+        return reader.read()

{kafka_python-2.2.17 → kafka_python-2.2.19}/kafka/consumer/fetcher.py

@@ -250,16 +250,17 @@ class Fetcher(six.Iterator):
                 break
 
             if future.succeeded():
-                fetched_offsets.update(future.value[0])
-                if not future.value[1]:
+                offsets, retry = future.value
+                fetched_offsets.update(offsets)
+                if not retry:
                     return fetched_offsets
 
-                timestamps = {tp: timestamps[tp] for tp in future.value[1]}
+                timestamps = {tp: timestamps[tp] for tp in retry}
 
             elif not future.retriable():
                 raise future.exception  # pylint: disable-msg=raising-bad-type
 
-            if future.exception.invalid_metadata or self._client.cluster.need_update:
+            elif future.exception.invalid_metadata or self._client.cluster.need_update:
                 refresh_future = self._client.cluster.request_update()
                 self._client.poll(future=refresh_future, timeout_ms=timer.timeout_ms)
 

{kafka_python-2.2.17 → kafka_python-2.2.19}/kafka/coordinator/consumer.py

@@ -443,7 +443,8 @@ class ConsumerCoordinator(BaseCoordinator):
             self._client.poll(future=future, timeout_ms=timer.timeout_ms)
 
             if future.is_done:
-                del self._offset_fetch_futures[future_key]
+                if future_key in self._offset_fetch_futures:
+                    del self._offset_fetch_futures[future_key]
 
                 if future.succeeded():
                     return future.value

{kafka_python-2.2.17 → kafka_python-2.2.19}/kafka/producer/record_accumulator.py

@@ -179,6 +179,10 @@ class ProducerBatch(object):
         return 'ProducerBatch(topic_partition=%s, record_count=%d)' % (
             self.topic_partition, self.records.next_offset())
 
+    # for heapq
+    def __lt__(self, other):
+        return self.created < other.created
+
 
 class RecordAccumulator(object):
     """

{kafka_python-2.2.17 → kafka_python-2.2.19}/kafka/protocol/types.py

@@ -213,6 +213,17 @@ class Array(AbstractType):
 
 
 class UnsignedVarInt32(AbstractType):
+    @classmethod
+    def decode(cls, data):
+        value = VarInt32.decode(data)
+        return (value << 1) ^ (value >> 31)
+
+    @classmethod
+    def encode(cls, value):
+        return VarInt32.encode((value >> 1) ^ -(value & 1))
+
+
+class VarInt32(AbstractType):
     @classmethod
     def decode(cls, data):
         value, i = 0, 0
@@ -225,10 +236,12 @@ class UnsignedVarInt32(AbstractType):
             if i > 28:
                 raise ValueError('Invalid value {}'.format(value))
         value |= b << i
-        return value
+        return (value >> 1) ^ -(value & 1)
 
     @classmethod
     def encode(cls, value):
+        # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 31)
         value &= 0xffffffff
         ret = b''
         while (value & 0xffffff80) != 0:
@@ -239,25 +252,12 @@ class UnsignedVarInt32(AbstractType):
         return ret
 
 
-class VarInt32(AbstractType):
-    @classmethod
-    def decode(cls, data):
-        value = UnsignedVarInt32.decode(data)
-        return (value >> 1) ^ -(value & 1)
-
-    @classmethod
-    def encode(cls, value):
-        # bring it in line with the java binary repr
-        value &= 0xffffffff
-        return UnsignedVarInt32.encode((value << 1) ^ (value >> 31))
-
-
 class VarInt64(AbstractType):
     @classmethod
     def decode(cls, data):
         value, i = 0, 0
         while True:
-            b = data.read(1)
+            b, = struct.unpack('B', data.read(1))
             if not (b & 0x80):
                 break
             value |= (b & 0x7f) << i
@@ -270,14 +270,14 @@ class VarInt64(AbstractType):
     @classmethod
     def encode(cls, value):
         # bring it in line with the java binary repr
+        value = (value << 1) ^ (value >> 63)
         value &= 0xffffffffffffffff
-        v = (value << 1) ^ (value >> 63)
         ret = b''
-        while (v & 0xffffffffffffff80) != 0:
+        while (value & 0xffffffffffffff80) != 0:
             b = (value & 0x7f) | 0x80
             ret += struct.pack('B', b)
-            v >>= 7
-        ret += struct.pack('B', v)
+            value >>= 7
+        ret += struct.pack('B', value)
         return ret
 
 
@@ -322,8 +322,9 @@ class TaggedFields(AbstractType):
         for k, v in value.items():
             # do we allow for other data types ?? It could get complicated really fast
             assert isinstance(v, bytes), 'Value {} is not a byte array'.format(v)
-            assert isinstance(k, int) and k > 0, 'Key {} is not a positive integer'.format(k)
+            assert isinstance(k, int) and k >= 0, 'Key {} is not a non-negative integer'.format(k)
             ret += UnsignedVarInt32.encode(k)
+            ret += UnsignedVarInt32.encode(len(v))
             ret += v
         return ret
 

kafka_python-2.2.19/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '2.2.19'

{kafka_python-2.2.17 → kafka_python-2.2.19}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.17
+Version: 2.2.19
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.17 → kafka_python-2.2.19}/kafka_python.egg-info/SOURCES.txt

@@ -149,7 +149,6 @@ test/test_package.py
 test/test_partition_movements.py
 test/test_partitioner.py
 test/test_producer.py
-test/test_protocol.py
 test/test_record_accumulator.py
 test/test_sender.py
 test/test_subscription_state.py

{kafka_python-2.2.17 → kafka_python-2.2.19}/test/integration/test_admin_integration.py

@@ -93,7 +93,7 @@ def test_create_describe_delete_acls(kafka_admin_client):
 def test_describe_configs_broker_resource_returns_configs(kafka_admin_client):
     """Tests that describe config returns configs for broker
     """
-    broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId
+    broker_id = kafka_admin_client._client.least_loaded_node()
     configs = kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)])
 
     assert len(configs) == 1
@@ -121,7 +121,7 @@ def test_describe_configs_topic_resource_returns_configs(topic, kafka_admin_clie
 def test_describe_configs_mixed_resources_returns_configs(topic, kafka_admin_client):
     """Tests that describe config returns configs for mixed resource types (topic + broker)
     """
-    broker_id = kafka_admin_client._client.cluster._brokers[0].nodeId
+    broker_id = kafka_admin_client._client.least_loaded_node()
     configs = kafka_admin_client.describe_configs([
         ConfigResource(ConfigResourceType.TOPIC, topic),
         ConfigResource(ConfigResourceType.BROKER, broker_id)])
@@ -146,14 +146,6 @@ def test_describe_configs_invalid_broker_id_raises(kafka_admin_client):
         kafka_admin_client.describe_configs([ConfigResource(ConfigResourceType.BROKER, broker_id)])
 
 
-@pytest.mark.skipif(env_kafka_version() < (0, 11), reason='Describe consumer group requires broker >=0.11')
-def test_describe_consumer_group_does_not_exist(kafka_admin_client):
-    """Tests that the describe consumer group call fails if the group coordinator is not available
-    """
-    with pytest.raises(CoordinatorNotAvailableError):
-        kafka_admin_client.describe_consumer_groups(['test'])
-
-
 @pytest.mark.skipif(env_kafka_version() < (0, 11), reason='Describe consumer group requires broker >=0.11')
 def test_describe_consumer_group_exists(kafka_admin_client, kafka_consumer_factory, topic):
     """Tests that the describe consumer group call returns valid consumer group information

{kafka_python-2.2.17 → kafka_python-2.2.19}/test/integration/test_consumer_integration.py

@@ -323,4 +323,4 @@ def test_kafka_consumer_position_after_seek_to_end(kafka_consumer_factory, topic
     position = consumer.position(tp, timeout_ms=1000)
 
     # Verify we got the expected position
-    assert position == 10, f"Expected position 10, got {position}"
+    assert position == 10, "Expected position 10, got {}".format(position)

{kafka_python-2.2.17 → kafka_python-2.2.19}/test/integration/test_sasl_integration.py

@@ -1,4 +1,5 @@
 import logging
+import os
 import uuid
 import time
 
@@ -8,6 +9,7 @@ from kafka.admin import NewTopic
 from kafka.protocol.metadata import MetadataRequest_v1
 from test.testutil import assert_message_count, env_kafka_version, random_string, special_to_underscore
 
+pytestmark = pytest.mark.skipif("KAFKA_URI" in os.environ, reason="Testing on external Kafka Broker")
 
 @pytest.fixture(
     params=[

{kafka_python-2.2.17 → kafka_python-2.2.19}/test/test_codec.py

@@ -124,3 +124,14 @@ def test_zstd():
         b1 = random_string(100).encode('utf-8')
         b2 = zstd_decode(zstd_encode(b1))
         assert b1 == b2
+
+
+@pytest.mark.skipif(not has_zstd(), reason="Zstd not available")
+def test_zstd_multi_frame():
+    """Test that zstd_decode handles multiple concatenated zstd frames."""
+    frame1_data = b'some payload data ' * 100
+    frame2_data = b'another frame of data ' * 100
+    # Concatenate two independently compressed zstd frames
+    multi_frame_payload = zstd_encode(frame1_data) + zstd_encode(frame2_data)
+    result = zstd_decode(multi_frame_payload)
+    assert result == frame1_data + frame2_data

{kafka_python-2.2.17 → kafka_python-2.2.19}/test/test_fetcher.py

@@ -770,3 +770,190 @@ def test_seek_before_exception(client, mocker):
     # Should not throw OffsetOutOfRangeError after the seek
     records, partial = fetcher.fetched_records()
     assert len(records) == 0
+
+
+class TestFetchOffsetsByTimes:
+    def _make_fetcher(self, client, mocker):
+        subscription_state = SubscriptionState()
+        subscription_state.subscribe(topics=['test'])
+        tp = TopicPartition('test', 0)
+        subscription_state.assign_from_subscribed([tp])
+        subscription_state.seek(tp, 0)
+        return Fetcher(client, subscription_state)
+
+    def test_empty_timestamps(self, client, metrics, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        assert fetcher._fetch_offsets_by_times({}) == {}
+
+    def test_success_no_retry(self, client, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        tp = TopicPartition('test', 0)
+        timestamps = {tp: 1000}
+        expected_offset = OffsetAndTimestamp(10, 1000, -1)
+
+        future = Future()
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', return_value=future)
+        mocker.patch.object(fetcher._client, 'poll', side_effect=lambda **kw: future.success(({tp: expected_offset}, set())))
+
+        result = fetcher._fetch_offsets_by_times(timestamps, timeout_ms=10000)
+        assert result == {tp: expected_offset}
+
+    def test_success_with_retry(self, client, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        tp0 = TopicPartition('test', 0)
+        tp1 = TopicPartition('test', 1)
+        timestamps = {tp0: 1000, tp1: 2000}
+        offset0 = OffsetAndTimestamp(10, 1000, -1)
+        offset1 = OffsetAndTimestamp(20, 2000, -1)
+
+        # First call succeeds for tp0 but needs retry for tp1
+        future1 = Future()
+        future2 = Future()
+        futures = iter([future1, future2])
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', side_effect=lambda ts: next(futures))
+
+        def poll_side_effect(**kw):
+            f = kw.get('future')
+            if f is future1:
+                f.success(({tp0: offset0}, {tp1}))
+            elif f is future2:
+                f.success(({tp1: offset1}, set()))
+
+        mocker.patch.object(fetcher._client, 'poll', side_effect=poll_side_effect)
+
+        result = fetcher._fetch_offsets_by_times(timestamps, timeout_ms=10000)
+        assert result == {tp0: offset0, tp1: offset1}
+
+    def test_timeout_raises(self, client, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        tp = TopicPartition('test', 0)
+        timestamps = {tp: 1000}
+
+        future = Future()
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', return_value=future)
+        # poll does not complete the future
+        mocker.patch.object(fetcher._client, 'poll')
+
+        with pytest.raises(Errors.KafkaTimeoutError):
+            fetcher._fetch_offsets_by_times(timestamps, timeout_ms=10000)
+
+    def test_non_retriable_error_raises(self, client, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        tp = TopicPartition('test', 0)
+        timestamps = {tp: 1000}
+
+        future = Future()
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', return_value=future)
+        # AuthorizationError is not retriable
+        error = Errors.TopicAuthorizationFailedError()
+        mocker.patch.object(fetcher._client, 'poll', side_effect=lambda **kw: future.failure(error))
+
+        with pytest.raises(Errors.TopicAuthorizationFailedError):
+            fetcher._fetch_offsets_by_times(timestamps, timeout_ms=10000)
+
+    def test_retriable_invalid_metadata_triggers_refresh(self, client, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        tp = TopicPartition('test', 0)
+        timestamps = {tp: 1000}
+        expected_offset = OffsetAndTimestamp(10, 1000, -1)
+
+        # First call fails with invalid_metadata error, second succeeds
+        future1 = Future()
+        future2 = Future()
+        futures = iter([future1, future2])
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', side_effect=lambda ts: next(futures))
+
+        refresh_future = Future()
+        mocker.patch.object(fetcher._client.cluster, 'request_update', return_value=refresh_future)
+
+        call_count = [0]
+        def poll_side_effect(**kw):
+            f = kw.get('future')
+            if f is future1:
+                f.failure(NotLeaderForPartitionError())
+            elif f is refresh_future:
+                refresh_future.success(None)
+            elif f is future2:
+                f.success(({tp: expected_offset}, set()))
+            call_count[0] += 1
+
+        mocker.patch.object(fetcher._client, 'poll', side_effect=poll_side_effect)
+
+        result = fetcher._fetch_offsets_by_times(timestamps, timeout_ms=10000)
+        assert result == {tp: expected_offset}
+        fetcher._client.cluster.request_update.assert_called_once()
+
+    def test_retriable_non_metadata_error_sleeps(self, client, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        tp = TopicPartition('test', 0)
+        timestamps = {tp: 1000}
+        expected_offset = OffsetAndTimestamp(10, 1000, -1)
+
+        # RequestTimedOutError is retriable but not invalid_metadata
+        future1 = Future()
+        future2 = Future()
+        futures = iter([future1, future2])
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', side_effect=lambda ts: next(futures))
+
+        # Ensure cluster does not need update
+        mocker.patch.object(type(fetcher._client.cluster), 'need_update', new_callable=mocker.PropertyMock, return_value=False)
+
+        def poll_side_effect(**kw):
+            f = kw.get('future')
+            if f is future1:
+                f.failure(Errors.RequestTimedOutError())
+            elif f is future2:
+                f.success(({tp: expected_offset}, set()))
+
+        mocker.patch.object(fetcher._client, 'poll', side_effect=poll_side_effect)
+        mock_sleep = mocker.patch('time.sleep')
+
+        result = fetcher._fetch_offsets_by_times(timestamps, timeout_ms=10000)
+        assert result == {tp: expected_offset}
+        mock_sleep.assert_called_once()
+
+    def test_success_does_not_check_exception(self, client, mocker):
+        """Regression: successful future should not fall through to check future.exception."""
+        fetcher = self._make_fetcher(client, mocker)
+        tp0 = TopicPartition('test', 0)
+        tp1 = TopicPartition('test', 1)
+        timestamps = {tp0: 1000, tp1: 2000}
+        offset0 = OffsetAndTimestamp(10, 1000, -1)
+        offset1 = OffsetAndTimestamp(20, 2000, -1)
+
+        future1 = Future()
+        future2 = Future()
+        futures = iter([future1, future2])
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', side_effect=lambda ts: next(futures))
+
+        def poll_side_effect(**kw):
+            f = kw.get('future')
+            if f is future1:
+                # Succeeds but has retry partitions -- the bug was that code
+                # would fall through to check future.exception (which is None),
+                # causing an AttributeError
+                f.success(({tp0: offset0}, {tp1}))
+            elif f is future2:
+                f.success(({tp1: offset1}, set()))
+
+        mocker.patch.object(fetcher._client, 'poll', side_effect=poll_side_effect)
+
+        # Should not raise AttributeError
+        result = fetcher._fetch_offsets_by_times(timestamps, timeout_ms=10000)
+        assert result == {tp0: offset0, tp1: offset1}
+
+    def test_no_timeout_passes_none(self, client, mocker):
+        fetcher = self._make_fetcher(client, mocker)
+        tp = TopicPartition('test', 0)
+        timestamps = {tp: 1000}
+        expected_offset = OffsetAndTimestamp(10, 1000, -1)
+
+        future = Future()
+        mocker.patch.object(fetcher, '_send_list_offsets_requests', return_value=future)
+        mocker.patch.object(fetcher._client, 'poll', side_effect=lambda **kw: future.success(({tp: expected_offset}, set())))
+
+        result = fetcher._fetch_offsets_by_times(timestamps, timeout_ms=None)
+        assert result == {tp: expected_offset}
+        # With timeout_ms=None, poll should receive None timeout
+        fetcher._client.poll.assert_called_once()
+        assert fetcher._client.poll.call_args[1]['timeout_ms'] is None

{kafka_python-2.2.17 → kafka_python-2.2.19}/test/test_record_accumulator.py

@@ -120,6 +120,22 @@ def test_batch_cannot_complete_twice():
     assert record_metadata.offset == 500
     assert record_metadata.timestamp == 10
 
+def test_producer_batch_lt(tp):
+    records = MemoryRecordsBuilder(
+        magic=2, compression_type=0, batch_size=100000)
+    b1 = ProducerBatch(tp, records, now=1)
+    b2 = ProducerBatch(tp, records, now=2)
+
+    assert b1 < b2
+    assert not b1 < b1
+
+    import heapq
+    q = []
+    heapq.heappush(q, b2)
+    heapq.heappush(q, b1)
+    assert q[0] == b1
+    assert q[1] == b2
+
 def test_linger(tp, cluster):
     now = 0
     accum = RecordAccumulator(linger_ms=10)

kafka_python-2.2.17/kafka/version.py

@@ -1 +0,0 @@
-__version__ = '2.2.17'