kafka-python 3.0.5__tar.gz → 3.0.7__tar.gz

{kafka_python-3.0.5 → kafka_python-3.0.7}/CHANGES.md

@@ -1,3 +1,17 @@
+# 3.0.7 (Jun 28, 2026)
+
+### Fixes
+* transport: cancel pending io coroutines on close() (#3109)
+* producer: fix stalled shutdown when brokers unreachable (#3108)
+
+# 3.0.6 (Jun 25, 2026)
+
+### Fixes
+* consumer: track current leader epoch in addition to record epoch (#3107)
+
+### CI
+* Also `wait_for_topics` on <0.10 test fixtures (#3105)
+
 # 3.0.5 (Jun 24, 2026)
 
 ### Fixes

{kafka_python-3.0.5 → kafka_python-3.0.7}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 3.0.5
+Version: 3.0.7
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 License-Expression: Apache-2.0

{kafka_python-3.0.5 → kafka_python-3.0.7}/kafka/consumer/subscription_state.py

@@ -564,6 +564,18 @@ class TopicPartitionState:
         # until OffsetForLeaderEpoch confirms the position is consistent with
         # the current leader's log; mutually exclusive with awaiting_reset.
         self._awaiting_validation = False
+        # Highest cluster leader epoch this position has already been reconciled
+        # against (a completed validation, or adoption of a position whose
+        # record epoch is unknown). This is deliberately distinct from the
+        # position's *record* epoch (``OffsetAndMetadata.leader_epoch``, which
+        # is sent as last_fetched_epoch for KIP-595 divergence detection): a
+        # leader election bumps the cluster epoch without changing the epoch of
+        # the record we're positioned on, and OffsetForLeaderEpoch reports the
+        # epoch of the requested (older) point, so the record epoch alone can't
+        # tell us whether we've validated against the current leader. Tracking
+        # it separately is what stops poll() from re-validating forever and
+        # stalling the consumer. See #3106.
+        self._current_leader_epoch = -1
         # KIP-392: preferred read replica chosen by the broker (rack-aware).
         # ``_preferred_read_replica_expiration`` is a monotonic deadline; after
         # it passes we fall back to the leader and re-learn. Cleared on
@@ -597,6 +609,7 @@ class TopicPartitionState:
         self._position = None
         self.next_allowed_retry_time = None
         self._awaiting_validation = False
+        self._current_leader_epoch = -1
         self.clear_preferred_read_replica()
 
     def is_reset_allowed(self):
@@ -625,6 +638,11 @@ class TopicPartitionState:
         self.drop_pending_record_batch = True
         self.next_allowed_retry_time = None
         self._awaiting_validation = False
+        # A freshly-seeked position has not been reconciled against any leader;
+        # force the next maybe_validate_position to validate it against the
+        # current leader (the prior position's validation says nothing about
+        # this offset).
+        self._current_leader_epoch = -1
         self.clear_preferred_read_replica()
 
     def pause(self):
@@ -686,7 +704,8 @@ class TopicPartitionState:
         return self._awaiting_validation
 
     def maybe_validate_position(self, current_leader_epoch):
-        """Mark for validation if current leader has advanced beyond our position's epoch.
+        """Mark for validation if the current leader has advanced beyond the
+        leader epoch this position was last reconciled against.
 
         Returns True if the partition is now awaiting validation.
         """
@@ -696,11 +715,26 @@ class TopicPartitionState:
             return False
         if current_leader_epoch is None or current_leader_epoch < 0:
             return False
-        # Positions without a known epoch (legacy data, post-seek to bare offset)
-        # can't be validated; treat as already-fetchable.
-        if self._position.leader_epoch < 0:
+        # Have we already reconciled this position against this leader epoch (or
+        # a newer one)? The floor is the max of:
+        #  - the record epoch of the position itself: a record we actually
+        #    fetched at this offset proves the position is valid through its
+        #    epoch (so a position fetched under the current leader needs no
+        #    validation), and
+        #  - the leader epoch a completed validation reconciled us against,
+        #    which can exceed the record epoch because OffsetForLeaderEpoch
+        #    reports the epoch of the requested (older) point, not the current
+        #    leader's. Without this second term the position is re-marked every
+        #    poll and the partition never becomes fetchable again (#3106).
+        reconciled_epoch = max(self._current_leader_epoch, self._position.leader_epoch)
+        if current_leader_epoch <= reconciled_epoch:
             return False
-        if self._position.leader_epoch >= current_leader_epoch:
+        # We are now tracking this leader epoch regardless of whether the
+        # position itself carries a record epoch we can validate against.
+        self._current_leader_epoch = current_leader_epoch
+        # Positions without a known record epoch (legacy data, post-seek to bare
+        # offset) can't be validated; treat as already-fetchable.
+        if self._position.leader_epoch < 0:
             return False
         self.clear_preferred_read_replica()
         self._awaiting_validation = True

{kafka_python-3.0.5 → kafka_python-3.0.7}/kafka/net/transport.py

@@ -19,6 +19,8 @@ class KafkaTCPTransport:
         self._closed = False
         self._write_buffer = deque()
         self._writing = False
+        self._read_task = None
+        self._write_task = None
         self._protocol = None
         self._read = False
         self._write = True
@@ -80,7 +82,7 @@ class KafkaTCPTransport:
         data_received() method.
         """
         if not self._read:
-            self._net.call_soon(self._read_from_sock)
+            self._read_task = self._net.call_soon(self._read_from_sock)
         self._read = True
         log.debug('%s: Resumed reading', self)
 
@@ -173,7 +175,7 @@ class KafkaTCPTransport:
         self._write_buffer.append(data)
         if not self._writing:
             self._writing = True
-            self._net.call_soon(self._write_to_sock)
+            self._write_task =  self._net.call_soon(self._write_to_sock)
 
     def writelines(self, list_of_data):
         """Write a list (or any iterable) of data bytes to the transport."""
@@ -182,7 +184,7 @@ class KafkaTCPTransport:
         self._write_buffer.extend(list_of_data)
         if not self._writing:
             self._writing = True
-            self._net.call_soon(self._write_to_sock)
+            self._write_task = self._net.call_soon(self._write_to_sock)
 
     async def _write_to_sock(self):
         try:
@@ -272,6 +274,10 @@ class KafkaTCPTransport:
             except OSError:
                 pass
             sock.close()
+        for task in (self._read_task, self._write_task):
+            if task is not None:
+                self._net.cancel(task)
+        self._read_task = self._write_task = None
         proto = self._protocol
         self._protocol = None
         if proto is not None:

{kafka_python-3.0.5 → kafka_python-3.0.7}/kafka/producer/kafka.py

@@ -646,7 +646,7 @@ class KafkaProducer:
         _self = weakref.proxy(self)
         def wrapper():
             try:
-                _self.close(timeout=0, null_logger=True)
+                _self.close(timeout=1, null_logger=True)
             except (ReferenceError, AttributeError):
                 pass
         return wrapper

{kafka_python-3.0.5 → kafka_python-3.0.7}/kafka/producer/sender.py

@@ -310,7 +310,7 @@ class Sender(threading.Thread):
             return False
 
         log.debug("%s: Sending transactional request %s", str(self), next_request_handler.request)
-        while not self._force_close:
+        while self._running and not self._force_close:
             target_node = None
             try:
                 if next_request_handler.needs_coordinator():
@@ -325,7 +325,8 @@ class Sender(threading.Thread):
                 else:
                     target_node = self._client.least_loaded_node()
                     if target_node is None:
-                        self._client.poll(future=self._metadata.request_update())
+                        self._client.poll(timeout_ms=self.config['retry_backoff_ms'],
+                                          future=self._metadata.request_update())
                     elif not self._client.await_ready(target_node, timeout_ms=self.config['request_timeout_ms']):
                         continue
 

kafka_python-3.0.7/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '3.0.7'

{kafka_python-3.0.5 → kafka_python-3.0.7}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 3.0.5
+Version: 3.0.7
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 License-Expression: Apache-2.0

{kafka_python-3.0.5 → kafka_python-3.0.7}/test/integration/fixtures.py

@@ -34,21 +34,29 @@ def create_topics(broker, topic_names, num_partitions=None, replication_factor=N
         for topic_name in topic_names:
             # TODO: verify kafka-topics.sh support for early 0.8 brokers
             broker._create_topic_via_cli(topic_name, num_partitions, replication_factor)
+        _wait_for_topics(broker, topic_names)
 
 
 def _create_topics_via_admin(broker, topic_names, num_partitions, replication_factor):
-    from kafka.admin import NewTopic
-    params = broker._enrich_client_params({}, client_id='topic_creator')
-    admin = KafkaAdminClient(**params)
-    try:
-        topics = [NewTopic(name, num_partitions, replication_factor) for name in topic_names]
-        admin.create_topics(topics, wait_for_metadata=True)
-    except InvalidReplicationFactorError:
-        time.sleep(0.5)
-        topics = [NewTopic(name, num_partitions, replication_factor) for name in topic_names]
-        admin.create_topics(topics, wait_for_metadata=True)
-    finally:
-        admin.close()
+    params = broker._enrich_client_params({'api_version': env_kafka_version(), 'client_id': 'topic_creator'})
+    with KafkaAdminClient(**params) as admin:
+        try:
+            topics = {name: {'num_partitions': num_partitions, 'replication_factor': replication_factor}
+                      for name in topic_names}
+            admin.create_topics(topics, wait_for_metadata=True)
+        except InvalidReplicationFactorError:
+            # wait and try again
+            # in CI the brokers sometimes take a while to find themselves
+            time.sleep(0.5)
+            topics = {name: {'num_partitions': num_partitions, 'replication_factor': replication_factor}
+                      for name in topic_names}
+            admin.create_topics(topics, wait_for_metadata=True)
+
+
+def _wait_for_topics(broker, topics):
+    params = broker._enrich_client_params({'api_version': env_kafka_version(), 'client_id': 'topic_creator'})
+    with KafkaAdminClient(**params) as admin:
+        admin.wait_for_topics(topics)
 
 
 def client_params(broker, client_id='client', **overrides):

{kafka_python-3.0.5 → kafka_python-3.0.7}/test/net/test_transport.py

@@ -6,7 +6,7 @@ import pytest
 
 import kafka.errors as Errors
 from kafka.future import Future
-from kafka.net.selector import NetworkSelector
+from kafka.net.selector import NetworkSelector, TaskState
 from kafka.net.transport import KafkaTCPTransport
 
 
@@ -365,3 +365,58 @@ class TestKafkaTCPTransport:
         t._closed = True
         s = str(t)
         assert 'closed' in s
+
+
+class TestTransportWaiterCleanup:
+    """Regression: a locally-initiated close()/abort() must reclaim the socket
+    read/write coroutine tasks parked in the event loop.
+
+    These tests fail until ``KafkaTCPTransport._close`` cancels its read/write
+    waiter tasks (``net.cancel(task)``); the selector's existing WAIT_IO branch
+    in ``cancel()`` then drives the io_guard finalizer and discards the task.
+    """
+
+    def test_local_close_reclaims_parked_reader(self, net, socketpair):
+        rsock, wsock = socketpair
+        t = KafkaTCPTransport(net, wsock)
+        t.set_protocol(MagicMock())
+        baseline = len(net._pending_tasks)
+
+        t.resume_reading()
+        net.drain()  # reader runs and parks in WAIT_IO on wait_read(wsock)
+        parked = [task for task in net._pending_tasks if task.state is TaskState.WAIT_IO]
+        assert len(parked) == 1, 'reader did not park as expected'
+        assert len(net._pending_tasks) == baseline + 1
+
+        # Empty write buffer -> close() tears the socket down synchronously. The
+        # peer (rsock) never sent anything, so there is no I/O event to wake the
+        # parked reader; close() itself must reclaim it.
+        t.close()
+        net.drain()  # running the loop must not be needed -- and must not help either
+
+        assert t._sock is None
+        assert len(net._pending_tasks) == baseline, (
+            'parked reader leaked into _pending_tasks after local close')
+        assert not any(task.state is TaskState.WAIT_IO for task in net._pending_tasks)
+
+    def test_local_close_reclaims_parked_writer(self, net, socketpair):
+        rsock, wsock = socketpair
+        t = KafkaTCPTransport(net, wsock)
+        t.set_protocol(MagicMock())
+        baseline = len(net._pending_tasks)
+
+        # write() schedules _write_to_sock, whose loop parks on the first
+        # wait_write before the bytes leave the buffer. drain() steps it once
+        # to the park and returns (no _ready left), leaving it suspended.
+        t.write(b'data')
+        net.drain()
+        parked = [task for task in net._pending_tasks if task.state is TaskState.WAIT_IO]
+        assert len(parked) == 1, 'writer did not park as expected'
+
+        t.abort(error=Errors.KafkaConnectionError('boom'))
+        net.drain()
+
+        assert t._sock is None
+        assert len(net._pending_tasks) == baseline, (
+            'parked writer leaked into _pending_tasks after abort')
+        assert not any(task.state is TaskState.WAIT_IO for task in net._pending_tasks)

kafka_python-3.0.5/kafka/version.py

@@ -1 +0,0 @@
-__version__ = '3.0.5'