kafka-python 3.0.5__py3-none-any.whl → 3.0.7__py3-none-any.whl

kafka/consumer/subscription_state.py

@@ -564,6 +564,18 @@ class TopicPartitionState:
         # until OffsetForLeaderEpoch confirms the position is consistent with
         # the current leader's log; mutually exclusive with awaiting_reset.
         self._awaiting_validation = False
+        # Highest cluster leader epoch this position has already been reconciled
+        # against (a completed validation, or adoption of a position whose
+        # record epoch is unknown). This is deliberately distinct from the
+        # position's *record* epoch (``OffsetAndMetadata.leader_epoch``, which
+        # is sent as last_fetched_epoch for KIP-595 divergence detection): a
+        # leader election bumps the cluster epoch without changing the epoch of
+        # the record we're positioned on, and OffsetForLeaderEpoch reports the
+        # epoch of the requested (older) point, so the record epoch alone can't
+        # tell us whether we've validated against the current leader. Tracking
+        # it separately is what stops poll() from re-validating forever and
+        # stalling the consumer. See #3106.
+        self._current_leader_epoch = -1
         # KIP-392: preferred read replica chosen by the broker (rack-aware).
         # ``_preferred_read_replica_expiration`` is a monotonic deadline; after
         # it passes we fall back to the leader and re-learn. Cleared on
@@ -597,6 +609,7 @@ class TopicPartitionState:
         self._position = None
         self.next_allowed_retry_time = None
         self._awaiting_validation = False
+        self._current_leader_epoch = -1
         self.clear_preferred_read_replica()
 
     def is_reset_allowed(self):
@@ -625,6 +638,11 @@ class TopicPartitionState:
         self.drop_pending_record_batch = True
         self.next_allowed_retry_time = None
         self._awaiting_validation = False
+        # A freshly-seeked position has not been reconciled against any leader;
+        # force the next maybe_validate_position to validate it against the
+        # current leader (the prior position's validation says nothing about
+        # this offset).
+        self._current_leader_epoch = -1
         self.clear_preferred_read_replica()
 
     def pause(self):
@@ -686,7 +704,8 @@ class TopicPartitionState:
         return self._awaiting_validation
 
     def maybe_validate_position(self, current_leader_epoch):
-        """Mark for validation if current leader has advanced beyond our position's epoch.
+        """Mark for validation if the current leader has advanced beyond the
+        leader epoch this position was last reconciled against.
 
         Returns True if the partition is now awaiting validation.
         """
@@ -696,11 +715,26 @@ class TopicPartitionState:
             return False
         if current_leader_epoch is None or current_leader_epoch < 0:
             return False
-        # Positions without a known epoch (legacy data, post-seek to bare offset)
-        # can't be validated; treat as already-fetchable.
-        if self._position.leader_epoch < 0:
+        # Have we already reconciled this position against this leader epoch (or
+        # a newer one)? The floor is the max of:
+        #  - the record epoch of the position itself: a record we actually
+        #    fetched at this offset proves the position is valid through its
+        #    epoch (so a position fetched under the current leader needs no
+        #    validation), and
+        #  - the leader epoch a completed validation reconciled us against,
+        #    which can exceed the record epoch because OffsetForLeaderEpoch
+        #    reports the epoch of the requested (older) point, not the current
+        #    leader's. Without this second term the position is re-marked every
+        #    poll and the partition never becomes fetchable again (#3106).
+        reconciled_epoch = max(self._current_leader_epoch, self._position.leader_epoch)
+        if current_leader_epoch <= reconciled_epoch:
             return False
-        if self._position.leader_epoch >= current_leader_epoch:
+        # We are now tracking this leader epoch regardless of whether the
+        # position itself carries a record epoch we can validate against.
+        self._current_leader_epoch = current_leader_epoch
+        # Positions without a known record epoch (legacy data, post-seek to bare
+        # offset) can't be validated; treat as already-fetchable.
+        if self._position.leader_epoch < 0:
             return False
         self.clear_preferred_read_replica()
         self._awaiting_validation = True

kafka/net/transport.py

@@ -19,6 +19,8 @@ class KafkaTCPTransport:
         self._closed = False
         self._write_buffer = deque()
         self._writing = False
+        self._read_task = None
+        self._write_task = None
         self._protocol = None
         self._read = False
         self._write = True
@@ -80,7 +82,7 @@ class KafkaTCPTransport:
         data_received() method.
         """
         if not self._read:
-            self._net.call_soon(self._read_from_sock)
+            self._read_task = self._net.call_soon(self._read_from_sock)
         self._read = True
         log.debug('%s: Resumed reading', self)
 
@@ -173,7 +175,7 @@ class KafkaTCPTransport:
         self._write_buffer.append(data)
         if not self._writing:
             self._writing = True
-            self._net.call_soon(self._write_to_sock)
+            self._write_task =  self._net.call_soon(self._write_to_sock)
 
     def writelines(self, list_of_data):
         """Write a list (or any iterable) of data bytes to the transport."""
@@ -182,7 +184,7 @@ class KafkaTCPTransport:
         self._write_buffer.extend(list_of_data)
         if not self._writing:
             self._writing = True
-            self._net.call_soon(self._write_to_sock)
+            self._write_task = self._net.call_soon(self._write_to_sock)
 
     async def _write_to_sock(self):
         try:
@@ -272,6 +274,10 @@ class KafkaTCPTransport:
             except OSError:
                 pass
             sock.close()
+        for task in (self._read_task, self._write_task):
+            if task is not None:
+                self._net.cancel(task)
+        self._read_task = self._write_task = None
         proto = self._protocol
         self._protocol = None
         if proto is not None:

kafka/producer/kafka.py

@@ -646,7 +646,7 @@ class KafkaProducer:
         _self = weakref.proxy(self)
         def wrapper():
             try:
-                _self.close(timeout=0, null_logger=True)
+                _self.close(timeout=1, null_logger=True)
             except (ReferenceError, AttributeError):
                 pass
         return wrapper

kafka/producer/sender.py

@@ -310,7 +310,7 @@ class Sender(threading.Thread):
             return False
 
         log.debug("%s: Sending transactional request %s", str(self), next_request_handler.request)
-        while not self._force_close:
+        while self._running and not self._force_close:
             target_node = None
             try:
                 if next_request_handler.needs_coordinator():
@@ -325,7 +325,8 @@ class Sender(threading.Thread):
                 else:
                     target_node = self._client.least_loaded_node()
                     if target_node is None:
-                        self._client.poll(future=self._metadata.request_update())
+                        self._client.poll(timeout_ms=self.config['retry_backoff_ms'],
+                                          future=self._metadata.request_update())
                     elif not self._client.await_ready(target_node, timeout_ms=self.config['request_timeout_ms']):
                         continue
 

kafka/version.py

@@ -1 +1 @@
-__version__ = '3.0.5'
+__version__ = '3.0.7'

{kafka_python-3.0.5.dist-info → kafka_python-3.0.7.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 3.0.5
+Version: 3.0.7
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 License-Expression: Apache-2.0

{kafka_python-3.0.5.dist-info → kafka_python-3.0.7.dist-info}/RECORD

@@ -6,7 +6,7 @@ kafka/errors.py,sha256=9owgtTIcttdTCD1vAF9wz5jL9G72ljpR7M-vjLKEGvY,33206
 kafka/future.py,sha256=INmMIy2-BPxJ4zlw-V_i8Tj_OG9DMbyYyUEjOQ5pn_o,5440
 kafka/structs.py,sha256=ZOFmVJRbhSXykkw9cnuv-DGYE7g2KjaUGsytSOv4xVw,3146
 kafka/util.py,sha256=hvSdRbh5-IsUbpD_BqjabjEldtNmXufZH68b_xCkNjE,5194
-kafka/version.py,sha256=kjxhuunOgPeuBYEEfHd5pRF6GonyFT-l4iWLchFfG1g,22
+kafka/version.py,sha256=nrBN-YleEph5Asdo18FBZNm74FlORYq6jGrwHBllWIU,22
 kafka/admin/__init__.py,sha256=nTXg2CQo8Bydxsj80pVBzF2-jq4IhK7_gLnboeNWG60,1575
 kafka/admin/__main__.py,sha256=l5ujkHa3lBFEbKPCl7U0_pi6IIgj1IJfmB219vtcj50,69
 kafka/admin/_acls.py,sha256=QJlHguEM9t46qlABoQqTEDFQErDgG2RKwfV1NOKeKmU,13983
@@ -85,7 +85,7 @@ kafka/consumer/__init__.py,sha256=I7nuPUb103S1CvCqcudbIRG_isJbRgTXkK1mpHIhoVs,82
 kafka/consumer/__main__.py,sha256=bnxM30LbGVvLb0ZBcgJEt_8bZHADJpboHZCxzd8QEhA,72
 kafka/consumer/fetcher.py,sha256=Xr-JNlyNZgVNanh6AlYn1ImsyuBbpMqjPGwMZL50Hmc,106075
 kafka/consumer/group.py,sha256=lJo2U_wHSk_j-qwfa2GeU_gOZq5kWbxh2qAkjyMer6Q,68423
-kafka/consumer/subscription_state.py,sha256=H1bDpDdy3uk3U8TtyjCuo_HmaZmSpOYevaYtrc7LvgY,38567
+kafka/consumer/subscription_state.py,sha256=0ixLOlO9KIxzFD0eMkLqrUjSJ44CMkkcvCQGWxt7nJg,40883
 kafka/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kafka/coordinator/base.py,sha256=UgMXWES0VVgDsvLEEZB8tAvchGFRzJFO6N9bL06vGm4,57794
 kafka/coordinator/consumer.py,sha256=cQ-zo0p_lL5H3j8DT4Q6ZrBBxCWWh4p1ub1X2IHlx5w,60186
@@ -135,7 +135,7 @@ kafka/net/manager.py,sha256=bsgpwH_9nCC0hXvNrPfclF4rsvcqbzWOplSN_ATn9rY,21134
 kafka/net/metrics.py,sha256=-cdWXQtonCO7-LpovLoEcWM-_3R9sAtyQxWf_QHOAWE,7299
 kafka/net/selector.py,sha256=kKKhVA6UdyiTNAnRDlRJNA3rC-C0anUcpYHnQy_yRZQ,29333
 kafka/net/socks5.py,sha256=euTwJl2avRmCJhVKyGmhZWInqgY6MNHPA1Tz2f2J1Z0,10361
-kafka/net/transport.py,sha256=ZQOTR8cqtjAmvsaQZndlyOKH_U0lJSBhkLPPknInxgU,15675
+kafka/net/transport.py,sha256=Loa43s09X-VVrNiHggdfzNIDrok4dHlR8LptY3b0yAw,15974
 kafka/net/wakeup_notifier.py,sha256=gLnfg-HEjVS5J9p-7kZfZpIeLT0AjvK5MOc6EMj3R88,2783
 kafka/net/sasl/__init__.py,sha256=vKE2jkS_BZ2OEVANlXZhW2xIVvH5C1o2kCxugw2PfYY,1006
 kafka/net/sasl/abc.py,sha256=H3tHUPevoaQdS395_ID88Wxhwvc8XjOTcAP8dKXxO9I,547
@@ -152,10 +152,10 @@ kafka/partitioner/sticky.py,sha256=A3pfCrFl-jhmpqTANlRcQ5ixd_e-F3K8-d42ugaE378,4
 kafka/producer/__init__.py,sha256=OPWP89JbP1dCYG4HSWl1iz6KLun0oJDYfqLAr0vAfHQ,82
 kafka/producer/__main__.py,sha256=h5u6-EfA7bJFszn0x-QV14pZt4MOJw0jiPHKuMSkCOY,72
 kafka/producer/future.py,sha256=D5_8Hz-8CqJZYEVMieUk7wBhKS83skFnisGlsyosIbk,6669
-kafka/producer/kafka.py,sha256=j01YfmZn3sNigHBNITd5CLeohu4WF0FVTFE1E6Pj5O8,59338
+kafka/producer/kafka.py,sha256=nWBHlweLZGc_oDC2ipT7ewd0M2bY4s_294GKllkmZT4,59338
 kafka/producer/producer_batch.py,sha256=gFuJNaUM9gHs0KpgQ0KFZCIlBy43OnrRyzo2xb0HSTo,8027
 kafka/producer/record_accumulator.py,sha256=B6C64fNzD7MIFsWV-kJFrahKRVS5dYniI3OFhnUqvRw,28039
-kafka/producer/sender.py,sha256=aV4skH8T1xncOidyS8do0Gv2jcr97u3CHx5wXlrPTik,44608
+kafka/producer/sender.py,sha256=fe9spMNSaIAMCypVFAL047eRx3pMH8ARAVVIrzV-aVM,44712
 kafka/producer/transaction_manager.py,sha256=E90BGF3_zU-k2jfIcKWKbCUWhxCoPT-d2vrJER23Q84,61408
 kafka/protocol/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kafka/protocol/api_data.py,sha256=fyiEuEftBfJl23SL3EkF-Sd0lps9ZmTlDT9PY6YV_PY,4793
@@ -365,9 +365,9 @@ kafka/serializer/default.py,sha256=ZKzTWlG9N4vS3QXovFLygNVrnjAcMZKS0RE4OvQ6DL4,4
 kafka/serializer/json.py,sha256=lErgU66KZGf33hofwlObYIaxLDk9gcolhy_rlcOuf8Y,469
 kafka/serializer/wrapper.py,sha256=RGXFj-PXQyL3rdMXI9ACaohRa6Kg5SHOZTRruU9jluM,485
 kafka/vendor/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-kafka_python-3.0.5.dist-info/licenses/LICENSE,sha256=vxnoJsqm6bKl3ZWdI1Q7Ikw_k9cOvO3vcvZNsY_1fP8,11374
-kafka_python-3.0.5.dist-info/METADATA,sha256=Sw4hH50e8W8I90Sfe-DbW27G7KGX6qA2LggJO_gG5ls,11590
-kafka_python-3.0.5.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
-kafka_python-3.0.5.dist-info/entry_points.txt,sha256=LQvqZj3uM785ainO5HrXmukbjSrK-oPqrESqpvoR-As,51
-kafka_python-3.0.5.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
-kafka_python-3.0.5.dist-info/RECORD,,
+kafka_python-3.0.7.dist-info/licenses/LICENSE,sha256=vxnoJsqm6bKl3ZWdI1Q7Ikw_k9cOvO3vcvZNsY_1fP8,11374
+kafka_python-3.0.7.dist-info/METADATA,sha256=qIDuv8vtC5xCXcx0W6Lor0EXAGSUmJIeRqKClE175qw,11590
+kafka_python-3.0.7.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91
+kafka_python-3.0.7.dist-info/entry_points.txt,sha256=LQvqZj3uM785ainO5HrXmukbjSrK-oPqrESqpvoR-As,51
+kafka_python-3.0.7.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
+kafka_python-3.0.7.dist-info/RECORD,,