kafka-python 2.0.4__tar.gz → 2.0.5__tar.gz

{kafka_python-2.0.4 → kafka_python-2.0.5}/CHANGES.md

@@ -1,3 +1,29 @@
+# 2.0.5 (Feb 25, 2025)
+
+Networking
+* Remove unused client bootstrap backoff code
+* 200ms timeout for client.poll in ensure_active_group and admin client
+
+Fixes
+* Admin client: check_version only if needed, use node_id kwarg for controller
+* Check for -1 controller_id in admin client
+* Only acquire coordinator lock in heartbeat thread close if not self thread
+
+Testing
+* Also sleep when waiting for consumers in test_describe_consumer_group_exists
+* Refactor sasl_integration test_client - wait for node ready; use send future
+* Add timeout to test_kafka_consumer
+* Add error str to assert_message_count checks
+* Retry on error in test fixture create_topic_via_metadata
+* Fixup variable interpolation in test fixture error
+
+Documentation
+* Update compatibility docs
+* Include client_id in BrokerConnection __str__ output
+
+Project Maintenance
+* Add make targets `servers/*/api_versions` and `servers/*/messages`
+
 # 2.0.4 (Feb 21, 2025)
 
 Networking

{kafka_python-2.0.4 → kafka_python-2.0.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: kafka-python
-Version: 2.0.4
+Version: 2.0.5
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python
@@ -41,7 +41,7 @@ Requires-Dist: pytest-mock; extra == "testing"
 Kafka Python client
 ------------------------
 
-.. image:: https://img.shields.io/badge/kafka-2.6%2C%202.5%2C%202.4%2C%202.3%2C%202.2%2C%202.1%2C%202.0%2C%201.1%2C%201.0%2C%200.11%2C%200.10%2C%200.9%2C%200.8-brightgreen.svg
+.. image:: https://img.shields.io/badge/kafka-3.9--0.8-brightgreen.svg
     :target: https://kafka-python.readthedocs.io/en/master/compatibility.html
 .. image:: https://img.shields.io/pypi/pyversions/kafka-python.svg
     :target: https://pypi.python.org/pypi/kafka-python

{kafka_python-2.0.4 → kafka_python-2.0.5}/README.rst

@@ -1,7 +1,7 @@
 Kafka Python client
 ------------------------
 
-.. image:: https://img.shields.io/badge/kafka-2.6%2C%202.5%2C%202.4%2C%202.3%2C%202.2%2C%202.1%2C%202.0%2C%201.1%2C%201.0%2C%200.11%2C%200.10%2C%200.9%2C%200.8-brightgreen.svg
+.. image:: https://img.shields.io/badge/kafka-3.9--0.8-brightgreen.svg
     :target: https://kafka-python.readthedocs.io/en/master/compatibility.html
 .. image:: https://img.shields.io/pypi/pyversions/kafka-python.svg
     :target: https://pypi.python.org/pypi/kafka-python

{kafka_python-2.0.4 → kafka_python-2.0.5}/kafka/admin/client.py

@@ -1,9 +1,10 @@
-from __future__ import absolute_import
+from __future__ import absolute_import, division
 
 from collections import defaultdict
 import copy
 import logging
 import socket
+import time
 
 from . import ConfigResourceType
 from kafka.vendor import six
@@ -212,11 +213,13 @@ class KafkaAdminClient(object):
             metric_group_prefix='admin',
             **self.config
         )
-        self._client.check_version(timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
 
         # Get auto-discovered version from client if necessary
         if self.config['api_version'] is None:
             self.config['api_version'] = self._client.config['api_version']
+        else:
+            # need to run check_version for get_api_versions()
+            self._client.check_version(timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
 
         self._closed = False
         self._refresh_controller_id()
@@ -273,24 +276,33 @@ class KafkaAdminClient(object):
         """
         return timeout_ms or self.config['request_timeout_ms']
 
-    def _refresh_controller_id(self):
+    def _refresh_controller_id(self, timeout_ms=30000):
         """Determine the Kafka cluster controller."""
         version = self._matching_api_version(MetadataRequest)
         if 1 <= version <= 6:
-            request = MetadataRequest[version]()
-            future = self._send_request_to_node(self._client.least_loaded_node(), request)
-
-            self._wait_for_futures([future])
-
-            response = future.value
-            controller_id = response.controller_id
-            # verify the controller is new enough to support our requests
-            controller_version = self._client.check_version(controller_id, timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
-            if controller_version < (0, 10, 0):
-                raise IncompatibleBrokerVersion(
-                    "The controller appears to be running Kafka {}. KafkaAdminClient requires brokers >= 0.10.0.0."
-                    .format(controller_version))
-            self._controller_id = controller_id
+            timeout_at = time.time() + timeout_ms / 1000
+            while time.time() < timeout_at:
+                request = MetadataRequest[version]()
+                future = self._send_request_to_node(self._client.least_loaded_node(), request)
+
+                self._wait_for_futures([future])
+
+                response = future.value
+                controller_id = response.controller_id
+                if controller_id == -1:
+                    log.warning("Controller ID not available, got -1")
+                    time.sleep(1)
+                    continue
+                # verify the controller is new enough to support our requests
+                controller_version = self._client.check_version(node_id=controller_id, timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
+                if controller_version < (0, 10, 0):
+                    raise IncompatibleBrokerVersion(
+                        "The controller appears to be running Kafka {}. KafkaAdminClient requires brokers >= 0.10.0.0."
+                        .format(controller_version))
+                self._controller_id = controller_id
+                return
+            else:
+                raise Errors.NodeNotAvailableError('controller')
         else:
             raise UnrecognizedBrokerVersion(
                 "Kafka Admin interface cannot determine the controller using MetadataRequest_v{}."
@@ -390,7 +402,7 @@ class KafkaAdminClient(object):
         while not self._client.ready(node_id):
             # poll until the connection to broker is ready, otherwise send()
             # will fail with NodeNotReadyError
-            self._client.poll()
+            self._client.poll(timeout_ms=200)
         return self._client.send(node_id, request, wakeup)
 
     def _send_request_to_controller(self, request):

{kafka_python-2.0.4 → kafka_python-2.0.5}/kafka/client_async.py

@@ -216,6 +216,8 @@ class KafkaClient(object):
         self._connecting = set()
         self._sending = set()
         self._refresh_on_disconnects = True
+
+        # Not currently used, but data is collected internally
         self._last_bootstrap = 0
         self._bootstrap_fails = 0
 
@@ -233,8 +235,6 @@ class KafkaClient(object):
                                                self.config['metric_group_prefix'],
                                                weakref.proxy(self._conns))
 
-        self._num_bootstrap_hosts = len(collect_hosts(self.config['bootstrap_servers']))
-
         # Check Broker Version if not set explicitly
         if self.config['api_version'] is None:
             check_timeout = self.config['api_version_auto_timeout_ms'] / 1000
@@ -259,20 +259,6 @@ class KafkaClient(object):
         self._wake_r = None
         self._wake_w = None
 
-    def _can_bootstrap(self):
-        effective_failures = self._bootstrap_fails // self._num_bootstrap_hosts
-        backoff_factor = 2 ** effective_failures
-        backoff_ms = min(self.config['reconnect_backoff_ms'] * backoff_factor,
-                         self.config['reconnect_backoff_max_ms'])
-
-        backoff_ms *= random.uniform(0.8, 1.2)
-
-        next_at = self._last_bootstrap + backoff_ms / 1000.0
-        now = time.time()
-        if next_at > now:
-            return False
-        return True
-
     def _can_connect(self, node_id):
         if node_id not in self._conns:
             if self.cluster.broker_metadata(node_id):

{kafka_python-2.0.4 → kafka_python-2.0.5}/kafka/conn.py

@@ -1315,8 +1315,8 @@ class BrokerConnection(object):
         return version
 
     def __str__(self):
-        return "<BrokerConnection node_id=%s host=%s:%d %s [%s %s]>" % (
-            self.node_id, self.host, self.port, self.state,
+        return "<BrokerConnection client_id=%s, node_id=%s host=%s:%d %s [%s %s]>" % (
+            self.config['client_id'], self.node_id, self.host, self.port, self.state,
             AFI_NAMES[self._sock_afi], self._sock_addr)
 
 

{kafka_python-2.0.4 → kafka_python-2.0.5}/kafka/coordinator/base.py

@@ -371,7 +371,7 @@ class BaseCoordinator(object):
                 while not self.coordinator_unknown():
                     if not self._client.in_flight_request_count(self.coordinator_id):
                         break
-                    self._client.poll()
+                    self._client.poll(timeout_ms=200)
                 else:
                     continue
 
@@ -923,8 +923,6 @@ class HeartbeatThread(threading.Thread):
         if self.closed:
             return
         self.closed = True
-        with self.coordinator._lock:
-            self.coordinator._lock.notify()
 
         # Generally this should not happen - close() is triggered
         # by the coordinator. But in some cases GC may close the coordinator
@@ -932,6 +930,9 @@ class HeartbeatThread(threading.Thread):
         if threading.current_thread() == self:
             return
 
+        with self.coordinator._lock:
+            self.coordinator._lock.notify()
+
         if self.is_alive():
             self.join(self.coordinator.config['heartbeat_interval_ms'] / 1000)
         if self.is_alive():

kafka_python-2.0.5/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '2.0.5'

{kafka_python-2.0.4 → kafka_python-2.0.5}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: kafka-python
-Version: 2.0.4
+Version: 2.0.5
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python
@@ -41,7 +41,7 @@ Requires-Dist: pytest-mock; extra == "testing"
 Kafka Python client
 ------------------------
 
-.. image:: https://img.shields.io/badge/kafka-2.6%2C%202.5%2C%202.4%2C%202.3%2C%202.2%2C%202.1%2C%202.0%2C%201.1%2C%201.0%2C%200.11%2C%200.10%2C%200.9%2C%200.8-brightgreen.svg
+.. image:: https://img.shields.io/badge/kafka-3.9--0.8-brightgreen.svg
     :target: https://kafka-python.readthedocs.io/en/master/compatibility.html
 .. image:: https://img.shields.io/pypi/pyversions/kafka-python.svg
     :target: https://pypi.python.org/pypi/kafka-python

{kafka_python-2.0.4 → kafka_python-2.0.5}/test/test_admin_integration.py

@@ -168,7 +168,7 @@ def test_describe_consumer_group_exists(kafka_admin_client, kafka_consumer_facto
         stop[i] = Event()
         consumers[i] = kafka_consumer_factory(group_id=group_id)
         while not stop[i].is_set():
-            consumers[i].poll(20)
+            consumers[i].poll(timeout_ms=200)
         consumers[i].close()
         consumers[i] = None
         stop[i] = None
@@ -183,6 +183,7 @@ def test_describe_consumer_group_exists(kafka_admin_client, kafka_consumer_facto
     try:
         timeout = time() + 35
         while True:
+            info('Checking consumers...')
             for c in range(num_consumers):
 
                 # Verify all consumers have been created
@@ -212,9 +213,9 @@ def test_describe_consumer_group_exists(kafka_admin_client, kafka_consumer_facto
 
                 if not rejoining and is_same_generation:
                     break
-                else:
-                    sleep(1)
             assert time() < timeout, "timeout waiting for assignments"
+            info('sleeping...')
+            sleep(1)
 
         info('Group stabilized; verifying assignment')
         output = kafka_admin_client.describe_consumer_groups(group_id_list)
@@ -236,6 +237,8 @@ def test_describe_consumer_group_exists(kafka_admin_client, kafka_consumer_facto
         for c in range(num_consumers):
             info('Stopping consumer %s', c)
             stop[c].set()
+        for c in range(num_consumers):
+            info('Waiting for consumer thread %s', c)
             threads[c].join()
             threads[c] = None
 

{kafka_python-2.0.4 → kafka_python-2.0.5}/test/test_consumer_integration.py

@@ -27,7 +27,7 @@ def test_kafka_version_infer(kafka_consumer_factory):
 @pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set")
 def test_kafka_consumer(kafka_consumer_factory, send_messages):
     """Test KafkaConsumer"""
-    consumer = kafka_consumer_factory(auto_offset_reset='earliest')
+    consumer = kafka_consumer_factory(auto_offset_reset='earliest', consumer_timeout_ms=2000)
     send_messages(range(0, 100), partition=0)
     send_messages(range(0, 100), partition=1)
     cnt = 0

{kafka_python-2.0.4 → kafka_python-2.0.5}/test/test_sasl_integration.py

@@ -1,5 +1,6 @@
 import logging
 import uuid
+import time
 
 import pytest
 
@@ -69,12 +70,17 @@ def test_client(request, sasl_kafka):
 
     client, = sasl_kafka.get_clients(1)
     request = MetadataRequest_v1(None)
-    client.send(0, request)
-    for _ in range(10):
-        result = client.poll(timeout_ms=10000)
-        if len(result) > 0:
-            break
-    else:
+    timeout_at = time.time() + 1
+    while not client.is_ready(0):
+        client.maybe_connect(0)
+        client.poll(timeout_ms=100)
+        if time.time() > timeout_at:
+            raise RuntimeError("Couldn't connect to node 0")
+    future = client.send(0, request)
+    client.poll(future=future, timeout_ms=10000)
+    if not future.is_done:
         raise RuntimeError("Couldn't fetch topic response from Broker.")
-    result = result[0]
+    elif future.failed():
+        raise future.exception
+    result = future.value
     assert topic_name in [t[1] for t in result.topics]

{kafka_python-2.0.4 → kafka_python-2.0.5}/test/testutil.py

@@ -28,12 +28,12 @@ def env_kafka_version():
 def assert_message_count(messages, num_messages):
     """Check that we received the expected number of messages with no duplicates."""
     # Make sure we got them all
-    assert len(messages) == num_messages
+    assert len(messages) == num_messages, 'Expected %d messages, got %d' % (num_messages, len(messages))
     # Make sure there are no duplicates
     # Note: Currently duplicates are identified only using key/value. Other attributes like topic, partition, headers,
     # timestamp, etc are ignored... this could be changed if necessary, but will be more tolerant of dupes.
     unique_messages = {(m.key, m.value) for m in messages}
-    assert len(unique_messages) == num_messages
+    assert len(unique_messages) == num_messages, 'Expected %d unique messages, got %d' % (num_messages, len(unique_messages))
 
 
 class Timer(object):

kafka_python-2.0.4/kafka/version.py

@@ -1 +0,0 @@
-__version__ = '2.0.4'