kafka-python 2.2.15__tar.gz → 2.2.17__tar.gz

{kafka_python-2.2.15 → kafka_python-2.2.17}/CHANGES.md

@@ -1,3 +1,21 @@
+# 2.2.17 (Nov 20, 2025)
+
+Fixes
+* Add internal poll to consumer.position() (#2696)
+* Initiate Coordinator Reconnect w/ Backoff from Heartbeat Thread (#2695)
+
+Networking
+* SOCKS5: support looking up names remotely (jschwartzenberg / #2666)
+
+Documentation
+* Add `transactional_id` to KafkaProducer Keyword Arguments docstring
+
+# 2.2.16 (Nov 18, 2025)
+
+Fixes
+* Fix thread not waking up when there is still data to be sent (gqmelo / #2670)
+* Ensure timeout is checked after each fetch position update in `Consumer.position()` (k61n / #2668)
+
 # 2.2.15 (July 1, 2025)
 
 Fixes
@@ -26,7 +44,6 @@ Fixes
 * Avoid RuntimeError on mutated `_completed_fetches` deque in consumer fetcher (#2646)
 * Throw exception on invalid bucket type (#2642)
 
-
 # 2.2.11 (June 5, 2025)
 
 Fixes

{kafka_python-2.2.15 → kafka_python-2.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.15
+Version: 2.2.17
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.15 → kafka_python-2.2.17}/kafka/conn.py

@@ -326,6 +326,9 @@ class BrokerConnection(object):
         return True
 
     def _next_afi_sockaddr(self):
+        if self.config["socks5_proxy"] and Socks5Wrapper.use_remote_lookup(self.config["socks5_proxy"]):
+            return (socket.AF_UNSPEC, (self.host, self.port))
+
         if not self._gai:
             if not self._dns_lookup():
                 return
@@ -379,6 +382,7 @@ class BrokerConnection(object):
                 self._sock_afi, self._sock_addr = next_lookup
                 try:
                     if self.config["socks5_proxy"] is not None:
+                        log.debug('%s: initializing Socks5 proxy at %s', self, self.config["socks5_proxy"])
                         self._socks5_proxy = Socks5Wrapper(self.config["socks5_proxy"], self.afi)
                         self._sock = self._socks5_proxy.socket(self._sock_afi, socket.SOCK_STREAM)
                     else:
@@ -864,6 +868,8 @@ class BrokerConnection(object):
         if self.disconnected() or self.connecting():
             if len(self._gai) > 0:
                 return 0
+            elif self.config["socks5_proxy"] and Socks5Wrapper.use_remote_lookup(self.config["socks5_proxy"]):
+                return 0
             else:
                 time_waited = time.time() - self.last_attempt
                 return max(self._reconnect_backoff - time_waited, 0) * 1000
@@ -964,6 +970,7 @@ class BrokerConnection(object):
             # the socket fd from selectors cleanly.
             sock = self._sock
             self._sock = None
+            self._socks5_proxy = None
 
         # drop lock before state change callback and processing futures
         self.config['state_change_callback'](self.node_id, sock, self)
@@ -1075,6 +1082,13 @@ class BrokerConnection(object):
                     total_bytes = self._send_bytes(self._send_buffer)
                     self._send_buffer = self._send_buffer[total_bytes:]
 
+                # If all data was sent, we need to get the new data from the protocol now, otherwise
+                # this function would return True, indicating that there are no more pending
+                # requests. This could cause the calling thread to wait indefinitely as it won't
+                # know that there is still buffered data to send.
+                if not self._send_buffer:
+                    self._send_buffer = self._protocol.send_bytes()
+
             if self._sensors:
                 self._sensors.bytes_sent.record(total_bytes)
             # Return True iff send buffer is empty

{kafka_python-2.2.15 → kafka_python-2.2.17}/kafka/consumer/group.py

@@ -723,7 +723,9 @@ class KafkaConsumer(six.Iterator):
 
         # We do not want to be stuck blocking in poll if we are missing some positions
         # since the offset lookup may be backing off after a failure
-        poll_timeout_ms = min(timer.timeout_ms, self._coordinator.time_to_next_poll() * 1000)
+        poll_timeout_ms = timer.timeout_ms
+        if self.config['group_id'] is not None:
+            poll_timeout_ms = min(poll_timeout_ms, self._coordinator.time_to_next_poll() * 1000)
         if not has_all_fetch_positions:
             log.debug('poll: do not have all fetch positions...')
             poll_timeout_ms = min(poll_timeout_ms, self.config['retry_backoff_ms'])
@@ -753,13 +755,12 @@ class KafkaConsumer(six.Iterator):
 
         timer = Timer(timeout_ms)
         position = self._subscription.assignment[partition].position
-        while position is None:
+        while position is None and not timer.expired:
             # batch update fetch positions for any partitions without a valid position
-            if self._update_fetch_positions(timeout_ms=timer.timeout_ms):
-                position = self._subscription.assignment[partition].position
-            elif timer.expired:
-                return None
-        else:
+            self._update_fetch_positions(timeout_ms=timer.timeout_ms)
+            self._client.poll(timeout_ms=timer.timeout_ms)
+            position = self._subscription.assignment[partition].position
+        if position is not None:
             return position.offset
 
     def highwater(self, partition):

{kafka_python-2.2.15 → kafka_python-2.2.17}/kafka/coordinator/base.py

@@ -1120,6 +1120,7 @@ class HeartbeatThread(threading.Thread):
                     self.coordinator._lock.wait(self.coordinator.config['retry_backoff_ms'] / 1000)
 
             elif not self.coordinator.connected():
+                self.coordinator._client.maybe_connect(self.coordinator.coordinator_id)
                 self.coordinator._client._lock.release()
                 self.coordinator._lock.wait(self.coordinator.config['retry_backoff_ms'] / 1000)
 

{kafka_python-2.2.15 → kafka_python-2.2.17}/kafka/producer/kafka.py

@@ -134,10 +134,14 @@ class KafkaProducer(object):
         value_serializer (callable): used to convert user-supplied message
             values to bytes. If not None, called as f(value), should return
             bytes. Default: None.
+        transactional_id (str): Enable transactional producer with a unique
+            identifier. This will be used to identify the same producer
+            instance across process restarts. Default: None.
         enable_idempotence (bool): When set to True, the producer will ensure
             that exactly one copy of each message is written in the stream.
             If False, producer retries due to broker failures, etc., may write
-            duplicates of the retried message in the stream. Default: False.
+            duplicates of the retried message in the stream.
+            Default: True if `transactional_id` is provided, otherwise False.
 
             Note that enabling idempotence requires
             `max_in_flight_requests_per_connection` to be set to 1 and `retries`

{kafka_python-2.2.15 → kafka_python-2.2.17}/kafka/socks5_wrapper.py

@@ -64,6 +64,15 @@ class Socks5Wrapper:
             log.warning("DNS lookup failed for proxy %s:%d, %r", host, port, ex)
             return []
 
+    @classmethod
+    def use_remote_lookup(cls, proxy_url):
+        if proxy_url is None:
+            return False
+        return urlparse(proxy_url).scheme == 'socks5h'
+
+    def _use_remote_lookup(self):
+        return self._proxy_url.scheme == 'socks5h'
+
     def socket(self, family, sock_type):
         """Open and record a socket.
 
@@ -187,7 +196,10 @@ class Socks5Wrapper:
                 return errno.ECONNREFUSED
 
         if self._state == ProxyConnectionStates.REQUEST_SUBMIT:
-            if self._target_afi == socket.AF_INET:
+            if self._use_remote_lookup():
+                addr_type = 3
+                addr_len = len(addr[0])
+            elif self._target_afi == socket.AF_INET:
                 addr_type = 1
                 addr_len = 4
             elif self._target_afi == socket.AF_INET6:
@@ -200,14 +212,28 @@ class Socks5Wrapper:
                 return errno.ECONNREFUSED
 
             self._buffer_out = struct.pack(
-                "!bbbb{}sh".format(addr_len),
+                "!bbbb",
                 5,  # version
                 1,  # command: connect
                 0,  # reserved
-                addr_type,  # 1 for ipv4, 4 for ipv6 address
-                socket.inet_pton(self._target_afi, addr[0]),  # either 4 or 16 bytes of actual address
-                addr[1],  # port
+                addr_type,  # 1 for ipv4, 4 for ipv6 address, 3 for domain name
             )
+            # Addr format depends on type
+            if addr_type == 3:
+                # len + domain name (no null terminator)
+                self._buffer_out += struct.pack(
+                    "!b{}s".format(addr_len),
+                    addr_len,
+                    addr[0].encode('ascii'),
+                )
+            else:
+                # either 4 (type 1) or 16 (type 4) bytes of actual address
+                self._buffer_out += struct.pack(
+                    "!{}s".format(addr_len),
+                    socket.inet_pton(self._target_afi, addr[0]),
+                )
+            self._buffer_out += struct.pack("!H", addr[1])  # port
+
             self._state = ProxyConnectionStates.REQUESTING
 
         if self._state == ProxyConnectionStates.REQUESTING:

kafka_python-2.2.17/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '2.2.17'

{kafka_python-2.2.15 → kafka_python-2.2.17}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.15
+Version: 2.2.17
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.15 → kafka_python-2.2.17}/test/integration/test_consumer_integration.py

@@ -302,3 +302,25 @@ def test_kafka_consumer_offsets_for_times_errors(kafka_consumer_factory, topic):
 
     with pytest.raises(KafkaTimeoutError):
         consumer.offsets_for_times({bad_tp: 0})
+
+
+@pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set")
+def test_kafka_consumer_position_after_seek_to_end(kafka_consumer_factory, topic, send_messages):
+    send_messages(range(0, 10), partition=0)
+
+    # Start a consumer with manual partition assignment.
+    consumer = kafka_consumer_factory(
+        topics=(),
+        group_id=None,
+        enable_auto_commit=False,
+    )
+    tp = TopicPartition(topic, 0)
+    consumer.assign([tp])
+
+    # Seek to the end of the partition, and call position() to synchronize the
+    # partition's offset without calling poll().
+    consumer.seek_to_end(tp)
+    position = consumer.position(tp, timeout_ms=1000)
+
+    # Verify we got the expected position
+    assert position == 10, f"Expected position 10, got {position}"

{kafka_python-2.2.15 → kafka_python-2.2.17}/test/test_conn.py

@@ -12,6 +12,9 @@ import pytest
 
 from kafka.conn import BrokerConnection, ConnectionStates
 from kafka.future import Future
+from kafka.conn import BrokerConnection, ConnectionStates, SSLWantWriteError
+from kafka.metrics.metrics import Metrics
+from kafka.metrics.stats.sensor import Sensor
 from kafka.protocol.api import RequestHeader
 from kafka.protocol.group import HeartbeatResponse
 from kafka.protocol.metadata import MetadataRequest
@@ -43,10 +46,20 @@ def _socket(mocker):
     mocker.patch('socket.socket', return_value=socket)
     return socket
 
+@pytest.fixture
+def metrics(mocker):
+    metrics = mocker.MagicMock(Metrics)
+    metrics.mocked_sensors = {}
+    def sensor(name, **kwargs):
+        if name not in metrics.mocked_sensors:
+            metrics.mocked_sensors[name] = mocker.MagicMock(Sensor)
+        return metrics.mocked_sensors[name]
+    metrics.sensor.side_effect = sensor
+    return metrics
 
 @pytest.fixture
-def conn(_socket, dns_lookup, mocker):
-    conn = BrokerConnection('localhost', 9092, socket.AF_INET)
+def conn(_socket, dns_lookup, metrics, mocker):
+    conn = BrokerConnection('localhost', 9092, socket.AF_INET, metrics=metrics)
     mocker.patch.object(conn, '_try_api_versions_check', return_value=True)
     return conn
 
@@ -228,6 +241,46 @@ def test_send_response(_socket, conn):
     assert len(conn.in_flight_requests) == 1
 
 
+def test_send_async_request_while_other_request_is_already_in_buffer(_socket, conn, metrics):
+    conn.connect()
+    assert conn.state is ConnectionStates.CONNECTED
+    assert 'node-0.bytes-sent' in metrics.mocked_sensors
+    bytes_sent_sensor = metrics.mocked_sensors['node-0.bytes-sent']
+
+    req1 = MetadataRequest[0](topics='foo')
+    header1 = RequestHeader(req1, client_id=conn.config['client_id'])
+    payload_bytes1 = len(header1.encode()) + len(req1.encode())
+    req2 = MetadataRequest[0]([])
+    header2 = RequestHeader(req2, client_id=conn.config['client_id'])
+    payload_bytes2 = len(header2.encode()) + len(req2.encode())
+
+    # The first call to the socket will raise a transient SSL exception. This will make the first
+    # request to be kept in the internal buffer to be sent in the next call of
+    # send_pending_requests_v2.
+    _socket.send.side_effect = [SSLWantWriteError, 4 + payload_bytes1, 4 + payload_bytes2]
+
+    conn.send(req1, blocking=False)
+    # This won't send any bytes because of the SSL exception and the request bytes will be kept in
+    # the buffer.
+    assert conn.send_pending_requests_v2() is False
+    assert bytes_sent_sensor.record.call_args_list[0].args == (0,)
+
+    conn.send(req2, blocking=False)
+    # This will send the remaining bytes in the buffer from the first request, but should notice
+    # that the second request was queued, therefore it should return False.
+    bytes_sent_sensor.record.reset_mock()
+    assert conn.send_pending_requests_v2() is False
+    bytes_sent_sensor.record.assert_called_once_with(4 + payload_bytes1)
+
+    bytes_sent_sensor.record.reset_mock()
+    assert conn.send_pending_requests_v2() is True
+    bytes_sent_sensor.record.assert_called_once_with(4 + payload_bytes2)
+
+    bytes_sent_sensor.record.reset_mock()
+    assert conn.send_pending_requests_v2() is True
+    bytes_sent_sensor.record.assert_called_once_with(0)
+
+
 def test_send_error(_socket, conn):
     conn.connect()
     assert conn.state is ConnectionStates.CONNECTED

kafka_python-2.2.15/kafka/version.py

@@ -1 +0,0 @@
-__version__ = '2.2.15'