kafka-python 2.2.16__tar.gz → 2.2.17__tar.gz

{kafka_python-2.2.16 → kafka_python-2.2.17}/CHANGES.md

@@ -1,3 +1,15 @@
+# 2.2.17 (Nov 20, 2025)
+
+Fixes
+* Add internal poll to consumer.position() (#2696)
+* Initiate Coordinator Reconnect w/ Backoff from Heartbeat Thread (#2695)
+
+Networking
+* SOCKS5: support looking up names remotely (jschwartzenberg / #2666)
+
+Documentation
+* Add `transactional_id` to KafkaProducer Keyword Arguments docstring
+
 # 2.2.16 (Nov 18, 2025)
 
 Fixes
@@ -32,7 +44,6 @@ Fixes
 * Avoid RuntimeError on mutated `_completed_fetches` deque in consumer fetcher (#2646)
 * Throw exception on invalid bucket type (#2642)
 
-
 # 2.2.11 (June 5, 2025)
 
 Fixes

{kafka_python-2.2.16 → kafka_python-2.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.16
+Version: 2.2.17
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.16 → kafka_python-2.2.17}/kafka/conn.py

@@ -326,6 +326,9 @@ class BrokerConnection(object):
         return True
 
     def _next_afi_sockaddr(self):
+        if self.config["socks5_proxy"] and Socks5Wrapper.use_remote_lookup(self.config["socks5_proxy"]):
+            return (socket.AF_UNSPEC, (self.host, self.port))
+
         if not self._gai:
             if not self._dns_lookup():
                 return
@@ -379,6 +382,7 @@ class BrokerConnection(object):
                 self._sock_afi, self._sock_addr = next_lookup
                 try:
                     if self.config["socks5_proxy"] is not None:
+                        log.debug('%s: initializing Socks5 proxy at %s', self, self.config["socks5_proxy"])
                         self._socks5_proxy = Socks5Wrapper(self.config["socks5_proxy"], self.afi)
                         self._sock = self._socks5_proxy.socket(self._sock_afi, socket.SOCK_STREAM)
                     else:
@@ -864,6 +868,8 @@ class BrokerConnection(object):
         if self.disconnected() or self.connecting():
             if len(self._gai) > 0:
                 return 0
+            elif self.config["socks5_proxy"] and Socks5Wrapper.use_remote_lookup(self.config["socks5_proxy"]):
+                return 0
             else:
                 time_waited = time.time() - self.last_attempt
                 return max(self._reconnect_backoff - time_waited, 0) * 1000
@@ -964,6 +970,7 @@ class BrokerConnection(object):
             # the socket fd from selectors cleanly.
             sock = self._sock
             self._sock = None
+            self._socks5_proxy = None
 
         # drop lock before state change callback and processing futures
         self.config['state_change_callback'](self.node_id, sock, self)

{kafka_python-2.2.16 → kafka_python-2.2.17}/kafka/consumer/group.py

@@ -723,7 +723,9 @@ class KafkaConsumer(six.Iterator):
 
         # We do not want to be stuck blocking in poll if we are missing some positions
         # since the offset lookup may be backing off after a failure
-        poll_timeout_ms = min(timer.timeout_ms, self._coordinator.time_to_next_poll() * 1000)
+        poll_timeout_ms = timer.timeout_ms
+        if self.config['group_id'] is not None:
+            poll_timeout_ms = min(poll_timeout_ms, self._coordinator.time_to_next_poll() * 1000)
         if not has_all_fetch_positions:
             log.debug('poll: do not have all fetch positions...')
             poll_timeout_ms = min(poll_timeout_ms, self.config['retry_backoff_ms'])
@@ -753,13 +755,12 @@ class KafkaConsumer(six.Iterator):
 
         timer = Timer(timeout_ms)
         position = self._subscription.assignment[partition].position
-        while position is None:
+        while position is None and not timer.expired:
             # batch update fetch positions for any partitions without a valid position
-            if self._update_fetch_positions(timeout_ms=timer.timeout_ms):
-                position = self._subscription.assignment[partition].position
-            if timer.expired:
-                return None
-        else:
+            self._update_fetch_positions(timeout_ms=timer.timeout_ms)
+            self._client.poll(timeout_ms=timer.timeout_ms)
+            position = self._subscription.assignment[partition].position
+        if position is not None:
             return position.offset
 
     def highwater(self, partition):

{kafka_python-2.2.16 → kafka_python-2.2.17}/kafka/coordinator/base.py

@@ -1120,6 +1120,7 @@ class HeartbeatThread(threading.Thread):
                     self.coordinator._lock.wait(self.coordinator.config['retry_backoff_ms'] / 1000)
 
             elif not self.coordinator.connected():
+                self.coordinator._client.maybe_connect(self.coordinator.coordinator_id)
                 self.coordinator._client._lock.release()
                 self.coordinator._lock.wait(self.coordinator.config['retry_backoff_ms'] / 1000)
 

{kafka_python-2.2.16 → kafka_python-2.2.17}/kafka/producer/kafka.py

@@ -134,10 +134,14 @@ class KafkaProducer(object):
         value_serializer (callable): used to convert user-supplied message
             values to bytes. If not None, called as f(value), should return
             bytes. Default: None.
+        transactional_id (str): Enable transactional producer with a unique
+            identifier. This will be used to identify the same producer
+            instance across process restarts. Default: None.
         enable_idempotence (bool): When set to True, the producer will ensure
             that exactly one copy of each message is written in the stream.
             If False, producer retries due to broker failures, etc., may write
-            duplicates of the retried message in the stream. Default: False.
+            duplicates of the retried message in the stream.
+            Default: True if `transactional_id` is provided, otherwise False.
 
             Note that enabling idempotence requires
             `max_in_flight_requests_per_connection` to be set to 1 and `retries`

{kafka_python-2.2.16 → kafka_python-2.2.17}/kafka/socks5_wrapper.py

@@ -64,6 +64,15 @@ class Socks5Wrapper:
             log.warning("DNS lookup failed for proxy %s:%d, %r", host, port, ex)
             return []
 
+    @classmethod
+    def use_remote_lookup(cls, proxy_url):
+        if proxy_url is None:
+            return False
+        return urlparse(proxy_url).scheme == 'socks5h'
+
+    def _use_remote_lookup(self):
+        return self._proxy_url.scheme == 'socks5h'
+
     def socket(self, family, sock_type):
         """Open and record a socket.
 
@@ -187,7 +196,10 @@ class Socks5Wrapper:
                 return errno.ECONNREFUSED
 
         if self._state == ProxyConnectionStates.REQUEST_SUBMIT:
-            if self._target_afi == socket.AF_INET:
+            if self._use_remote_lookup():
+                addr_type = 3
+                addr_len = len(addr[0])
+            elif self._target_afi == socket.AF_INET:
                 addr_type = 1
                 addr_len = 4
             elif self._target_afi == socket.AF_INET6:
@@ -200,14 +212,28 @@ class Socks5Wrapper:
                 return errno.ECONNREFUSED
 
             self._buffer_out = struct.pack(
-                "!bbbb{}sh".format(addr_len),
+                "!bbbb",
                 5,  # version
                 1,  # command: connect
                 0,  # reserved
-                addr_type,  # 1 for ipv4, 4 for ipv6 address
-                socket.inet_pton(self._target_afi, addr[0]),  # either 4 or 16 bytes of actual address
-                addr[1],  # port
+                addr_type,  # 1 for ipv4, 4 for ipv6 address, 3 for domain name
             )
+            # Addr format depends on type
+            if addr_type == 3:
+                # len + domain name (no null terminator)
+                self._buffer_out += struct.pack(
+                    "!b{}s".format(addr_len),
+                    addr_len,
+                    addr[0].encode('ascii'),
+                )
+            else:
+                # either 4 (type 1) or 16 (type 4) bytes of actual address
+                self._buffer_out += struct.pack(
+                    "!{}s".format(addr_len),
+                    socket.inet_pton(self._target_afi, addr[0]),
+                )
+            self._buffer_out += struct.pack("!H", addr[1])  # port
+
             self._state = ProxyConnectionStates.REQUESTING
 
         if self._state == ProxyConnectionStates.REQUESTING:

kafka_python-2.2.17/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '2.2.17'

{kafka_python-2.2.16 → kafka_python-2.2.17}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.16
+Version: 2.2.17
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.16 → kafka_python-2.2.17}/test/integration/test_consumer_integration.py

@@ -302,3 +302,25 @@ def test_kafka_consumer_offsets_for_times_errors(kafka_consumer_factory, topic):
 
     with pytest.raises(KafkaTimeoutError):
         consumer.offsets_for_times({bad_tp: 0})
+
+
+@pytest.mark.skipif(not env_kafka_version(), reason="No KAFKA_VERSION set")
+def test_kafka_consumer_position_after_seek_to_end(kafka_consumer_factory, topic, send_messages):
+    send_messages(range(0, 10), partition=0)
+
+    # Start a consumer with manual partition assignment.
+    consumer = kafka_consumer_factory(
+        topics=(),
+        group_id=None,
+        enable_auto_commit=False,
+    )
+    tp = TopicPartition(topic, 0)
+    consumer.assign([tp])
+
+    # Seek to the end of the partition, and call position() to synchronize the
+    # partition's offset without calling poll().
+    consumer.seek_to_end(tp)
+    position = consumer.position(tp, timeout_ms=1000)
+
+    # Verify we got the expected position
+    assert position == 10, f"Expected position 10, got {position}"

kafka_python-2.2.16/kafka/version.py

@@ -1 +0,0 @@
-__version__ = '2.2.16'