kafka-python 2.2.1__py2.py3-none-any.whl → 2.2.3__py2.py3-none-any.whl

kafka/admin/client.py

@@ -274,7 +274,7 @@ class KafkaAdminClient(object):
                 self._controller_id = controller_id
                 return
             else:
-                raise Errors.NodeNotAvailableError('controller')
+                raise Errors.NodeNotReadyError('controller')
         else:
             raise UnrecognizedBrokerVersion(
                 "Kafka Admin interface cannot determine the controller using MetadataRequest_v{}."

kafka/cluster.py

@@ -3,13 +3,15 @@ from __future__ import absolute_import
 import collections
 import copy
 import logging
+import random
+import re
 import threading
 import time
 
 from kafka.vendor import six
 
 from kafka import errors as Errors
-from kafka.conn import collect_hosts
+from kafka.conn import get_ip_port_afi
 from kafka.future import Future
 from kafka.structs import BrokerMetadata, PartitionMetadata, TopicPartition
 
@@ -422,3 +424,24 @@ class ClusterMetadata(object):
     def __str__(self):
         return 'ClusterMetadata(brokers: %d, topics: %d, coordinators: %d)' % \
                (len(self._brokers), len(self._partitions), len(self._coordinators))
+
+
+def collect_hosts(hosts, randomize=True):
+    """
+    Collects a comma-separated set of hosts (host:port) and optionally
+    randomize the returned list.
+    """
+
+    if isinstance(hosts, six.string_types):
+        hosts = hosts.strip().split(',')
+
+    result = []
+    for host_port in hosts:
+        # ignore leading SECURITY_PROTOCOL:// to mimic java client
+        host_port = re.sub('^.*://', '', host_port)
+        host, port, afi = get_ip_port_afi(host_port)
+        result.append((host, port, afi))
+
+    if randomize:
+        random.shuffle(result)
+    return result

kafka/conn.py

@@ -4,7 +4,7 @@ import copy
 import errno
 import io
 import logging
-from random import shuffle, uniform
+from random import uniform
 
 # selectors in stdlib as of py3.4
 try:
@@ -1496,32 +1496,6 @@ def get_ip_port_afi(host_and_port_str):
             return host, port, af
 
 
-def collect_hosts(hosts, randomize=True):
-    """
-    Collects a comma-separated set of hosts (host:port) and optionally
-    randomize the returned list.
-    """
-
-    if isinstance(hosts, six.string_types):
-        hosts = hosts.strip().split(',')
-
-    result = []
-    afi = socket.AF_INET
-    for host_port in hosts:
-
-        host, port, afi = get_ip_port_afi(host_port)
-
-        if port < 0:
-            port = DEFAULT_KAFKA_PORT
-
-        result.append((host, port, afi))
-
-    if randomize:
-        shuffle(result)
-
-    return result
-
-
 def is_inet_4_or_6(gai):
     """Given a getaddrinfo struct, return True iff ipv4 or ipv6"""
     return gai[0] in (socket.AF_INET, socket.AF_INET6)

kafka/consumer/fetcher.py

@@ -153,6 +153,7 @@ class Fetcher(six.Iterator):
             future = self._client.send(node_id, request, wakeup=False)
             future.add_callback(self._handle_fetch_response, node_id, fetch_offsets, time.time())
             future.add_errback(self._handle_fetch_error, node_id)
+            future.add_both(self._clear_pending_fetch_request, node_id)
             futures.append(future)
         self._fetch_futures.extend(futures)
         self._clean_done_fetch_futures()
@@ -643,36 +644,42 @@ class Fetcher(six.Iterator):
                 log.debug("Skipping fetch for partition %s because node %s is throttled",
                         partition, node_id)
 
+            elif not self._client.ready(node_id):
+                # Until we support send request queues, any attempt to send to a not-ready node will be
+                # immediately failed with NodeNotReadyError.
+                log.debug("Skipping fetch for partition %s because connection to leader node is not ready yet")
+
             elif node_id in self._nodes_with_pending_fetch_requests:
                 log.debug("Skipping fetch for partition %s because there is a pending fetch request to node %s",
                         partition, node_id)
-                continue
 
-            if version < 5:
-                partition_info = (
-                    partition.partition,
-                    position.offset,
-                    self.config['max_partition_fetch_bytes']
-                )
-            elif version <= 8:
-                partition_info = (
-                    partition.partition,
-                    position.offset,
-                    -1, # log_start_offset is used internally by brokers / replicas only
-                    self.config['max_partition_fetch_bytes'],
-                )
             else:
-                partition_info = (
-                    partition.partition,
-                    position.leader_epoch,
-                    position.offset,
-                    -1, # log_start_offset is used internally by brokers / replicas only
-                    self.config['max_partition_fetch_bytes'],
-                )
-
-            fetchable[node_id][partition] = partition_info
-            log.debug("Adding fetch request for partition %s at offset %d",
-                      partition, position.offset)
+                # Leader is connected and does not have a pending fetch request
+                if version < 5:
+                    partition_info = (
+                        partition.partition,
+                        position.offset,
+                        self.config['max_partition_fetch_bytes']
+                    )
+                elif version <= 8:
+                    partition_info = (
+                        partition.partition,
+                        position.offset,
+                        -1, # log_start_offset is used internally by brokers / replicas only
+                        self.config['max_partition_fetch_bytes'],
+                    )
+                else:
+                    partition_info = (
+                        partition.partition,
+                        position.leader_epoch,
+                        position.offset,
+                        -1, # log_start_offset is used internally by brokers / replicas only
+                        self.config['max_partition_fetch_bytes'],
+                    )
+
+                fetchable[node_id][partition] = partition_info
+                log.debug("Adding fetch request for partition %s at offset %d",
+                          partition, position.offset)
 
         requests = {}
         for node_id, next_partitions in six.iteritems(fetchable):
@@ -761,14 +768,18 @@ class Fetcher(six.Iterator):
 
         if self._sensors:
             self._sensors.fetch_latency.record((time.time() - send_time) * 1000)
-        self._nodes_with_pending_fetch_requests.remove(node_id)
 
     def _handle_fetch_error(self, node_id, exception):
         level = logging.INFO if isinstance(exception, Errors.Cancelled) else logging.ERROR
         log.log(level, 'Fetch to node %s failed: %s', node_id, exception)
         if node_id in self._session_handlers:
             self._session_handlers[node_id].handle_error(exception)
-        self._nodes_with_pending_fetch_requests.remove(node_id)
+
+    def _clear_pending_fetch_request(self, node_id, _):
+        try:
+            self._nodes_with_pending_fetch_requests.remove(node_id)
+        except KeyError:
+            pass
 
     def _parse_fetched_data(self, completed_fetch):
         tp = completed_fetch.topic_partition

kafka/producer/sender.py

@@ -315,7 +315,7 @@ class Sender(threading.Thread):
                     return True
 
             except Exception as e:
-                log.warn("%s: Got an exception when trying to find a node to send a transactional request to. Going to back off and retry", str(self), e)
+                log.warn("%s: Got an exception when trying to find a node to send a transactional request to. Going to back off and retry: %s", str(self), e)
                 if next_request_handler.needs_coordinator():
                     self._transaction_manager.lookup_coordinator_for_request(next_request_handler)
                     break

kafka/producer/transaction_manager.py

@@ -260,7 +260,7 @@ class TransactionManager(object):
         with self._lock:
             if self._current_state == TransactionState.ABORTING_TRANSACTION:
                 log.debug("Skipping transition to abortable error state since the transaction is already being "
-                          " aborted. Underlying exception: ", exc)
+                          " aborted. Underlying exception: %s", exc)
                 return
             self._transition_to(TransactionState.ABORTABLE_ERROR, error=exc)
 
@@ -687,7 +687,7 @@ class AddPartitionsToTxnHandler(TxnRequestHandler):
             if error is Errors.NoError:
                 continue
             elif error in (Errors.CoordinatorNotAvailableError, Errors.NotCoordinatorError):
-                self.transaction_manager._lookup_coordinator('transaction', self.transactiona_id)
+                self.transaction_manager._lookup_coordinator('transaction', self.transactional_id)
                 self.reenqueue()
                 return
             elif error is Errors.ConcurrentTransactionsError:
@@ -726,7 +726,7 @@ class AddPartitionsToTxnHandler(TxnRequestHandler):
         self.transaction_manager._pending_partitions_in_transaction -= partitions
 
         if unauthorized_topics:
-            self.abortable_error(Errors.TopicAuthorizationError(unauthorized_topics))
+            self.abortable_error(Errors.TopicAuthorizationFailedError(unauthorized_topics))
         elif has_partition_errors:
             self.abortable_error(Errors.KafkaError("Could not add partitions to transaction due to errors: %s" % (results)))
         else:
@@ -795,7 +795,7 @@ class FindCoordinatorHandler(TxnRequestHandler):
         elif error is Errors.TransactionalIdAuthorizationFailedError:
             self.fatal_error(error())
         elif error is Errors.GroupAuthorizationFailedError:
-            self.abortable_error(Errors.GroupAuthorizationError(self._coord_key))
+            self.abortable_error(error(self._coord_key))
         else:
             self.fatal_error(Errors.KafkaError(
                 "Could not find a coordinator with type %s with key %s due to"
@@ -888,7 +888,7 @@ class AddOffsetsToTxnHandler(TxnRequestHandler):
         elif error is Errors.TransactionalIdAuthorizationFailedError:
             self.fatal_error(error())
         elif error is Errors.GroupAuthorizationFailedError:
-            self.abortable_error(Errors.GroupAuthorizationError(self.consumer_group_id))
+            self.abortable_error(error(self.consumer_group_id))
         else:
             self.fatal_error(Errors.KafkaError("Unexpected error in AddOffsetsToTxnResponse: %s" % (error())))
 
@@ -955,7 +955,7 @@ class TxnOffsetCommitHandler(TxnRequestHandler):
             elif error is Errors.UnknownTopicOrPartitionError:
                 retriable_failure = True
             elif error is Errors.GroupAuthorizationFailedError:
-                self.abortable_error(Errors.GroupAuthorizationError(self.consumer_group_id))
+                self.abortable_error(error(self.consumer_group_id))
                 return
             elif error in (Errors.TransactionalIdAuthorizationFailedError,
                            Errors.InvalidProducerEpochError,

kafka/record/default_records.py

@@ -117,6 +117,8 @@ class DefaultRecordBase(object):
             checker, name = codecs.has_lz4, "lz4"
         elif compression_type == self.CODEC_ZSTD:
             checker, name = codecs.has_zstd, "zstd"
+        else:
+            raise UnsupportedCodecError("Unrecognized compression type: %s" % (compression_type,))
         if not checker():
             raise UnsupportedCodecError(
                 "Libraries for {} compression codec not found".format(name))

kafka/version.py

@@ -1 +1 @@
-__version__ = '2.2.1'
+__version__ = '2.2.3'

{kafka_python-2.2.1.dist-info → kafka_python-2.2.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.1
+Version: 2.2.3
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.1.dist-info → kafka_python-2.2.3.dist-info}/RECORD

@@ -1,17 +1,17 @@
 kafka/__init__.py,sha256=4dvHKZAxmD_4tfJ5wGcRV2X78vPcm8vsUoqceULevjA,1077
 kafka/client_async.py,sha256=joZB3AnL1mLwvV5fv61Pqn8mkP90FVvzcZ2tZsTGmvM,57060
-kafka/cluster.py,sha256=f9VNkWanz8X2Rn67XDdmunfx5aJ4s35JUsF5VYJ5hvk,16143
+kafka/cluster.py,sha256=N3_Al4We4ZhWzz6lVHy6SfqwDZfQy73iV7Qg4g4nxRs,16745
 kafka/codec.py,sha256=8NZpnehzNrhSBIjzbPVSvyFbSeLAqEntE7BfVHu-_9I,10036
-kafka/conn.py,sha256=IdWJTEYTalvsmnT74P0eu1Jzx6YdPyemvQovRoBQ8NM,70019
+kafka/conn.py,sha256=pDmzcn-m8oiFdvYh-97qbRLEBXh0sSl9nT74VIIRuEE,69472
 kafka/errors.py,sha256=J3R7z2hkbWA1hsD-bGHdRjcz6BYjP6RNVSQswA2UMmE,33749
 kafka/future.py,sha256=ZQStbfUYIPJRrgMfAWxxjrIRVxsw4WCtSR0J0bkyGno,2847
 kafka/socks5_wrapper.py,sha256=6woOaCTJXJ5e89_zdyW5BjOpyE4rCbYFH-kd-FeuPuk,9827
 kafka/structs.py,sha256=SJGzmLdV21jZyQ7247k0WFy16UiusgTHK3I-e4qzI-E,3058
 kafka/util.py,sha256=LV6BlELC8-889FpWM1RECX25sccoVrY2U0r5dRZjLNo,3781
-kafka/version.py,sha256=-7agX4LcQ0956RU4mY0EYyxYQU9llH7JklZOsqoujdU,22
+kafka/version.py,sha256=imyOcBgptJng0fWUAVwWSHYVE3csDgLCIYFSbnvEA-U,22
 kafka/admin/__init__.py,sha256=S_XxqyyV480_yXhttK79XZqNAmZyXRjspd3SoqYykE8,720
 kafka/admin/acl_resource.py,sha256=ak_dUsSni4SyP0ORbSKenZpwTy0Ykxq3FSt_9XgLR8k,8265
-kafka/admin/client.py,sha256=Mg0iuTFLOwc34JNQs99JvC0uDI_ZwZhF7x2jaEVstT4,78933
+kafka/admin/client.py,sha256=RabA8l8Im3iBEXgPVkiofNW6QyeatQHaymBWFZ8Sxkw,78929
 kafka/admin/config_resource.py,sha256=_JZWN_Q7jbuTtq2kdfHxWyTt_jI1LI-xnVGsf6oYGyY,1039
 kafka/admin/new_partitions.py,sha256=rYSb7S6VL706ZauSmiN5J9GDsep0HYRmkkAZUgT2JIg,757
 kafka/admin/new_topic.py,sha256=fvezLP9JXumqX-nU27Fgo0tj4d85ybcJgKluQImm3-0,1306
@@ -23,7 +23,7 @@ kafka/benchmarks/record_batch_compose.py,sha256=CnUreNg1lUT0Qx9enmSr-THmBl9PjVMf
 kafka/benchmarks/record_batch_read.py,sha256=vlFaWU2YWI379n_2M8qieb_S2uHUWKV0NquEYy5b-Ho,2184
 kafka/benchmarks/varint_speed.py,sha256=s4CuvKgDZL-_zna5E3vM8RgHjhXuW6pcaO1z1WYZ_0Y,12585
 kafka/consumer/__init__.py,sha256=NDdvtyuJgFyQZahqL9i5sYXGP6rOMIXWwHQEaZ1fCcs,122
-kafka/consumer/fetcher.py,sha256=0SUg8_8W0b-sVDg8dMq06ydR3P_mIGvUwxfbYI8cUIE,67963
+kafka/consumer/fetcher.py,sha256=EP7SHDS35BaIa3TqAu8GbI1HG8An15twGc9zia6LZ9M,68584
 kafka/consumer/group.py,sha256=Jvoal4SdOniweXeUhhYR_HxDUJmmUiKf4WrI_tuJfCQ,58857
 kafka/consumer/subscription_state.py,sha256=f_qJQMhTWQnUd_7lPj43gsagWSKGEmP4jpnEwA6s1Ec,23661
 kafka/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -69,8 +69,8 @@ kafka/producer/__init__.py,sha256=i3Wxih0NHjmqCkRNE54ial8fBp9siqabUE6ZGyL6oX8,12
 kafka/producer/future.py,sha256=UC3-g9QlgVFmbitrtMXVpeP0Pbvr7xl2kcw6bAehKG8,2983
 kafka/producer/kafka.py,sha256=tDCw1qr--ij4T88h2bIGxH7JQaRukkTl_kgnU83rRio,53379
 kafka/producer/record_accumulator.py,sha256=a_mdSATxl-3dVT2rVFh1gTwAv0wUzNbGwVXScwWJ5AE,28072
-kafka/producer/sender.py,sha256=-QOIp74ony5VwTykxksoIIJVbqZy2N2RJidx2tozpMs,37686
-kafka/producer/transaction_manager.py,sha256=cZlKFEZI6CY5zXEcTmCQE7TOyvK5V3H4FZLS1BKKEdY,41603
+kafka/producer/sender.py,sha256=2EeA3c7po89F2BLTPjex8-MFKzrCdbXAPvHHDa0SOec,37690
+kafka/producer/transaction_manager.py,sha256=HNfJNZwNfJtYdftn9SeaDfi7I5MKk0LD3sK64inuPt0,41537
 kafka/protocol/__init__.py,sha256=T1RBBlTH3zze0Cr1RqemPD4Z1b3IUDRmLOBfZTsPgLs,1184
 kafka/protocol/abstract.py,sha256=uOnuf6D8OTkL31Tp2QXG3VlzDPHVELGzM_bpSVa-_iw,424
 kafka/protocol/add_offsets_to_txn.py,sha256=Hya7vg6yqsV9XGLKWi8rES_VuN47-H4fdycg6mx8GLY,1486
@@ -101,7 +101,7 @@ kafka/protocol/types.py,sha256=f-lwfCqsJulYnBT1loek_KbMnZZqItN4YRIONjg3kbE,10244
 kafka/record/__init__.py,sha256=Q20hP_R5XX3AEnAlPkpoWzTLShESvxUT2OLXmI-JYEQ,129
 kafka/record/_crc32c.py,sha256=Ok-P62Yvg6D6rMGM9Z56OMjZWQlnps4xBbakg-sdxvI,5761
 kafka/record/abc.py,sha256=z1UYURHbD2RyyGRpVXKP598jck5eXU9p4M6iUo6ZSFo,4110
-kafka/record/default_records.py,sha256=l4DoU9eAkSn_f-ISfrMS8AOVYc846oLV_yMV91hsXH8,25879
+kafka/record/default_records.py,sha256=IuICFp0soETihkp8bUyjjksqTlzU45o-UYmo8joLBmo,25992
 kafka/record/legacy_records.py,sha256=bm1Y24PLVgLKtWqamESKvMk9P01uw3aQ8Z8q2QHxJy8,18858
 kafka/record/memory_records.py,sha256=b7RFxvaQ93drXSk3o3_YB3FQlVoESoBlGj3Z5PD25n8,8874
 kafka/record/util.py,sha256=LDajBWdYVetmXts_t9Q76CxEx7njgC9LnjMgz9yPEMM,3556
@@ -120,7 +120,7 @@ kafka/vendor/enum34.py,sha256=-u-lxAiJMt6ru4Do7NUDY9OpeWkYJMksb2xengJawFE,31204
 kafka/vendor/selectors34.py,sha256=gxejLO4eXf8mRSGXaQiknPig3GdX1rtsZiYOQJVuAy8,20594
 kafka/vendor/six.py,sha256=lLBa9_HrANP5BMZ7twEzg1M3wofwPmXyptuWmHX0brY,34826
 kafka/vendor/socketpair.py,sha256=Fi3PoY1Okkppab720wFk1BhHXyjcw7hi5DwhqrYZH2Y,2737
-kafka_python-2.2.1.dist-info/METADATA,sha256=zgYWaouiO6KsqTYxb5PQi5LbEk9iSzttcvJjBq3ie7Q,9951
-kafka_python-2.2.1.dist-info/WHEEL,sha256=XAkygS4h1cf0JYWV13kJhTWht2y9NqKAsZuiTHc2920,109
-kafka_python-2.2.1.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
-kafka_python-2.2.1.dist-info/RECORD,,
+kafka_python-2.2.3.dist-info/METADATA,sha256=5rHeRnLYzvBgmUIntERI45fmOmLAcZtPttVu2DDDZTs,9951
+kafka_python-2.2.3.dist-info/WHEEL,sha256=_itY3bZllKbLk93i0gzNzdweAt5eocJdfN7atrjOnvQ,109
+kafka_python-2.2.3.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
+kafka_python-2.2.3.dist-info/RECORD,,

{kafka_python-2.2.1.dist-info → kafka_python-2.2.3.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (80.0.0)
+Generator: setuptools (80.1.0)
 Root-Is-Purelib: true
 Tag: py2-none-any
 Tag: py3-none-any