kafka-python 2.2.8__py2.py3-none-any.whl → 2.2.10__py2.py3-none-any.whl

kafka/conn.py

@@ -313,7 +313,7 @@ class BrokerConnection(object):
 
     def _init_sasl_mechanism(self):
         if self.config['security_protocol'] in ('SASL_PLAINTEXT', 'SASL_SSL'):
-            self._sasl_mechanism = get_sasl_mechanism(self.config['sasl_mechanism'])(**self.config)
+            self._sasl_mechanism = get_sasl_mechanism(self.config['sasl_mechanism'])(host=self.host, **self.config)
         else:
             self._sasl_mechanism = None
 

kafka/consumer/fetcher.py

@@ -178,6 +178,9 @@ class Fetcher(six.Iterator):
         Arguments:
             partitions ([TopicPartition]): the partitions that need offsets reset
 
+        Returns:
+            bool: True if any partitions need reset; otherwise False (no reset pending)
+
         Raises:
             NoOffsetForPartitionError: if no offset reset strategy is defined
             KafkaTimeoutError if timeout_ms provided
@@ -189,7 +192,8 @@ class Fetcher(six.Iterator):
 
         partitions = self._subscriptions.partitions_needing_reset()
         if not partitions:
-            return
+            return False
+        log.debug('Resetting offsets for %s', partitions)
 
         offset_resets = dict()
         for tp in partitions:
@@ -198,6 +202,7 @@ class Fetcher(six.Iterator):
                 offset_resets[tp] = ts
 
         self._reset_offsets_async(offset_resets)
+        return True
 
     def offsets_by_times(self, timestamps, timeout_ms=None):
         """Fetch offset for each partition passed in ``timestamps`` map.

kafka/consumer/group.py

@@ -699,6 +699,7 @@ class KafkaConsumer(six.Iterator):
             dict: Map of topic to list of records (may be empty).
         """
         if not self._coordinator.poll(timeout_ms=timer.timeout_ms):
+            log.debug('poll: timeout during coordinator.poll(); returning early')
             return {}
 
         has_all_fetch_positions = self._update_fetch_positions(timeout_ms=timer.timeout_ms)
@@ -706,13 +707,13 @@ class KafkaConsumer(six.Iterator):
         # If data is available already, e.g. from a previous network client
         # poll() call to commit, then just return it immediately
         records, partial = self._fetcher.fetched_records(max_records, update_offsets=update_offsets)
-        log.debug('Fetched records: %s, %s', records, partial)
+        log.debug('poll: fetched records: %s, %s', records, partial)
         # Before returning the fetched records, we can send off the
         # next round of fetches and avoid block waiting for their
         # responses to enable pipelining while the user is handling the
         # fetched records.
         if not partial:
-            log.debug("Sending fetches")
+            log.debug("poll: Sending fetches")
             futures = self._fetcher.send_fetches()
             if len(futures):
                 self._client.poll(timeout_ms=0)
@@ -724,12 +725,14 @@ class KafkaConsumer(six.Iterator):
         # since the offset lookup may be backing off after a failure
         poll_timeout_ms = min(timer.timeout_ms, self._coordinator.time_to_next_poll() * 1000)
         if not has_all_fetch_positions:
+            log.debug('poll: do not have all fetch positions...')
             poll_timeout_ms = min(poll_timeout_ms, self.config['retry_backoff_ms'])
 
         self._client.poll(timeout_ms=poll_timeout_ms)
         # after the long poll, we should check whether the group needs to rebalance
         # prior to returning data so that the group can stabilize faster
         if self._coordinator.need_rejoin():
+            log.debug('poll: coordinator needs rejoin; returning early')
             return {}
 
         records, _ = self._fetcher.fetched_records(max_records, update_offsets=update_offsets)
@@ -1124,7 +1127,7 @@ class KafkaConsumer(six.Iterator):
             partitions (List[TopicPartition]): The partitions that need
                 updating fetch positions.
 
-        Returns True if fetch positions updated, False if timeout
+        Returns True if fetch positions updated, False if timeout or async reset is pending
 
         Raises:
             NoOffsetForPartitionError: If no offset is stored for a given
@@ -1135,15 +1138,13 @@ class KafkaConsumer(six.Iterator):
 
         if (self.config['api_version'] >= (0, 8, 1) and
             self.config['group_id'] is not None):
-            try:
-                # If there are any partitions which do not have a valid position and are not
-                # awaiting reset, then we need to fetch committed offsets. We will only do a
-                # coordinator lookup if there are partitions which have missing positions, so
-                # a consumer with manually assigned partitions can avoid a coordinator dependence
-                # by always ensuring that assigned partitions have an initial position.
-                self._coordinator.refresh_committed_offsets_if_needed(timeout_ms=timeout_ms)
-            except KafkaTimeoutError:
-                pass
+            # If there are any partitions which do not have a valid position and are not
+            # awaiting reset, then we need to fetch committed offsets. We will only do a
+            # coordinator lookup if there are partitions which have missing positions, so
+            # a consumer with manually assigned partitions can avoid a coordinator dependence
+            # by always ensuring that assigned partitions have an initial position.
+            if not self._coordinator.refresh_committed_offsets_if_needed(timeout_ms=timeout_ms):
+                return False
 
         # If there are partitions still needing a position and a reset policy is defined,
         # request reset using the default policy. If no reset strategy is defined and there
@@ -1152,8 +1153,7 @@ class KafkaConsumer(six.Iterator):
 
         # Finally send an asynchronous request to lookup and update the positions of any
         # partitions which are awaiting reset.
-        self._fetcher.reset_offsets_if_needed()
-        return False
+        return not self._fetcher.reset_offsets_if_needed()
 
     def _message_generator_v2(self):
         timeout_ms = 1000 * max(0, self._consumer_timeout - time.time())

kafka/coordinator/consumer.py

@@ -274,6 +274,7 @@ class ConsumerCoordinator(BaseCoordinator):
         try:
             self._invoke_completed_offset_commit_callbacks()
             if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
+                log.debug('coordinator.poll: timeout in ensure_coordinator_ready; returning early')
                 return False
 
             if self.config['api_version'] >= (0, 9) and self._subscription.partitions_auto_assigned():
@@ -293,9 +294,11 @@ class ConsumerCoordinator(BaseCoordinator):
                         metadata_update = self._client.cluster.request_update()
                         self._client.poll(future=metadata_update, timeout_ms=timer.timeout_ms)
                         if not metadata_update.is_done:
+                            log.debug('coordinator.poll: timeout updating metadata; returning early')
                             return False
 
                     if not self.ensure_active_group(timeout_ms=timer.timeout_ms):
+                        log.debug('coordinator.poll: timeout in ensure_active_group; returning early')
                         return False
 
                 self.poll_heartbeat()
@@ -427,7 +430,8 @@ class ConsumerCoordinator(BaseCoordinator):
         future_key = frozenset(partitions)
         timer = Timer(timeout_ms)
         while True:
-            self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms)
+            if not self.ensure_coordinator_ready(timeout_ms=timer.timeout_ms):
+                timer.maybe_raise()
 
             # contact coordinator to fetch committed offsets
             if future_key in self._offset_fetch_futures:
@@ -722,6 +726,7 @@ class ConsumerCoordinator(BaseCoordinator):
         return future
 
     def _handle_offset_commit_response(self, offsets, future, send_time, response):
+        log.debug("Received OffsetCommitResponse: %s", response)
         # TODO look at adding request_latency_ms to response (like java kafka)
         if self._consumer_sensors:
             self._consumer_sensors.commit_latency.record((time.time() - send_time) * 1000)
@@ -848,6 +853,7 @@ class ConsumerCoordinator(BaseCoordinator):
         return future
 
     def _handle_offset_fetch_response(self, future, response):
+        log.debug("Received OffsetFetchResponse: %s", response)
         if response.API_VERSION >= 2 and response.error_code != Errors.NoError.errno:
             error_type = Errors.for_code(response.error_code)
             log.debug("Offset fetch failed: %s", error_type.__name__)

kafka/sasl/gssapi.py

@@ -26,14 +26,15 @@ class SaslMechanismGSSAPI(SaslMechanism):
             raise ValueError('sasl_kerberos_service_name or sasl_kerberos_name required for GSSAPI sasl configuration')
         self._is_done = False
         self._is_authenticated = False
+        self.gssapi_name = None
         if config.get('sasl_kerberos_name', None) is not None:
             self.auth_id = str(config['sasl_kerberos_name'])
+            if isinstance(config['sasl_kerberos_name'], gssapi.Name):
+                self.gssapi_name = config['sasl_kerberos_name']
         else:
             kerberos_domain_name = config.get('sasl_kerberos_domain_name', '') or config.get('host', '')
             self.auth_id = config['sasl_kerberos_service_name'] + '@' + kerberos_domain_name
-        if isinstance(config.get('sasl_kerberos_name', None), gssapi.Name):
-            self.gssapi_name = config['sasl_kerberos_name']
-        else:
+        if self.gssapi_name is None:
             self.gssapi_name = gssapi.Name(self.auth_id, name_type=gssapi.NameType.hostbased_service).canonicalize(gssapi.MechType.kerberos)
         self._client_ctx = gssapi.SecurityContext(name=self.gssapi_name, usage='initiate')
         self._next_token = self._client_ctx.step(None)
@@ -43,9 +44,8 @@ class SaslMechanismGSSAPI(SaslMechanism):
         # so mark is_done after the final auth_bytes are provided
         # in practice we'll still receive a response when using SaslAuthenticate
         # but not when using the prior unframed approach.
-        if self._client_ctx.complete:
+        if self._is_authenticated:
             self._is_done = True
-            self._is_authenticated = True
         return self._next_token or b''
 
     def receive(self, auth_bytes):
@@ -74,6 +74,13 @@ class SaslMechanismGSSAPI(SaslMechanism):
             ]
             # add authorization identity to the response, and GSS-wrap
             self._next_token = self._client_ctx.wrap(b''.join(message_parts), False).message
+            # We need to identify the last token in auth_bytes();
+            # we can't rely on client_ctx.complete because it becomes True after generating
+            # the second-to-last token (after calling .step(auth_bytes) for the final time)
+            # We could introduce an additional state variable (i.e., self._final_token),
+            # but instead we just set _is_authenticated. Since the plugin interface does
+            # not read is_authenticated() until after is_done() is True, this should be fine.
+            self._is_authenticated = True
 
     def is_done(self):
         return self._is_done

kafka/version.py

@@ -1 +1 @@
-__version__ = '2.2.8'
+__version__ = '2.2.10'

{kafka_python-2.2.8.dist-info → kafka_python-2.2.10.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.8
+Version: 2.2.10
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.8.dist-info → kafka_python-2.2.10.dist-info}/RECORD

@@ -2,13 +2,13 @@ kafka/__init__.py,sha256=4dvHKZAxmD_4tfJ5wGcRV2X78vPcm8vsUoqceULevjA,1077
 kafka/client_async.py,sha256=R8q_rRpG3RrYrRmcZo7XgO2oSdpLJATNcq8w-1vIJ_8,56878
 kafka/cluster.py,sha256=N3_Al4We4ZhWzz6lVHy6SfqwDZfQy73iV7Qg4g4nxRs,16745
 kafka/codec.py,sha256=8NZpnehzNrhSBIjzbPVSvyFbSeLAqEntE7BfVHu-_9I,10036
-kafka/conn.py,sha256=pDmzcn-m8oiFdvYh-97qbRLEBXh0sSl9nT74VIIRuEE,69472
+kafka/conn.py,sha256=_yP-pGwEbkDmeutMOZjVilQXAnF4PWF_CDc60qC3DuE,69488
 kafka/errors.py,sha256=qX2Fp0qawU_HBNcZCwB7EDCmx3C2PehrETi6qSEJHmk,33290
 kafka/future.py,sha256=ZQStbfUYIPJRrgMfAWxxjrIRVxsw4WCtSR0J0bkyGno,2847
 kafka/socks5_wrapper.py,sha256=6woOaCTJXJ5e89_zdyW5BjOpyE4rCbYFH-kd-FeuPuk,9827
 kafka/structs.py,sha256=SJGzmLdV21jZyQ7247k0WFy16UiusgTHK3I-e4qzI-E,3058
 kafka/util.py,sha256=EnzCJuRkQ6Kh2lIdNwFKvT4PddkZ5bzop4ooGGIhe5g,4366
-kafka/version.py,sha256=IqD9h-jsDDriviuNsOqmA0rXPSWRSWW_V-IX7uKeRnk,22
+kafka/version.py,sha256=lfEF2tRAjIf7jwdP4hzCfb5zYNcswMyGc2yOh47sA9k,23
 kafka/admin/__init__.py,sha256=S_XxqyyV480_yXhttK79XZqNAmZyXRjspd3SoqYykE8,720
 kafka/admin/acl_resource.py,sha256=ak_dUsSni4SyP0ORbSKenZpwTy0Ykxq3FSt_9XgLR8k,8265
 kafka/admin/client.py,sha256=RabA8l8Im3iBEXgPVkiofNW6QyeatQHaymBWFZ8Sxkw,78929
@@ -23,12 +23,12 @@ kafka/benchmarks/record_batch_compose.py,sha256=CnUreNg1lUT0Qx9enmSr-THmBl9PjVMf
 kafka/benchmarks/record_batch_read.py,sha256=vlFaWU2YWI379n_2M8qieb_S2uHUWKV0NquEYy5b-Ho,2184
 kafka/benchmarks/varint_speed.py,sha256=s4CuvKgDZL-_zna5E3vM8RgHjhXuW6pcaO1z1WYZ_0Y,12585
 kafka/consumer/__init__.py,sha256=NDdvtyuJgFyQZahqL9i5sYXGP6rOMIXWwHQEaZ1fCcs,122
-kafka/consumer/fetcher.py,sha256=iwYhWotaEQ55oXTzGKPUOYxvC_6FcoIks_ZqL-gu3DE,68855
-kafka/consumer/group.py,sha256=xmEpVMPJbCAk9__pdAOMswh8I-Ujj5hBax_hPZHZb_s,58758
+kafka/consumer/fetcher.py,sha256=5b-_4VsmQXrRd2Ul8LMZ93TZJHVEoYpmTPB6QcOMizw,69045
+kafka/consumer/group.py,sha256=oieWNHM1NWiOZT8pasOLfFJAbmJEXJ4h7PgUtklxo_Q,58944
 kafka/consumer/subscription_state.py,sha256=f_qJQMhTWQnUd_7lPj43gsagWSKGEmP4jpnEwA6s1Ec,23661
 kafka/coordinator/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kafka/coordinator/base.py,sha256=NmHXyqoJZVXL2KhahXLCOH1zVx9gyTdhrt-_unxIAaE,54365
-kafka/coordinator/consumer.py,sha256=52yYmM5o_VzNOKvMJkN7CwHlvWWHoNu6xAFuX52JCdo,45870
+kafka/coordinator/consumer.py,sha256=le4bGbHfrDK4pperYXekPKzuZW576uXL324IOwS4Kmw,46348
 kafka/coordinator/heartbeat.py,sha256=LeJJlwz1oUEOfEMIFT-R7ZOHBQ-b-luVKwmKyWxLfDo,3242
 kafka/coordinator/protocol.py,sha256=wTaIOnUVbj0CKXZ82FktZo-zMRvOCk3hdQAoHJ62e3I,1041
 kafka/coordinator/assignors/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
@@ -107,7 +107,7 @@ kafka/record/memory_records.py,sha256=b7RFxvaQ93drXSk3o3_YB3FQlVoESoBlGj3Z5PD25n
 kafka/record/util.py,sha256=LDajBWdYVetmXts_t9Q76CxEx7njgC9LnjMgz9yPEMM,3556
 kafka/sasl/__init__.py,sha256=wUUGIKRe52J6Qekj7hSypg44vWTrkYsEdVafQC7cX5s,1106
 kafka/sasl/abc.py,sha256=R0BZOk3AYEGyehiGbbg-LMRvFAlWZsh0fBiESgUpBYw,657
-kafka/sasl/gssapi.py,sha256=joeQjWVDujcG8cdaRdUagSy_KoPhpkzXuVDi8OBqId0,4019
+kafka/sasl/gssapi.py,sha256=HqN9yikeT75zvq42SxuKhsH1WAMT3ZaT9j-E6LAwggw,4591
 kafka/sasl/msk.py,sha256=ndUZqPTdgItptiRimVlUAGuFZz1cerQc1KufYMIcPkg,7684
 kafka/sasl/oauth.py,sha256=dh87tVi-dlS5lIzgYsC4m7IXUhlLdejaMb9Ua6oYaB0,3425
 kafka/sasl/plain.py,sha256=PMfoWT856wx6nF_LhpfPKEnD7BRNx5l6rDhAqxBnMWU,1317
@@ -120,7 +120,7 @@ kafka/vendor/enum34.py,sha256=-u-lxAiJMt6ru4Do7NUDY9OpeWkYJMksb2xengJawFE,31204
 kafka/vendor/selectors34.py,sha256=gxejLO4eXf8mRSGXaQiknPig3GdX1rtsZiYOQJVuAy8,20594
 kafka/vendor/six.py,sha256=lLBa9_HrANP5BMZ7twEzg1M3wofwPmXyptuWmHX0brY,34826
 kafka/vendor/socketpair.py,sha256=Fi3PoY1Okkppab720wFk1BhHXyjcw7hi5DwhqrYZH2Y,2737
-kafka_python-2.2.8.dist-info/METADATA,sha256=SHY6zV6gEmQJmTCl59wWgrwSG-D88F6gEaqkZGfndR8,9951
-kafka_python-2.2.8.dist-info/WHEEL,sha256=egKm5cKfE6OqlHwodY8Jjp4yqZDBXgsj09UsV5ojd_U,109
-kafka_python-2.2.8.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
-kafka_python-2.2.8.dist-info/RECORD,,
+kafka_python-2.2.10.dist-info/METADATA,sha256=LNkDuppqocj9ondpnWf2PmlMQkYN5w-K8nnozxZKKmQ,9952
+kafka_python-2.2.10.dist-info/WHEEL,sha256=egKm5cKfE6OqlHwodY8Jjp4yqZDBXgsj09UsV5ojd_U,109
+kafka_python-2.2.10.dist-info/top_level.txt,sha256=IivJz7l5WHdLNDT6RIiVAlhjQzYRwGqBBmKHZ7WjPeM,6
+kafka_python-2.2.10.dist-info/RECORD,,