kafka-python 2.2.10__tar.gz → 2.2.11__tar.gz

{kafka_python-2.2.10 → kafka_python-2.2.11}/CHANGES.md

@@ -1,3 +1,11 @@
+# 2.2.11 (June 5, 2025)
+
+Fixes
+* Do not ignore metadata response for single topic with error (#2640)
+* Fix decoding bug in AWS_MSK_IAM mechanism (#2639)
+* Add synchronized decorator; add lock to subscription state (#2636)
+* Update build links in documentation (#2634)
+
 # 2.2.10 (May 22, 2025)
 
 Fixes

{kafka_python-2.2.10 → kafka_python-2.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.10
+Version: 2.2.11
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.10 → kafka_python-2.2.11}/kafka/cluster.py

@@ -245,13 +245,6 @@ class ClusterMetadata(object):
 
         Returns: None
         """
-        # In the common case where we ask for a single topic and get back an
-        # error, we should fail the future
-        if len(metadata.topics) == 1 and metadata.topics[0][0] != Errors.NoError.errno:
-            error_code, topic = metadata.topics[0][:2]
-            error = Errors.for_code(error_code)(topic)
-            return self.failed_update(error)
-
         if not metadata.brokers:
             log.warning("No broker metadata found in MetadataResponse -- ignoring.")
             return self.failed_update(Errors.MetadataEmptyBrokerList(metadata))
@@ -349,7 +342,15 @@ class ClusterMetadata(object):
         self._last_successful_refresh_ms = now
 
         if f:
-            f.success(self)
+            # In the common case where we ask for a single topic and get back an
+            # error, we should fail the future
+            if len(metadata.topics) == 1 and metadata.topics[0][0] != Errors.NoError.errno:
+                error_code, topic = metadata.topics[0][:2]
+                error = Errors.for_code(error_code)(topic)
+                f.failure(error)
+            else:
+                f.success(self)
+
         log.debug("Updated cluster metadata to %s", self)
 
         for listener in self._listeners:

{kafka_python-2.2.10 → kafka_python-2.2.11}/kafka/consumer/subscription_state.py

@@ -15,6 +15,7 @@ except ImportError:
 import logging
 import random
 import re
+import threading
 import time
 
 from kafka.vendor import six
@@ -22,7 +23,7 @@ from kafka.vendor import six
 import kafka.errors as Errors
 from kafka.protocol.list_offsets import OffsetResetStrategy
 from kafka.structs import OffsetAndMetadata
-from kafka.util import ensure_valid_topic_name
+from kafka.util import ensure_valid_topic_name, synchronized
 
 log = logging.getLogger(__name__)
 
@@ -84,6 +85,7 @@ class SubscriptionState(object):
         self.assignment = OrderedDict()
         self.rebalance_listener = None
         self.listeners = []
+        self._lock = threading.RLock()
 
     def _set_subscription_type(self, subscription_type):
         if not isinstance(subscription_type, SubscriptionType):
@@ -93,6 +95,7 @@ class SubscriptionState(object):
         elif self.subscription_type != subscription_type:
             raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
 
+    @synchronized
     def subscribe(self, topics=(), pattern=None, listener=None):
         """Subscribe to a list of topics, or a topic regex pattern.
 
@@ -147,6 +150,7 @@ class SubscriptionState(object):
             raise TypeError('listener must be a ConsumerRebalanceListener')
         self.rebalance_listener = listener
 
+    @synchronized
     def change_subscription(self, topics):
         """Change the topic subscription.
 
@@ -178,6 +182,7 @@ class SubscriptionState(object):
         self.subscription = set(topics)
         self._group_subscription.update(topics)
 
+    @synchronized
     def group_subscribe(self, topics):
         """Add topics to the current group subscription.
 
@@ -191,6 +196,7 @@ class SubscriptionState(object):
             raise Errors.IllegalStateError(self._SUBSCRIPTION_EXCEPTION_MESSAGE)
         self._group_subscription.update(topics)
 
+    @synchronized
     def reset_group_subscription(self):
         """Reset the group's subscription to only contain topics subscribed by this consumer."""
         if not self.partitions_auto_assigned():
@@ -198,6 +204,7 @@ class SubscriptionState(object):
         assert self.subscription is not None, 'Subscription required'
         self._group_subscription.intersection_update(self.subscription)
 
+    @synchronized
     def assign_from_user(self, partitions):
         """Manually assign a list of TopicPartitions to this consumer.
 
@@ -222,6 +229,7 @@ class SubscriptionState(object):
             self._set_assignment({partition: self.assignment.get(partition, TopicPartitionState())
                                   for partition in partitions})
 
+    @synchronized
     def assign_from_subscribed(self, assignments):
         """Update the assignment to the specified partitions
 
@@ -258,6 +266,7 @@ class SubscriptionState(object):
             for tp in topic_partitions[topic]:
                 self.assignment[tp] = partition_states[tp]
 
+    @synchronized
     def unsubscribe(self):
         """Clear all topic subscriptions and partition assignments"""
         self.subscription = None
@@ -266,6 +275,7 @@ class SubscriptionState(object):
         self.subscribed_pattern = None
         self.subscription_type = SubscriptionType.NONE
 
+    @synchronized
     def group_subscription(self):
         """Get the topic subscription for the group.
 
@@ -281,6 +291,7 @@ class SubscriptionState(object):
         """
         return self._group_subscription
 
+    @synchronized
     def seek(self, partition, offset):
         """Manually specify the fetch offset for a TopicPartition.
 
@@ -298,15 +309,18 @@ class SubscriptionState(object):
             raise TypeError("offset must be type in or OffsetAndMetadata")
         self.assignment[partition].seek(offset)
 
+    @synchronized
     def assigned_partitions(self):
         """Return set of TopicPartitions in current assignment."""
         return set(self.assignment.keys())
 
+    @synchronized
     def paused_partitions(self):
         """Return current set of paused TopicPartitions."""
         return set(partition for partition in self.assignment
                    if self.is_paused(partition))
 
+    @synchronized
     def fetchable_partitions(self):
         """Return ordered list of TopicPartitions that should be Fetched."""
         fetchable = list()
@@ -315,10 +329,12 @@ class SubscriptionState(object):
                 fetchable.append(partition)
         return fetchable
 
+    @synchronized
     def partitions_auto_assigned(self):
         """Return True unless user supplied partitions manually."""
         return self.subscription_type in (SubscriptionType.AUTO_TOPICS, SubscriptionType.AUTO_PATTERN)
 
+    @synchronized
     def all_consumed_offsets(self):
         """Returns consumed offsets as {TopicPartition: OffsetAndMetadata}"""
         all_consumed = {}
@@ -327,6 +343,7 @@ class SubscriptionState(object):
                 all_consumed[partition] = state.position
         return all_consumed
 
+    @synchronized
     def request_offset_reset(self, partition, offset_reset_strategy=None):
         """Mark partition for offset reset using specified or default strategy.
 
@@ -338,23 +355,28 @@ class SubscriptionState(object):
             offset_reset_strategy = self._default_offset_reset_strategy
         self.assignment[partition].reset(offset_reset_strategy)
 
+    @synchronized
     def set_reset_pending(self, partitions, next_allowed_reset_time):
         for partition in partitions:
             self.assignment[partition].set_reset_pending(next_allowed_reset_time)
 
+    @synchronized
     def has_default_offset_reset_policy(self):
         """Return True if default offset reset policy is Earliest or Latest"""
         return self._default_offset_reset_strategy != OffsetResetStrategy.NONE
 
+    @synchronized
     def is_offset_reset_needed(self, partition):
         return self.assignment[partition].awaiting_reset
 
+    @synchronized
     def has_all_fetch_positions(self):
         for state in six.itervalues(self.assignment):
             if not state.has_valid_position:
                 return False
         return True
 
+    @synchronized
     def missing_fetch_positions(self):
         missing = set()
         for partition, state in six.iteritems(self.assignment):
@@ -362,9 +384,11 @@ class SubscriptionState(object):
                 missing.add(partition)
         return missing
 
+    @synchronized
     def has_valid_position(self, partition):
         return partition in self.assignment and self.assignment[partition].has_valid_position
 
+    @synchronized
     def reset_missing_positions(self):
         partitions_with_no_offsets = set()
         for tp, state in six.iteritems(self.assignment):
@@ -377,6 +401,7 @@ class SubscriptionState(object):
         if partitions_with_no_offsets:
             raise Errors.NoOffsetForPartitionError(partitions_with_no_offsets)
 
+    @synchronized
     def partitions_needing_reset(self):
         partitions = set()
         for tp, state in six.iteritems(self.assignment):
@@ -384,25 +409,32 @@ class SubscriptionState(object):
                 partitions.add(tp)
         return partitions
 
+    @synchronized
     def is_assigned(self, partition):
         return partition in self.assignment
 
+    @synchronized
     def is_paused(self, partition):
         return partition in self.assignment and self.assignment[partition].paused
 
+    @synchronized
     def is_fetchable(self, partition):
         return partition in self.assignment and self.assignment[partition].is_fetchable()
 
+    @synchronized
     def pause(self, partition):
         self.assignment[partition].pause()
 
+    @synchronized
     def resume(self, partition):
         self.assignment[partition].resume()
 
+    @synchronized
     def reset_failed(self, partitions, next_retry_time):
         for partition in partitions:
             self.assignment[partition].reset_failed(next_retry_time)
 
+    @synchronized
     def move_partition_to_end(self, partition):
         if partition in self.assignment:
             try:
@@ -411,6 +443,7 @@ class SubscriptionState(object):
                 state = self.assignment.pop(partition)
                 self.assignment[partition] = state
 
+    @synchronized
     def position(self, partition):
         return self.assignment[partition].position
 

{kafka_python-2.2.10 → kafka_python-2.2.11}/kafka/producer/kafka.py

@@ -960,9 +960,11 @@ class KafkaProducer(object):
             future.add_both(lambda e, *args: e.set(), metadata_event)
             self._sender.wakeup()
             metadata_event.wait(timer.timeout_ms / 1000)
-            if not metadata_event.is_set():
+            if not future.is_done:
                 raise Errors.KafkaTimeoutError(
                     "Failed to update metadata after %.1f secs." % (max_wait_ms / 1000,))
+            elif future.failed() and not future.retriable():
+                raise future.exception
             elif topic in self._metadata.unauthorized_topics:
                 raise Errors.TopicAuthorizationFailedError(set([topic]))
             else:

{kafka_python-2.2.10 → kafka_python-2.2.11}/kafka/sasl/msk.py

@@ -4,6 +4,7 @@ import datetime
 import hashlib
 import hmac
 import json
+import logging
 import string
 
 # needed for AWS_MSK_IAM authentication:
@@ -13,10 +14,14 @@ except ImportError:
     # no botocore available, will disable AWS_MSK_IAM mechanism
     BotoSession = None
 
+from kafka.errors import KafkaConfigurationError
 from kafka.sasl.abc import SaslMechanism
 from kafka.vendor.six.moves import urllib
 
 
+log = logging.getLogger(__name__)
+
+
 class SaslMechanismAwsMskIam(SaslMechanism):
     def __init__(self, **config):
         assert BotoSession is not None, 'AWS_MSK_IAM requires the "botocore" package'
@@ -27,22 +32,28 @@ class SaslMechanismAwsMskIam(SaslMechanism):
         self._is_done = False
         self._is_authenticated = False
 
-    def auth_bytes(self):
+    def _build_client(self):
         session = BotoSession()
         credentials = session.get_credentials().get_frozen_credentials()
-        client = AwsMskIamClient(
+        if not session.get_config_variable('region'):
+            raise KafkaConfigurationError('Unable to determine region for AWS MSK cluster. Is AWS_DEFAULT_REGION set?')
+        return AwsMskIamClient(
             host=self.host,
             access_key=credentials.access_key,
             secret_key=credentials.secret_key,
             region=session.get_config_variable('region'),
             token=credentials.token,
         )
+
+    def auth_bytes(self):
+        client = self._build_client()
+        log.debug("Generating auth token for MSK scope: %s", client._scope)
         return client.first_message()
 
     def receive(self, auth_bytes):
         self._is_done = True
         self._is_authenticated = auth_bytes != b''
-        self._auth = auth_bytes.deode('utf-8')
+        self._auth = auth_bytes.decode('utf-8')
 
     def is_done(self):
         return self._is_done

{kafka_python-2.2.10 → kafka_python-2.2.11}/kafka/util.py

@@ -1,6 +1,7 @@
 from __future__ import absolute_import, division
 
 import binascii
+import functools
 import re
 import time
 import weakref
@@ -129,3 +130,11 @@ class Dict(dict):
     See: https://docs.python.org/2/library/weakref.html
     """
     pass
+
+
+def synchronized(func):
+    def wrapper(self, *args, **kwargs):
+        with self._lock:
+            return func(self, *args, **kwargs)
+    functools.update_wrapper(wrapper, func)
+    return wrapper

kafka_python-2.2.11/kafka/version.py

@@ -0,0 +1 @@
+__version__ = '2.2.11'

{kafka_python-2.2.10 → kafka_python-2.2.11}/kafka_python.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kafka-python
-Version: 2.2.10
+Version: 2.2.11
 Summary: Pure Python client for Apache Kafka
 Author-email: Dana Powers <dana.powers@gmail.com>
 Project-URL: Homepage, https://github.com/dpkp/kafka-python

{kafka_python-2.2.10 → kafka_python-2.2.11}/test/test_cluster.py

@@ -136,6 +136,21 @@ def test_metadata_v7():
     assert cluster._partitions['topic-1'][0].leader_epoch == 0
 
 
+def test_unauthorized_topic():
+    cluster = ClusterMetadata()
+    assert len(cluster.brokers()) == 0
+
+    cluster.update_metadata(MetadataResponse[0](
+        [(0, 'foo', 12), (1, 'bar', 34)],
+        [(29, 'unauthorized-topic', [])]))  # single topic w/ unauthorized error
+
+    # broker metadata should get updated
+    assert len(cluster.brokers()) == 2
+
+    # topic should be added to unauthorized list
+    assert 'unauthorized-topic' in cluster.unauthorized_topics
+
+
 def test_collect_hosts__happy_path():
     hosts = "127.0.0.1:1234,127.0.0.1"
     results = collect_hosts(hosts)

kafka_python-2.2.10/kafka/version.py

@@ -1 +0,0 @@
-__version__ = '2.2.10'