kafka-python 2.0.3__py2.py3-none-any.whl → 2.0.5__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
kafka/admin/client.py CHANGED
@@ -1,9 +1,10 @@
1
- from __future__ import absolute_import
1
+ from __future__ import absolute_import, division
2
2
 
3
3
  from collections import defaultdict
4
4
  import copy
5
5
  import logging
6
6
  import socket
7
+ import time
7
8
 
8
9
  from . import ConfigResourceType
9
10
  from kafka.vendor import six
@@ -72,7 +73,7 @@ class KafkaAdminClient(object):
72
73
  reconnection attempts will continue periodically with this fixed
73
74
  rate. To avoid connection storms, a randomization factor of 0.2
74
75
  will be applied to the backoff resulting in a random range between
75
- 20% below and 20% above the computed value. Default: 1000.
76
+ 20% below and 20% above the computed value. Default: 30000.
76
77
  request_timeout_ms (int): Client request timeout in milliseconds.
77
78
  Default: 30000.
78
79
  connections_max_idle_ms: Close idle connections after the number of
@@ -156,7 +157,7 @@ class KafkaAdminClient(object):
156
157
  'request_timeout_ms': 30000,
157
158
  'connections_max_idle_ms': 9 * 60 * 1000,
158
159
  'reconnect_backoff_ms': 50,
159
- 'reconnect_backoff_max_ms': 1000,
160
+ 'reconnect_backoff_max_ms': 30000,
160
161
  'max_in_flight_requests_per_connection': 5,
161
162
  'receive_buffer_bytes': None,
162
163
  'send_buffer_bytes': None,
@@ -212,11 +213,13 @@ class KafkaAdminClient(object):
212
213
  metric_group_prefix='admin',
213
214
  **self.config
214
215
  )
215
- self._client.check_version(timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
216
216
 
217
217
  # Get auto-discovered version from client if necessary
218
218
  if self.config['api_version'] is None:
219
219
  self.config['api_version'] = self._client.config['api_version']
220
+ else:
221
+ # need to run check_version for get_api_versions()
222
+ self._client.check_version(timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
220
223
 
221
224
  self._closed = False
222
225
  self._refresh_controller_id()
@@ -240,8 +243,11 @@ class KafkaAdminClient(object):
240
243
  This resolves to the lesser of either the latest api version this
241
244
  library supports, or the max version supported by the broker.
242
245
 
243
- :param operation: A list of protocol operation versions from kafka.protocol.
244
- :return: The max matching version number between client and broker.
246
+ Arguments:
247
+ operation: A list of protocol operation versions from kafka.protocol.
248
+
249
+ Returns:
250
+ int: The max matching version number between client and broker.
245
251
  """
246
252
  broker_api_versions = self._client.get_api_versions()
247
253
  api_key = operation[0].API_KEY
@@ -262,29 +268,41 @@ class KafkaAdminClient(object):
262
268
  def _validate_timeout(self, timeout_ms):
263
269
  """Validate the timeout is set or use the configuration default.
264
270
 
265
- :param timeout_ms: The timeout provided by api call, in milliseconds.
266
- :return: The timeout to use for the operation.
271
+ Arguments:
272
+ timeout_ms: The timeout provided by api call, in milliseconds.
273
+
274
+ Returns:
275
+ The timeout to use for the operation.
267
276
  """
268
277
  return timeout_ms or self.config['request_timeout_ms']
269
278
 
270
- def _refresh_controller_id(self):
279
+ def _refresh_controller_id(self, timeout_ms=30000):
271
280
  """Determine the Kafka cluster controller."""
272
281
  version = self._matching_api_version(MetadataRequest)
273
282
  if 1 <= version <= 6:
274
- request = MetadataRequest[version]()
275
- future = self._send_request_to_node(self._client.least_loaded_node(), request)
276
-
277
- self._wait_for_futures([future])
278
-
279
- response = future.value
280
- controller_id = response.controller_id
281
- # verify the controller is new enough to support our requests
282
- controller_version = self._client.check_version(controller_id, timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
283
- if controller_version < (0, 10, 0):
284
- raise IncompatibleBrokerVersion(
285
- "The controller appears to be running Kafka {}. KafkaAdminClient requires brokers >= 0.10.0.0."
286
- .format(controller_version))
287
- self._controller_id = controller_id
283
+ timeout_at = time.time() + timeout_ms / 1000
284
+ while time.time() < timeout_at:
285
+ request = MetadataRequest[version]()
286
+ future = self._send_request_to_node(self._client.least_loaded_node(), request)
287
+
288
+ self._wait_for_futures([future])
289
+
290
+ response = future.value
291
+ controller_id = response.controller_id
292
+ if controller_id == -1:
293
+ log.warning("Controller ID not available, got -1")
294
+ time.sleep(1)
295
+ continue
296
+ # verify the controller is new enough to support our requests
297
+ controller_version = self._client.check_version(node_id=controller_id, timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
298
+ if controller_version < (0, 10, 0):
299
+ raise IncompatibleBrokerVersion(
300
+ "The controller appears to be running Kafka {}. KafkaAdminClient requires brokers >= 0.10.0.0."
301
+ .format(controller_version))
302
+ self._controller_id = controller_id
303
+ return
304
+ else:
305
+ raise Errors.NodeNotAvailableError('controller')
288
306
  else:
289
307
  raise UnrecognizedBrokerVersion(
290
308
  "Kafka Admin interface cannot determine the controller using MetadataRequest_v{}."
@@ -293,9 +311,12 @@ class KafkaAdminClient(object):
293
311
  def _find_coordinator_id_send_request(self, group_id):
294
312
  """Send a FindCoordinatorRequest to a broker.
295
313
 
296
- :param group_id: The consumer group ID. This is typically the group
314
+ Arguments:
315
+ group_id: The consumer group ID. This is typically the group
297
316
  name as a string.
298
- :return: A message future
317
+
318
+ Returns:
319
+ A message future
299
320
  """
300
321
  # TODO add support for dynamically picking version of
301
322
  # GroupCoordinatorRequest which was renamed to FindCoordinatorRequest.
@@ -315,8 +336,11 @@ class KafkaAdminClient(object):
315
336
  def _find_coordinator_id_process_response(self, response):
316
337
  """Process a FindCoordinatorResponse.
317
338
 
318
- :param response: a FindCoordinatorResponse.
319
- :return: The node_id of the broker that is the coordinator.
339
+ Arguments:
340
+ response: a FindCoordinatorResponse.
341
+
342
+ Returns:
343
+ The node_id of the broker that is the coordinator.
320
344
  """
321
345
  if response.API_VERSION <= 0:
322
346
  error_type = Errors.for_code(response.error_code)
@@ -339,9 +363,12 @@ class KafkaAdminClient(object):
339
363
  Will block until the FindCoordinatorResponse is received for all groups.
340
364
  Any errors are immediately raised.
341
365
 
342
- :param group_ids: A list of consumer group IDs. This is typically the group
366
+ Arguments:
367
+ group_ids: A list of consumer group IDs. This is typically the group
343
368
  name as a string.
344
- :return: A dict of {group_id: node_id} where node_id is the id of the
369
+
370
+ Returns:
371
+ A dict of {group_id: node_id} where node_id is the id of the
345
372
  broker that is the coordinator for the corresponding group.
346
373
  """
347
374
  groups_futures = {
@@ -358,18 +385,24 @@ class KafkaAdminClient(object):
358
385
  def _send_request_to_node(self, node_id, request, wakeup=True):
359
386
  """Send a Kafka protocol message to a specific broker.
360
387
 
361
- Returns a future that may be polled for status and results.
388
+ Arguments:
389
+ node_id: The broker id to which to send the message.
390
+ request: The message to send.
391
+
362
392
 
363
- :param node_id: The broker id to which to send the message.
364
- :param request: The message to send.
365
- :param wakeup: Optional flag to disable thread-wakeup.
366
- :return: A future object that may be polled for status and results.
367
- :exception: The exception if the message could not be sent.
393
+ Keyword Arguments:
394
+ wakeup (bool, optional): Optional flag to disable thread-wakeup.
395
+
396
+ Returns:
397
+ A future object that may be polled for status and results.
398
+
399
+ Raises:
400
+ The exception if the message could not be sent.
368
401
  """
369
402
  while not self._client.ready(node_id):
370
403
  # poll until the connection to broker is ready, otherwise send()
371
404
  # will fail with NodeNotReadyError
372
- self._client.poll()
405
+ self._client.poll(timeout_ms=200)
373
406
  return self._client.send(node_id, request, wakeup)
374
407
 
375
408
  def _send_request_to_controller(self, request):
@@ -377,8 +410,11 @@ class KafkaAdminClient(object):
377
410
 
378
411
  Will block until the message result is received.
379
412
 
380
- :param request: The message to send.
381
- :return: The Kafka protocol response for the message.
413
+ Arguments:
414
+ request: The message to send.
415
+
416
+ Returns:
417
+ The Kafka protocol response for the message.
382
418
  """
383
419
  tries = 2 # in case our cached self._controller_id is outdated
384
420
  while tries:
@@ -418,6 +454,18 @@ class KafkaAdminClient(object):
418
454
 
419
455
  @staticmethod
420
456
  def _convert_new_topic_request(new_topic):
457
+ """
458
+ Build the tuple required by CreateTopicsRequest from a NewTopic object.
459
+
460
+ Arguments:
461
+ new_topic: A NewTopic instance containing name, partition count, replication factor,
462
+ replica assignments, and config entries.
463
+
464
+ Returns:
465
+ A tuple in the form:
466
+ (topic_name, num_partitions, replication_factor, [(partition_id, [replicas])...],
467
+ [(config_key, config_value)...])
468
+ """
421
469
  return (
422
470
  new_topic.name,
423
471
  new_topic.num_partitions,
@@ -433,12 +481,17 @@ class KafkaAdminClient(object):
433
481
  def create_topics(self, new_topics, timeout_ms=None, validate_only=False):
434
482
  """Create new topics in the cluster.
435
483
 
436
- :param new_topics: A list of NewTopic objects.
437
- :param timeout_ms: Milliseconds to wait for new topics to be created
438
- before the broker returns.
439
- :param validate_only: If True, don't actually create new topics.
440
- Not supported by all versions. Default: False
441
- :return: Appropriate version of CreateTopicResponse class.
484
+ Arguments:
485
+ new_topics: A list of NewTopic objects.
486
+
487
+ Keyword Arguments:
488
+ timeout_ms (numeric, optional): Milliseconds to wait for new topics to be created
489
+ before the broker returns.
490
+ validate_only (bool, optional): If True, don't actually create new topics.
491
+ Not supported by all versions. Default: False
492
+
493
+ Returns:
494
+ Appropriate version of CreateTopicResponse class.
442
495
  """
443
496
  version = self._matching_api_version(CreateTopicsRequest)
444
497
  timeout_ms = self._validate_timeout(timeout_ms)
@@ -468,10 +521,15 @@ class KafkaAdminClient(object):
468
521
  def delete_topics(self, topics, timeout_ms=None):
469
522
  """Delete topics from the cluster.
470
523
 
471
- :param topics: A list of topic name strings.
472
- :param timeout_ms: Milliseconds to wait for topics to be deleted
473
- before the broker returns.
474
- :return: Appropriate version of DeleteTopicsResponse class.
524
+ Arguments:
525
+ topics ([str]): A list of topic name strings.
526
+
527
+ Keyword Arguments:
528
+ timeout_ms (numeric, optional): Milliseconds to wait for topics to be deleted
529
+ before the broker returns.
530
+
531
+ Returns:
532
+ Appropriate version of DeleteTopicsResponse class.
475
533
  """
476
534
  version = self._matching_api_version(DeleteTopicsRequest)
477
535
  timeout_ms = self._validate_timeout(timeout_ms)
@@ -515,16 +573,38 @@ class KafkaAdminClient(object):
515
573
  return future.value
516
574
 
517
575
  def list_topics(self):
576
+ """Retrieve a list of all topic names in the cluster.
577
+
578
+ Returns:
579
+ A list of topic name strings.
580
+ """
518
581
  metadata = self._get_cluster_metadata(topics=None)
519
582
  obj = metadata.to_object()
520
583
  return [t['topic'] for t in obj['topics']]
521
584
 
522
585
  def describe_topics(self, topics=None):
586
+ """Fetch metadata for the specified topics or all topics if None.
587
+
588
+ Keyword Arguments:
589
+ topics ([str], optional) A list of topic names. If None, metadata for all
590
+ topics is retrieved.
591
+
592
+ Returns:
593
+ A list of dicts describing each topic (including partition info).
594
+ """
523
595
  metadata = self._get_cluster_metadata(topics=topics)
524
596
  obj = metadata.to_object()
525
597
  return obj['topics']
526
598
 
527
599
  def describe_cluster(self):
600
+ """
601
+ Fetch cluster-wide metadata such as the list of brokers, the controller ID,
602
+ and the cluster ID.
603
+
604
+
605
+ Returns:
606
+ A dict with cluster-wide metadata, excluding topic details.
607
+ """
528
608
  metadata = self._get_cluster_metadata()
529
609
  obj = metadata.to_object()
530
610
  obj.pop('topics') # We have 'describe_topics' for this
@@ -532,6 +612,15 @@ class KafkaAdminClient(object):
532
612
 
533
613
  @staticmethod
534
614
  def _convert_describe_acls_response_to_acls(describe_response):
615
+ """Convert a DescribeAclsResponse into a list of ACL objects and a KafkaError.
616
+
617
+ Arguments:
618
+ describe_response: The response object from the DescribeAclsRequest.
619
+
620
+ Returns:
621
+ A tuple of (list_of_acl_objects, error) where error is an instance
622
+ of KafkaError (NoError if successful).
623
+ """
535
624
  version = describe_response.API_VERSION
536
625
 
537
626
  error = Errors.for_code(describe_response.error_code)
@@ -571,8 +660,11 @@ class KafkaAdminClient(object):
571
660
  The cluster must be configured with an authorizer for this to work, or
572
661
  you will get a SecurityDisabledError
573
662
 
574
- :param acl_filter: an ACLFilter object
575
- :return: tuple of a list of matching ACL objects and a KafkaError (NoError if successful)
663
+ Arguments:
664
+ acl_filter: an ACLFilter object
665
+
666
+ Returns:
667
+ tuple of a list of matching ACL objects and a KafkaError (NoError if successful)
576
668
  """
577
669
 
578
670
  version = self._matching_api_version(DescribeAclsRequest)
@@ -617,6 +709,14 @@ class KafkaAdminClient(object):
617
709
 
618
710
  @staticmethod
619
711
  def _convert_create_acls_resource_request_v0(acl):
712
+ """Convert an ACL object into the CreateAclsRequest v0 format.
713
+
714
+ Arguments:
715
+ acl: An ACL object with resource pattern and permissions.
716
+
717
+ Returns:
718
+ A tuple: (resource_type, resource_name, principal, host, operation, permission_type).
719
+ """
620
720
 
621
721
  return (
622
722
  acl.resource_pattern.resource_type,
@@ -629,7 +729,14 @@ class KafkaAdminClient(object):
629
729
 
630
730
  @staticmethod
631
731
  def _convert_create_acls_resource_request_v1(acl):
732
+ """Convert an ACL object into the CreateAclsRequest v1 format.
632
733
 
734
+ Arguments:
735
+ acl: An ACL object with resource pattern and permissions.
736
+
737
+ Returns:
738
+ A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type).
739
+ """
633
740
  return (
634
741
  acl.resource_pattern.resource_type,
635
742
  acl.resource_pattern.resource_name,
@@ -642,6 +749,19 @@ class KafkaAdminClient(object):
642
749
 
643
750
  @staticmethod
644
751
  def _convert_create_acls_response_to_acls(acls, create_response):
752
+ """Parse CreateAclsResponse and correlate success/failure with original ACL objects.
753
+
754
+ Arguments:
755
+ acls: A list of ACL objects that were requested for creation.
756
+ create_response: The broker's CreateAclsResponse object.
757
+
758
+ Returns:
759
+ A dict with:
760
+ {
761
+ 'succeeded': [list of ACL objects successfully created],
762
+ 'failed': [(acl_object, KafkaError), ...]
763
+ }
764
+ """
645
765
  version = create_response.API_VERSION
646
766
 
647
767
  creations_error = []
@@ -670,8 +790,11 @@ class KafkaAdminClient(object):
670
790
  This endpoint only accepts a list of concrete ACL objects, no ACLFilters.
671
791
  Throws TopicAlreadyExistsError if topic is already present.
672
792
 
673
- :param acls: a list of ACL objects
674
- :return: dict of successes and failures
793
+ Arguments:
794
+ acls: a list of ACL objects
795
+
796
+ Returns:
797
+ dict of successes and failures
675
798
  """
676
799
 
677
800
  for acl in acls:
@@ -701,6 +824,14 @@ class KafkaAdminClient(object):
701
824
 
702
825
  @staticmethod
703
826
  def _convert_delete_acls_resource_request_v0(acl):
827
+ """Convert an ACLFilter object into the DeleteAclsRequest v0 format.
828
+
829
+ Arguments:
830
+ acl: An ACLFilter object identifying the ACLs to be deleted.
831
+
832
+ Returns:
833
+ A tuple: (resource_type, resource_name, principal, host, operation, permission_type).
834
+ """
704
835
  return (
705
836
  acl.resource_pattern.resource_type,
706
837
  acl.resource_pattern.resource_name,
@@ -712,6 +843,14 @@ class KafkaAdminClient(object):
712
843
 
713
844
  @staticmethod
714
845
  def _convert_delete_acls_resource_request_v1(acl):
846
+ """Convert an ACLFilter object into the DeleteAclsRequest v1 format.
847
+
848
+ Arguments:
849
+ acl: An ACLFilter object identifying the ACLs to be deleted.
850
+
851
+ Returns:
852
+ A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type).
853
+ """
715
854
  return (
716
855
  acl.resource_pattern.resource_type,
717
856
  acl.resource_pattern.resource_name,
@@ -724,6 +863,16 @@ class KafkaAdminClient(object):
724
863
 
725
864
  @staticmethod
726
865
  def _convert_delete_acls_response_to_matching_acls(acl_filters, delete_response):
866
+ """Parse the DeleteAclsResponse and map the results back to each input ACLFilter.
867
+
868
+ Arguments:
869
+ acl_filters: A list of ACLFilter objects that were provided in the request.
870
+ delete_response: The response from the DeleteAclsRequest.
871
+
872
+ Returns:
873
+ A list of tuples of the form:
874
+ (acl_filter, [(matching_acl, KafkaError), ...], filter_level_error).
875
+ """
727
876
  version = delete_response.API_VERSION
728
877
  filter_result_list = []
729
878
  for i, filter_responses in enumerate(delete_response.filter_responses):
@@ -762,8 +911,11 @@ class KafkaAdminClient(object):
762
911
 
763
912
  Deletes all ACLs matching the list of input ACLFilter
764
913
 
765
- :param acl_filters: a list of ACLFilter
766
- :return: a list of 3-tuples corresponding to the list of input filters.
914
+ Arguments:
915
+ acl_filters: a list of ACLFilter
916
+
917
+ Returns:
918
+ a list of 3-tuples corresponding to the list of input filters.
767
919
  The tuples hold (the input ACLFilter, list of affected ACLs, KafkaError instance)
768
920
  """
769
921
 
@@ -795,6 +947,14 @@ class KafkaAdminClient(object):
795
947
 
796
948
  @staticmethod
797
949
  def _convert_describe_config_resource_request(config_resource):
950
+ """Convert a ConfigResource into the format required by DescribeConfigsRequest.
951
+
952
+ Arguments:
953
+ config_resource: A ConfigResource with resource_type, name, and optional config keys.
954
+
955
+ Returns:
956
+ A tuple: (resource_type, resource_name, [list_of_config_keys] or None).
957
+ """
798
958
  return (
799
959
  config_resource.resource_type,
800
960
  config_resource.name,
@@ -806,13 +966,18 @@ class KafkaAdminClient(object):
806
966
  def describe_configs(self, config_resources, include_synonyms=False):
807
967
  """Fetch configuration parameters for one or more Kafka resources.
808
968
 
809
- :param config_resources: An list of ConfigResource objects.
810
- Any keys in ConfigResource.configs dict will be used to filter the
811
- result. Setting the configs dict to None will get all values. An
812
- empty dict will get zero values (as per Kafka protocol).
813
- :param include_synonyms: If True, return synonyms in response. Not
814
- supported by all versions. Default: False.
815
- :return: Appropriate version of DescribeConfigsResponse class.
969
+ Arguments:
970
+ config_resources: An list of ConfigResource objects.
971
+ Any keys in ConfigResource.configs dict will be used to filter the
972
+ result. Setting the configs dict to None will get all values. An
973
+ empty dict will get zero values (as per Kafka protocol).
974
+
975
+ Keyword Arguments:
976
+ include_synonyms (bool, optional): If True, return synonyms in response. Not
977
+ supported by all versions. Default: False.
978
+
979
+ Returns:
980
+ Appropriate version of DescribeConfigsResponse class.
816
981
  """
817
982
 
818
983
  # Break up requests by type - a broker config request must be sent to the specific broker.
@@ -881,6 +1046,14 @@ class KafkaAdminClient(object):
881
1046
 
882
1047
  @staticmethod
883
1048
  def _convert_alter_config_resource_request(config_resource):
1049
+ """Convert a ConfigResource into the format required by AlterConfigsRequest.
1050
+
1051
+ Arguments:
1052
+ config_resource: A ConfigResource with resource_type, name, and config (key, value) pairs.
1053
+
1054
+ Returns:
1055
+ A tuple: (resource_type, resource_name, [(config_key, config_value), ...]).
1056
+ """
884
1057
  return (
885
1058
  config_resource.resource_type,
886
1059
  config_resource.name,
@@ -898,8 +1071,11 @@ class KafkaAdminClient(object):
898
1071
  least-loaded node. See the comment in the source code for details.
899
1072
  We would happily accept a PR fixing this.
900
1073
 
901
- :param config_resources: A list of ConfigResource objects.
902
- :return: Appropriate version of AlterConfigsResponse class.
1074
+ Arguments:
1075
+ config_resources: A list of ConfigResource objects.
1076
+
1077
+ Returns:
1078
+ Appropriate version of AlterConfigsResponse class.
903
1079
  """
904
1080
  version = self._matching_api_version(AlterConfigsRequest)
905
1081
  if version <= 1:
@@ -930,6 +1106,15 @@ class KafkaAdminClient(object):
930
1106
 
931
1107
  @staticmethod
932
1108
  def _convert_create_partitions_request(topic_name, new_partitions):
1109
+ """Convert a NewPartitions object into the tuple format for CreatePartitionsRequest.
1110
+
1111
+ Arguments:
1112
+ topic_name: The name of the existing topic.
1113
+ new_partitions: A NewPartitions instance with total_count and new_assignments.
1114
+
1115
+ Returns:
1116
+ A tuple: (topic_name, (total_count, [list_of_assignments])).
1117
+ """
933
1118
  return (
934
1119
  topic_name,
935
1120
  (
@@ -941,12 +1126,17 @@ class KafkaAdminClient(object):
941
1126
  def create_partitions(self, topic_partitions, timeout_ms=None, validate_only=False):
942
1127
  """Create additional partitions for an existing topic.
943
1128
 
944
- :param topic_partitions: A map of topic name strings to NewPartition objects.
945
- :param timeout_ms: Milliseconds to wait for new partitions to be
946
- created before the broker returns.
947
- :param validate_only: If True, don't actually create new partitions.
948
- Default: False
949
- :return: Appropriate version of CreatePartitionsResponse class.
1129
+ Arguments:
1130
+ topic_partitions: A map of topic name strings to NewPartition objects.
1131
+
1132
+ Keyword Arguments:
1133
+ timeout_ms (numeric, optional): Milliseconds to wait for new partitions to be
1134
+ created before the broker returns.
1135
+ validate_only (bool, optional): If True, don't actually create new partitions.
1136
+ Default: False
1137
+
1138
+ Returns:
1139
+ Appropriate version of CreatePartitionsResponse class.
950
1140
  """
951
1141
  version = self._matching_api_version(CreatePartitionsRequest)
952
1142
  timeout_ms = self._validate_timeout(timeout_ms)
@@ -980,10 +1170,12 @@ class KafkaAdminClient(object):
980
1170
  def _describe_consumer_groups_send_request(self, group_id, group_coordinator_id, include_authorized_operations=False):
981
1171
  """Send a DescribeGroupsRequest to the group's coordinator.
982
1172
 
983
- :param group_id: The group name as a string
984
- :param group_coordinator_id: The node_id of the groups' coordinator
985
- broker.
986
- :return: A message future.
1173
+ Arguments:
1174
+ group_id: The group name as a string
1175
+ group_coordinator_id: The node_id of the groups' coordinator broker.
1176
+
1177
+ Returns:
1178
+ A message future.
987
1179
  """
988
1180
  version = self._matching_api_version(DescribeGroupsRequest)
989
1181
  if version <= 2:
@@ -1066,18 +1258,23 @@ class KafkaAdminClient(object):
1066
1258
 
1067
1259
  Any errors are immediately raised.
1068
1260
 
1069
- :param group_ids: A list of consumer group IDs. These are typically the
1070
- group names as strings.
1071
- :param group_coordinator_id: The node_id of the groups' coordinator
1072
- broker. If set to None, it will query the cluster for each group to
1073
- find that group's coordinator. Explicitly specifying this can be
1074
- useful for avoiding extra network round trips if you already know
1075
- the group coordinator. This is only useful when all the group_ids
1076
- have the same coordinator, otherwise it will error. Default: None.
1077
- :param include_authorized_operations: Whether or not to include
1078
- information about the operations a group is allowed to perform.
1079
- Only supported on API version >= v3. Default: False.
1080
- :return: A list of group descriptions. For now the group descriptions
1261
+ Arguments:
1262
+ group_ids: A list of consumer group IDs. These are typically the
1263
+ group names as strings.
1264
+
1265
+ Keyword Arguments:
1266
+ group_coordinator_id (int, optional): The node_id of the groups' coordinator
1267
+ broker. If set to None, it will query the cluster for each group to
1268
+ find that group's coordinator. Explicitly specifying this can be
1269
+ useful for avoiding extra network round trips if you already know
1270
+ the group coordinator. This is only useful when all the group_ids
1271
+ have the same coordinator, otherwise it will error. Default: None.
1272
+ include_authorized_operations (bool, optional): Whether or not to include
1273
+ information about the operations a group is allowed to perform.
1274
+ Only supported on API version >= v3. Default: False.
1275
+
1276
+ Returns:
1277
+ A list of group descriptions. For now the group descriptions
1081
1278
  are the raw results from the DescribeGroupsResponse. Long-term, we
1082
1279
  plan to change this to return namedtuples as well as decoding the
1083
1280
  partition assignments.
@@ -1108,8 +1305,11 @@ class KafkaAdminClient(object):
1108
1305
  def _list_consumer_groups_send_request(self, broker_id):
1109
1306
  """Send a ListGroupsRequest to a broker.
1110
1307
 
1111
- :param broker_id: The broker's node_id.
1112
- :return: A message future
1308
+ Arguments:
1309
+ broker_id (int): The broker's node_id.
1310
+
1311
+ Returns:
1312
+ A message future
1113
1313
  """
1114
1314
  version = self._matching_api_version(ListGroupsRequest)
1115
1315
  if version <= 2:
@@ -1149,15 +1349,20 @@ class KafkaAdminClient(object):
1149
1349
 
1150
1350
  As soon as any error is encountered, it is immediately raised.
1151
1351
 
1152
- :param broker_ids: A list of broker node_ids to query for consumer
1153
- groups. If set to None, will query all brokers in the cluster.
1154
- Explicitly specifying broker(s) can be useful for determining which
1155
- consumer groups are coordinated by those broker(s). Default: None
1156
- :return list: List of tuples of Consumer Groups.
1157
- :exception GroupCoordinatorNotAvailableError: The coordinator is not
1158
- available, so cannot process requests.
1159
- :exception GroupLoadInProgressError: The coordinator is loading and
1160
- hence can't process requests.
1352
+ Keyword Arguments:
1353
+ broker_ids ([int], optional): A list of broker node_ids to query for consumer
1354
+ groups. If set to None, will query all brokers in the cluster.
1355
+ Explicitly specifying broker(s) can be useful for determining which
1356
+ consumer groups are coordinated by those broker(s). Default: None
1357
+
1358
+ Returns:
1359
+ list: List of tuples of Consumer Groups.
1360
+
1361
+ Raises:
1362
+ GroupCoordinatorNotAvailableError: The coordinator is not
1363
+ available, so cannot process requests.
1364
+ GroupLoadInProgressError: The coordinator is loading and
1365
+ hence can't process requests.
1161
1366
  """
1162
1367
  # While we return a list, internally use a set to prevent duplicates
1163
1368
  # because if a group coordinator fails after being queried, and its
@@ -1177,10 +1382,17 @@ class KafkaAdminClient(object):
1177
1382
  group_coordinator_id, partitions=None):
1178
1383
  """Send an OffsetFetchRequest to a broker.
1179
1384
 
1180
- :param group_id: The consumer group id name for which to fetch offsets.
1181
- :param group_coordinator_id: The node_id of the group's coordinator
1182
- broker.
1183
- :return: A message future
1385
+ Arguments:
1386
+ group_id (str): The consumer group id name for which to fetch offsets.
1387
+ group_coordinator_id (int): The node_id of the group's coordinator broker.
1388
+
1389
+ Keyword Arguments:
1390
+ partitions: A list of TopicPartitions for which to fetch
1391
+ offsets. On brokers >= 0.10.2, this can be set to None to fetch all
1392
+ known offsets for the consumer group. Default: None.
1393
+
1394
+ Returns:
1395
+ A message future
1184
1396
  """
1185
1397
  version = self._matching_api_version(OffsetFetchRequest)
1186
1398
  if version <= 3:
@@ -1208,8 +1420,11 @@ class KafkaAdminClient(object):
1208
1420
  def _list_consumer_group_offsets_process_response(self, response):
1209
1421
  """Process an OffsetFetchResponse.
1210
1422
 
1211
- :param response: an OffsetFetchResponse.
1212
- :return: A dictionary composed of TopicPartition keys and
1423
+ Arguments:
1424
+ response: an OffsetFetchResponse.
1425
+
1426
+ Returns:
1427
+ A dictionary composed of TopicPartition keys and
1213
1428
  OffsetAndMetadata values.
1214
1429
  """
1215
1430
  if response.API_VERSION <= 3:
@@ -1250,16 +1465,21 @@ class KafkaAdminClient(object):
1250
1465
 
1251
1466
  As soon as any error is encountered, it is immediately raised.
1252
1467
 
1253
- :param group_id: The consumer group id name for which to fetch offsets.
1254
- :param group_coordinator_id: The node_id of the group's coordinator
1255
- broker. If set to None, will query the cluster to find the group
1256
- coordinator. Explicitly specifying this can be useful to prevent
1257
- that extra network round trip if you already know the group
1258
- coordinator. Default: None.
1259
- :param partitions: A list of TopicPartitions for which to fetch
1260
- offsets. On brokers >= 0.10.2, this can be set to None to fetch all
1261
- known offsets for the consumer group. Default: None.
1262
- :return dictionary: A dictionary with TopicPartition keys and
1468
+ Arguments:
1469
+ group_id (str): The consumer group id name for which to fetch offsets.
1470
+
1471
+ Keyword Arguments:
1472
+ group_coordinator_id (int, optional): The node_id of the group's coordinator
1473
+ broker. If set to None, will query the cluster to find the group
1474
+ coordinator. Explicitly specifying this can be useful to prevent
1475
+ that extra network round trip if you already know the group
1476
+ coordinator. Default: None.
1477
+ partitions: A list of TopicPartitions for which to fetch
1478
+ offsets. On brokers >= 0.10.2, this can be set to None to fetch all
1479
+ known offsets for the consumer group. Default: None.
1480
+
1481
+ Returns:
1482
+ dictionary: A dictionary with TopicPartition keys and
1263
1483
  OffsetAndMetada values. Partitions that are not specified and for
1264
1484
  which the group_id does not have a recorded offset are omitted. An
1265
1485
  offset value of `-1` indicates the group_id has no offset for that
@@ -1283,14 +1503,19 @@ class KafkaAdminClient(object):
1283
1503
 
1284
1504
  The result needs checking for potential errors.
1285
1505
 
1286
- :param group_ids: The consumer group ids of the groups which are to be deleted.
1287
- :param group_coordinator_id: The node_id of the broker which is the coordinator for
1288
- all the groups. Use only if all groups are coordinated by the same broker.
1289
- If set to None, will query the cluster to find the coordinator for every single group.
1290
- Explicitly specifying this can be useful to prevent
1291
- that extra network round trips if you already know the group
1292
- coordinator. Default: None.
1293
- :return: A list of tuples (group_id, KafkaError)
1506
+ Arguments:
1507
+ group_ids ([str]): The consumer group ids of the groups which are to be deleted.
1508
+
1509
+ Keyword Arguments:
1510
+ group_coordinator_id (int, optional): The node_id of the broker which is
1511
+ the coordinator for all the groups. Use only if all groups are coordinated
1512
+ by the same broker. If set to None, will query the cluster to find the coordinator
1513
+ for every single group. Explicitly specifying this can be useful to prevent
1514
+ that extra network round trips if you already know the group coordinator.
1515
+ Default: None.
1516
+
1517
+ Returns:
1518
+ A list of tuples (group_id, KafkaError)
1294
1519
  """
1295
1520
  if group_coordinator_id is not None:
1296
1521
  futures = [self._delete_consumer_groups_send_request(group_ids, group_coordinator_id)]
@@ -1311,6 +1536,14 @@ class KafkaAdminClient(object):
1311
1536
  return results
1312
1537
 
1313
1538
  def _convert_delete_groups_response(self, response):
1539
+ """Parse the DeleteGroupsResponse, mapping group IDs to their respective errors.
1540
+
1541
+ Arguments:
1542
+ response: A DeleteGroupsResponse object from the broker.
1543
+
1544
+ Returns:
1545
+ A list of (group_id, KafkaError) for each deleted group.
1546
+ """
1314
1547
  if response.API_VERSION <= 1:
1315
1548
  results = []
1316
1549
  for group_id, error_code in response.results:
@@ -1322,12 +1555,14 @@ class KafkaAdminClient(object):
1322
1555
  .format(response.API_VERSION))
1323
1556
 
1324
1557
  def _delete_consumer_groups_send_request(self, group_ids, group_coordinator_id):
1325
- """Send a DeleteGroups request to a broker.
1558
+ """Send a DeleteGroupsRequest to the specified broker (the group coordinator).
1559
+
1560
+ Arguments:
1561
+ group_ids ([str]): A list of consumer group IDs to be deleted.
1562
+ group_coordinator_id (int): The node_id of the broker coordinating these groups.
1326
1563
 
1327
- :param group_ids: The consumer group ids of the groups which are to be deleted.
1328
- :param group_coordinator_id: The node_id of the broker which is the coordinator for
1329
- all the groups.
1330
- :return: A message future
1564
+ Returns:
1565
+ A future representing the in-flight DeleteGroupsRequest.
1331
1566
  """
1332
1567
  version = self._matching_api_version(DeleteGroupsRequest)
1333
1568
  if version <= 1:
@@ -1339,6 +1574,14 @@ class KafkaAdminClient(object):
1339
1574
  return self._send_request_to_node(group_coordinator_id, request)
1340
1575
 
1341
1576
  def _wait_for_futures(self, futures):
1577
+ """Block until all futures complete. If any fail, raise the encountered exception.
1578
+
1579
+ Arguments:
1580
+ futures: A list of Future objects awaiting results.
1581
+
1582
+ Raises:
1583
+ The first encountered exception if a future fails.
1584
+ """
1342
1585
  while not all(future.succeeded() for future in futures):
1343
1586
  for future in futures:
1344
1587
  self._client.poll(future=future)
@@ -1349,7 +1592,8 @@ class KafkaAdminClient(object):
1349
1592
  def describe_log_dirs(self):
1350
1593
  """Send a DescribeLogDirsRequest request to a broker.
1351
1594
 
1352
- :return: A message future
1595
+ Returns:
1596
+ A message future
1353
1597
  """
1354
1598
  version = self._matching_api_version(DescribeLogDirsRequest)
1355
1599
  if version <= 0: