kafka-python 2.0.3__py2.py3-none-any.whl → 2.0.4__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kafka/admin/client.py +340 -108
- kafka/client_async.py +81 -37
- kafka/cluster.py +1 -1
- kafka/codec.py +9 -2
- kafka/conn.py +36 -22
- kafka/consumer/group.py +2 -2
- kafka/coordinator/base.py +9 -0
- kafka/errors.py +587 -28
- kafka/producer/kafka.py +3 -3
- kafka/producer/sender.py +6 -6
- kafka/version.py +1 -1
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.4.dist-info}/METADATA +13 -13
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.4.dist-info}/RECORD +15 -16
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.4.dist-info}/WHEEL +1 -1
- kafka_python-2.0.3.dist-info/LICENSE +0 -202
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.4.dist-info}/top_level.txt +0 -0
kafka/admin/client.py
CHANGED
|
@@ -72,7 +72,7 @@ class KafkaAdminClient(object):
|
|
|
72
72
|
reconnection attempts will continue periodically with this fixed
|
|
73
73
|
rate. To avoid connection storms, a randomization factor of 0.2
|
|
74
74
|
will be applied to the backoff resulting in a random range between
|
|
75
|
-
20% below and 20% above the computed value. Default:
|
|
75
|
+
20% below and 20% above the computed value. Default: 30000.
|
|
76
76
|
request_timeout_ms (int): Client request timeout in milliseconds.
|
|
77
77
|
Default: 30000.
|
|
78
78
|
connections_max_idle_ms: Close idle connections after the number of
|
|
@@ -156,7 +156,7 @@ class KafkaAdminClient(object):
|
|
|
156
156
|
'request_timeout_ms': 30000,
|
|
157
157
|
'connections_max_idle_ms': 9 * 60 * 1000,
|
|
158
158
|
'reconnect_backoff_ms': 50,
|
|
159
|
-
'reconnect_backoff_max_ms':
|
|
159
|
+
'reconnect_backoff_max_ms': 30000,
|
|
160
160
|
'max_in_flight_requests_per_connection': 5,
|
|
161
161
|
'receive_buffer_bytes': None,
|
|
162
162
|
'send_buffer_bytes': None,
|
|
@@ -240,8 +240,11 @@ class KafkaAdminClient(object):
|
|
|
240
240
|
This resolves to the lesser of either the latest api version this
|
|
241
241
|
library supports, or the max version supported by the broker.
|
|
242
242
|
|
|
243
|
-
:
|
|
244
|
-
|
|
243
|
+
Arguments:
|
|
244
|
+
operation: A list of protocol operation versions from kafka.protocol.
|
|
245
|
+
|
|
246
|
+
Returns:
|
|
247
|
+
int: The max matching version number between client and broker.
|
|
245
248
|
"""
|
|
246
249
|
broker_api_versions = self._client.get_api_versions()
|
|
247
250
|
api_key = operation[0].API_KEY
|
|
@@ -262,8 +265,11 @@ class KafkaAdminClient(object):
|
|
|
262
265
|
def _validate_timeout(self, timeout_ms):
|
|
263
266
|
"""Validate the timeout is set or use the configuration default.
|
|
264
267
|
|
|
265
|
-
:
|
|
266
|
-
|
|
268
|
+
Arguments:
|
|
269
|
+
timeout_ms: The timeout provided by api call, in milliseconds.
|
|
270
|
+
|
|
271
|
+
Returns:
|
|
272
|
+
The timeout to use for the operation.
|
|
267
273
|
"""
|
|
268
274
|
return timeout_ms or self.config['request_timeout_ms']
|
|
269
275
|
|
|
@@ -293,9 +299,12 @@ class KafkaAdminClient(object):
|
|
|
293
299
|
def _find_coordinator_id_send_request(self, group_id):
|
|
294
300
|
"""Send a FindCoordinatorRequest to a broker.
|
|
295
301
|
|
|
296
|
-
:
|
|
302
|
+
Arguments:
|
|
303
|
+
group_id: The consumer group ID. This is typically the group
|
|
297
304
|
name as a string.
|
|
298
|
-
|
|
305
|
+
|
|
306
|
+
Returns:
|
|
307
|
+
A message future
|
|
299
308
|
"""
|
|
300
309
|
# TODO add support for dynamically picking version of
|
|
301
310
|
# GroupCoordinatorRequest which was renamed to FindCoordinatorRequest.
|
|
@@ -315,8 +324,11 @@ class KafkaAdminClient(object):
|
|
|
315
324
|
def _find_coordinator_id_process_response(self, response):
|
|
316
325
|
"""Process a FindCoordinatorResponse.
|
|
317
326
|
|
|
318
|
-
:
|
|
319
|
-
|
|
327
|
+
Arguments:
|
|
328
|
+
response: a FindCoordinatorResponse.
|
|
329
|
+
|
|
330
|
+
Returns:
|
|
331
|
+
The node_id of the broker that is the coordinator.
|
|
320
332
|
"""
|
|
321
333
|
if response.API_VERSION <= 0:
|
|
322
334
|
error_type = Errors.for_code(response.error_code)
|
|
@@ -339,9 +351,12 @@ class KafkaAdminClient(object):
|
|
|
339
351
|
Will block until the FindCoordinatorResponse is received for all groups.
|
|
340
352
|
Any errors are immediately raised.
|
|
341
353
|
|
|
342
|
-
:
|
|
354
|
+
Arguments:
|
|
355
|
+
group_ids: A list of consumer group IDs. This is typically the group
|
|
343
356
|
name as a string.
|
|
344
|
-
|
|
357
|
+
|
|
358
|
+
Returns:
|
|
359
|
+
A dict of {group_id: node_id} where node_id is the id of the
|
|
345
360
|
broker that is the coordinator for the corresponding group.
|
|
346
361
|
"""
|
|
347
362
|
groups_futures = {
|
|
@@ -358,13 +373,19 @@ class KafkaAdminClient(object):
|
|
|
358
373
|
def _send_request_to_node(self, node_id, request, wakeup=True):
|
|
359
374
|
"""Send a Kafka protocol message to a specific broker.
|
|
360
375
|
|
|
361
|
-
|
|
376
|
+
Arguments:
|
|
377
|
+
node_id: The broker id to which to send the message.
|
|
378
|
+
request: The message to send.
|
|
362
379
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
:
|
|
380
|
+
|
|
381
|
+
Keyword Arguments:
|
|
382
|
+
wakeup (bool, optional): Optional flag to disable thread-wakeup.
|
|
383
|
+
|
|
384
|
+
Returns:
|
|
385
|
+
A future object that may be polled for status and results.
|
|
386
|
+
|
|
387
|
+
Raises:
|
|
388
|
+
The exception if the message could not be sent.
|
|
368
389
|
"""
|
|
369
390
|
while not self._client.ready(node_id):
|
|
370
391
|
# poll until the connection to broker is ready, otherwise send()
|
|
@@ -377,8 +398,11 @@ class KafkaAdminClient(object):
|
|
|
377
398
|
|
|
378
399
|
Will block until the message result is received.
|
|
379
400
|
|
|
380
|
-
:
|
|
381
|
-
|
|
401
|
+
Arguments:
|
|
402
|
+
request: The message to send.
|
|
403
|
+
|
|
404
|
+
Returns:
|
|
405
|
+
The Kafka protocol response for the message.
|
|
382
406
|
"""
|
|
383
407
|
tries = 2 # in case our cached self._controller_id is outdated
|
|
384
408
|
while tries:
|
|
@@ -418,6 +442,18 @@ class KafkaAdminClient(object):
|
|
|
418
442
|
|
|
419
443
|
@staticmethod
|
|
420
444
|
def _convert_new_topic_request(new_topic):
|
|
445
|
+
"""
|
|
446
|
+
Build the tuple required by CreateTopicsRequest from a NewTopic object.
|
|
447
|
+
|
|
448
|
+
Arguments:
|
|
449
|
+
new_topic: A NewTopic instance containing name, partition count, replication factor,
|
|
450
|
+
replica assignments, and config entries.
|
|
451
|
+
|
|
452
|
+
Returns:
|
|
453
|
+
A tuple in the form:
|
|
454
|
+
(topic_name, num_partitions, replication_factor, [(partition_id, [replicas])...],
|
|
455
|
+
[(config_key, config_value)...])
|
|
456
|
+
"""
|
|
421
457
|
return (
|
|
422
458
|
new_topic.name,
|
|
423
459
|
new_topic.num_partitions,
|
|
@@ -433,12 +469,17 @@ class KafkaAdminClient(object):
|
|
|
433
469
|
def create_topics(self, new_topics, timeout_ms=None, validate_only=False):
|
|
434
470
|
"""Create new topics in the cluster.
|
|
435
471
|
|
|
436
|
-
:
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
472
|
+
Arguments:
|
|
473
|
+
new_topics: A list of NewTopic objects.
|
|
474
|
+
|
|
475
|
+
Keyword Arguments:
|
|
476
|
+
timeout_ms (numeric, optional): Milliseconds to wait for new topics to be created
|
|
477
|
+
before the broker returns.
|
|
478
|
+
validate_only (bool, optional): If True, don't actually create new topics.
|
|
479
|
+
Not supported by all versions. Default: False
|
|
480
|
+
|
|
481
|
+
Returns:
|
|
482
|
+
Appropriate version of CreateTopicResponse class.
|
|
442
483
|
"""
|
|
443
484
|
version = self._matching_api_version(CreateTopicsRequest)
|
|
444
485
|
timeout_ms = self._validate_timeout(timeout_ms)
|
|
@@ -468,10 +509,15 @@ class KafkaAdminClient(object):
|
|
|
468
509
|
def delete_topics(self, topics, timeout_ms=None):
|
|
469
510
|
"""Delete topics from the cluster.
|
|
470
511
|
|
|
471
|
-
:
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
:
|
|
512
|
+
Arguments:
|
|
513
|
+
topics ([str]): A list of topic name strings.
|
|
514
|
+
|
|
515
|
+
Keyword Arguments:
|
|
516
|
+
timeout_ms (numeric, optional): Milliseconds to wait for topics to be deleted
|
|
517
|
+
before the broker returns.
|
|
518
|
+
|
|
519
|
+
Returns:
|
|
520
|
+
Appropriate version of DeleteTopicsResponse class.
|
|
475
521
|
"""
|
|
476
522
|
version = self._matching_api_version(DeleteTopicsRequest)
|
|
477
523
|
timeout_ms = self._validate_timeout(timeout_ms)
|
|
@@ -515,16 +561,38 @@ class KafkaAdminClient(object):
|
|
|
515
561
|
return future.value
|
|
516
562
|
|
|
517
563
|
def list_topics(self):
|
|
564
|
+
"""Retrieve a list of all topic names in the cluster.
|
|
565
|
+
|
|
566
|
+
Returns:
|
|
567
|
+
A list of topic name strings.
|
|
568
|
+
"""
|
|
518
569
|
metadata = self._get_cluster_metadata(topics=None)
|
|
519
570
|
obj = metadata.to_object()
|
|
520
571
|
return [t['topic'] for t in obj['topics']]
|
|
521
572
|
|
|
522
573
|
def describe_topics(self, topics=None):
|
|
574
|
+
"""Fetch metadata for the specified topics or all topics if None.
|
|
575
|
+
|
|
576
|
+
Keyword Arguments:
|
|
577
|
+
topics ([str], optional) A list of topic names. If None, metadata for all
|
|
578
|
+
topics is retrieved.
|
|
579
|
+
|
|
580
|
+
Returns:
|
|
581
|
+
A list of dicts describing each topic (including partition info).
|
|
582
|
+
"""
|
|
523
583
|
metadata = self._get_cluster_metadata(topics=topics)
|
|
524
584
|
obj = metadata.to_object()
|
|
525
585
|
return obj['topics']
|
|
526
586
|
|
|
527
587
|
def describe_cluster(self):
|
|
588
|
+
"""
|
|
589
|
+
Fetch cluster-wide metadata such as the list of brokers, the controller ID,
|
|
590
|
+
and the cluster ID.
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
Returns:
|
|
594
|
+
A dict with cluster-wide metadata, excluding topic details.
|
|
595
|
+
"""
|
|
528
596
|
metadata = self._get_cluster_metadata()
|
|
529
597
|
obj = metadata.to_object()
|
|
530
598
|
obj.pop('topics') # We have 'describe_topics' for this
|
|
@@ -532,6 +600,15 @@ class KafkaAdminClient(object):
|
|
|
532
600
|
|
|
533
601
|
@staticmethod
|
|
534
602
|
def _convert_describe_acls_response_to_acls(describe_response):
|
|
603
|
+
"""Convert a DescribeAclsResponse into a list of ACL objects and a KafkaError.
|
|
604
|
+
|
|
605
|
+
Arguments:
|
|
606
|
+
describe_response: The response object from the DescribeAclsRequest.
|
|
607
|
+
|
|
608
|
+
Returns:
|
|
609
|
+
A tuple of (list_of_acl_objects, error) where error is an instance
|
|
610
|
+
of KafkaError (NoError if successful).
|
|
611
|
+
"""
|
|
535
612
|
version = describe_response.API_VERSION
|
|
536
613
|
|
|
537
614
|
error = Errors.for_code(describe_response.error_code)
|
|
@@ -571,8 +648,11 @@ class KafkaAdminClient(object):
|
|
|
571
648
|
The cluster must be configured with an authorizer for this to work, or
|
|
572
649
|
you will get a SecurityDisabledError
|
|
573
650
|
|
|
574
|
-
:
|
|
575
|
-
|
|
651
|
+
Arguments:
|
|
652
|
+
acl_filter: an ACLFilter object
|
|
653
|
+
|
|
654
|
+
Returns:
|
|
655
|
+
tuple of a list of matching ACL objects and a KafkaError (NoError if successful)
|
|
576
656
|
"""
|
|
577
657
|
|
|
578
658
|
version = self._matching_api_version(DescribeAclsRequest)
|
|
@@ -617,6 +697,14 @@ class KafkaAdminClient(object):
|
|
|
617
697
|
|
|
618
698
|
@staticmethod
|
|
619
699
|
def _convert_create_acls_resource_request_v0(acl):
|
|
700
|
+
"""Convert an ACL object into the CreateAclsRequest v0 format.
|
|
701
|
+
|
|
702
|
+
Arguments:
|
|
703
|
+
acl: An ACL object with resource pattern and permissions.
|
|
704
|
+
|
|
705
|
+
Returns:
|
|
706
|
+
A tuple: (resource_type, resource_name, principal, host, operation, permission_type).
|
|
707
|
+
"""
|
|
620
708
|
|
|
621
709
|
return (
|
|
622
710
|
acl.resource_pattern.resource_type,
|
|
@@ -629,7 +717,14 @@ class KafkaAdminClient(object):
|
|
|
629
717
|
|
|
630
718
|
@staticmethod
|
|
631
719
|
def _convert_create_acls_resource_request_v1(acl):
|
|
720
|
+
"""Convert an ACL object into the CreateAclsRequest v1 format.
|
|
632
721
|
|
|
722
|
+
Arguments:
|
|
723
|
+
acl: An ACL object with resource pattern and permissions.
|
|
724
|
+
|
|
725
|
+
Returns:
|
|
726
|
+
A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type).
|
|
727
|
+
"""
|
|
633
728
|
return (
|
|
634
729
|
acl.resource_pattern.resource_type,
|
|
635
730
|
acl.resource_pattern.resource_name,
|
|
@@ -642,6 +737,19 @@ class KafkaAdminClient(object):
|
|
|
642
737
|
|
|
643
738
|
@staticmethod
|
|
644
739
|
def _convert_create_acls_response_to_acls(acls, create_response):
|
|
740
|
+
"""Parse CreateAclsResponse and correlate success/failure with original ACL objects.
|
|
741
|
+
|
|
742
|
+
Arguments:
|
|
743
|
+
acls: A list of ACL objects that were requested for creation.
|
|
744
|
+
create_response: The broker's CreateAclsResponse object.
|
|
745
|
+
|
|
746
|
+
Returns:
|
|
747
|
+
A dict with:
|
|
748
|
+
{
|
|
749
|
+
'succeeded': [list of ACL objects successfully created],
|
|
750
|
+
'failed': [(acl_object, KafkaError), ...]
|
|
751
|
+
}
|
|
752
|
+
"""
|
|
645
753
|
version = create_response.API_VERSION
|
|
646
754
|
|
|
647
755
|
creations_error = []
|
|
@@ -670,8 +778,11 @@ class KafkaAdminClient(object):
|
|
|
670
778
|
This endpoint only accepts a list of concrete ACL objects, no ACLFilters.
|
|
671
779
|
Throws TopicAlreadyExistsError if topic is already present.
|
|
672
780
|
|
|
673
|
-
:
|
|
674
|
-
|
|
781
|
+
Arguments:
|
|
782
|
+
acls: a list of ACL objects
|
|
783
|
+
|
|
784
|
+
Returns:
|
|
785
|
+
dict of successes and failures
|
|
675
786
|
"""
|
|
676
787
|
|
|
677
788
|
for acl in acls:
|
|
@@ -701,6 +812,14 @@ class KafkaAdminClient(object):
|
|
|
701
812
|
|
|
702
813
|
@staticmethod
|
|
703
814
|
def _convert_delete_acls_resource_request_v0(acl):
|
|
815
|
+
"""Convert an ACLFilter object into the DeleteAclsRequest v0 format.
|
|
816
|
+
|
|
817
|
+
Arguments:
|
|
818
|
+
acl: An ACLFilter object identifying the ACLs to be deleted.
|
|
819
|
+
|
|
820
|
+
Returns:
|
|
821
|
+
A tuple: (resource_type, resource_name, principal, host, operation, permission_type).
|
|
822
|
+
"""
|
|
704
823
|
return (
|
|
705
824
|
acl.resource_pattern.resource_type,
|
|
706
825
|
acl.resource_pattern.resource_name,
|
|
@@ -712,6 +831,14 @@ class KafkaAdminClient(object):
|
|
|
712
831
|
|
|
713
832
|
@staticmethod
|
|
714
833
|
def _convert_delete_acls_resource_request_v1(acl):
|
|
834
|
+
"""Convert an ACLFilter object into the DeleteAclsRequest v1 format.
|
|
835
|
+
|
|
836
|
+
Arguments:
|
|
837
|
+
acl: An ACLFilter object identifying the ACLs to be deleted.
|
|
838
|
+
|
|
839
|
+
Returns:
|
|
840
|
+
A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type).
|
|
841
|
+
"""
|
|
715
842
|
return (
|
|
716
843
|
acl.resource_pattern.resource_type,
|
|
717
844
|
acl.resource_pattern.resource_name,
|
|
@@ -724,6 +851,16 @@ class KafkaAdminClient(object):
|
|
|
724
851
|
|
|
725
852
|
@staticmethod
|
|
726
853
|
def _convert_delete_acls_response_to_matching_acls(acl_filters, delete_response):
|
|
854
|
+
"""Parse the DeleteAclsResponse and map the results back to each input ACLFilter.
|
|
855
|
+
|
|
856
|
+
Arguments:
|
|
857
|
+
acl_filters: A list of ACLFilter objects that were provided in the request.
|
|
858
|
+
delete_response: The response from the DeleteAclsRequest.
|
|
859
|
+
|
|
860
|
+
Returns:
|
|
861
|
+
A list of tuples of the form:
|
|
862
|
+
(acl_filter, [(matching_acl, KafkaError), ...], filter_level_error).
|
|
863
|
+
"""
|
|
727
864
|
version = delete_response.API_VERSION
|
|
728
865
|
filter_result_list = []
|
|
729
866
|
for i, filter_responses in enumerate(delete_response.filter_responses):
|
|
@@ -762,8 +899,11 @@ class KafkaAdminClient(object):
|
|
|
762
899
|
|
|
763
900
|
Deletes all ACLs matching the list of input ACLFilter
|
|
764
901
|
|
|
765
|
-
:
|
|
766
|
-
|
|
902
|
+
Arguments:
|
|
903
|
+
acl_filters: a list of ACLFilter
|
|
904
|
+
|
|
905
|
+
Returns:
|
|
906
|
+
a list of 3-tuples corresponding to the list of input filters.
|
|
767
907
|
The tuples hold (the input ACLFilter, list of affected ACLs, KafkaError instance)
|
|
768
908
|
"""
|
|
769
909
|
|
|
@@ -795,6 +935,14 @@ class KafkaAdminClient(object):
|
|
|
795
935
|
|
|
796
936
|
@staticmethod
|
|
797
937
|
def _convert_describe_config_resource_request(config_resource):
|
|
938
|
+
"""Convert a ConfigResource into the format required by DescribeConfigsRequest.
|
|
939
|
+
|
|
940
|
+
Arguments:
|
|
941
|
+
config_resource: A ConfigResource with resource_type, name, and optional config keys.
|
|
942
|
+
|
|
943
|
+
Returns:
|
|
944
|
+
A tuple: (resource_type, resource_name, [list_of_config_keys] or None).
|
|
945
|
+
"""
|
|
798
946
|
return (
|
|
799
947
|
config_resource.resource_type,
|
|
800
948
|
config_resource.name,
|
|
@@ -806,13 +954,18 @@ class KafkaAdminClient(object):
|
|
|
806
954
|
def describe_configs(self, config_resources, include_synonyms=False):
|
|
807
955
|
"""Fetch configuration parameters for one or more Kafka resources.
|
|
808
956
|
|
|
809
|
-
:
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
:
|
|
957
|
+
Arguments:
|
|
958
|
+
config_resources: An list of ConfigResource objects.
|
|
959
|
+
Any keys in ConfigResource.configs dict will be used to filter the
|
|
960
|
+
result. Setting the configs dict to None will get all values. An
|
|
961
|
+
empty dict will get zero values (as per Kafka protocol).
|
|
962
|
+
|
|
963
|
+
Keyword Arguments:
|
|
964
|
+
include_synonyms (bool, optional): If True, return synonyms in response. Not
|
|
965
|
+
supported by all versions. Default: False.
|
|
966
|
+
|
|
967
|
+
Returns:
|
|
968
|
+
Appropriate version of DescribeConfigsResponse class.
|
|
816
969
|
"""
|
|
817
970
|
|
|
818
971
|
# Break up requests by type - a broker config request must be sent to the specific broker.
|
|
@@ -881,6 +1034,14 @@ class KafkaAdminClient(object):
|
|
|
881
1034
|
|
|
882
1035
|
@staticmethod
|
|
883
1036
|
def _convert_alter_config_resource_request(config_resource):
|
|
1037
|
+
"""Convert a ConfigResource into the format required by AlterConfigsRequest.
|
|
1038
|
+
|
|
1039
|
+
Arguments:
|
|
1040
|
+
config_resource: A ConfigResource with resource_type, name, and config (key, value) pairs.
|
|
1041
|
+
|
|
1042
|
+
Returns:
|
|
1043
|
+
A tuple: (resource_type, resource_name, [(config_key, config_value), ...]).
|
|
1044
|
+
"""
|
|
884
1045
|
return (
|
|
885
1046
|
config_resource.resource_type,
|
|
886
1047
|
config_resource.name,
|
|
@@ -898,8 +1059,11 @@ class KafkaAdminClient(object):
|
|
|
898
1059
|
least-loaded node. See the comment in the source code for details.
|
|
899
1060
|
We would happily accept a PR fixing this.
|
|
900
1061
|
|
|
901
|
-
:
|
|
902
|
-
|
|
1062
|
+
Arguments:
|
|
1063
|
+
config_resources: A list of ConfigResource objects.
|
|
1064
|
+
|
|
1065
|
+
Returns:
|
|
1066
|
+
Appropriate version of AlterConfigsResponse class.
|
|
903
1067
|
"""
|
|
904
1068
|
version = self._matching_api_version(AlterConfigsRequest)
|
|
905
1069
|
if version <= 1:
|
|
@@ -930,6 +1094,15 @@ class KafkaAdminClient(object):
|
|
|
930
1094
|
|
|
931
1095
|
@staticmethod
|
|
932
1096
|
def _convert_create_partitions_request(topic_name, new_partitions):
|
|
1097
|
+
"""Convert a NewPartitions object into the tuple format for CreatePartitionsRequest.
|
|
1098
|
+
|
|
1099
|
+
Arguments:
|
|
1100
|
+
topic_name: The name of the existing topic.
|
|
1101
|
+
new_partitions: A NewPartitions instance with total_count and new_assignments.
|
|
1102
|
+
|
|
1103
|
+
Returns:
|
|
1104
|
+
A tuple: (topic_name, (total_count, [list_of_assignments])).
|
|
1105
|
+
"""
|
|
933
1106
|
return (
|
|
934
1107
|
topic_name,
|
|
935
1108
|
(
|
|
@@ -941,12 +1114,17 @@ class KafkaAdminClient(object):
|
|
|
941
1114
|
def create_partitions(self, topic_partitions, timeout_ms=None, validate_only=False):
|
|
942
1115
|
"""Create additional partitions for an existing topic.
|
|
943
1116
|
|
|
944
|
-
:
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
1117
|
+
Arguments:
|
|
1118
|
+
topic_partitions: A map of topic name strings to NewPartition objects.
|
|
1119
|
+
|
|
1120
|
+
Keyword Arguments:
|
|
1121
|
+
timeout_ms (numeric, optional): Milliseconds to wait for new partitions to be
|
|
1122
|
+
created before the broker returns.
|
|
1123
|
+
validate_only (bool, optional): If True, don't actually create new partitions.
|
|
1124
|
+
Default: False
|
|
1125
|
+
|
|
1126
|
+
Returns:
|
|
1127
|
+
Appropriate version of CreatePartitionsResponse class.
|
|
950
1128
|
"""
|
|
951
1129
|
version = self._matching_api_version(CreatePartitionsRequest)
|
|
952
1130
|
timeout_ms = self._validate_timeout(timeout_ms)
|
|
@@ -980,10 +1158,12 @@ class KafkaAdminClient(object):
|
|
|
980
1158
|
def _describe_consumer_groups_send_request(self, group_id, group_coordinator_id, include_authorized_operations=False):
|
|
981
1159
|
"""Send a DescribeGroupsRequest to the group's coordinator.
|
|
982
1160
|
|
|
983
|
-
:
|
|
984
|
-
|
|
985
|
-
broker.
|
|
986
|
-
|
|
1161
|
+
Arguments:
|
|
1162
|
+
group_id: The group name as a string
|
|
1163
|
+
group_coordinator_id: The node_id of the groups' coordinator broker.
|
|
1164
|
+
|
|
1165
|
+
Returns:
|
|
1166
|
+
A message future.
|
|
987
1167
|
"""
|
|
988
1168
|
version = self._matching_api_version(DescribeGroupsRequest)
|
|
989
1169
|
if version <= 2:
|
|
@@ -1066,18 +1246,23 @@ class KafkaAdminClient(object):
|
|
|
1066
1246
|
|
|
1067
1247
|
Any errors are immediately raised.
|
|
1068
1248
|
|
|
1069
|
-
:
|
|
1070
|
-
group
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1249
|
+
Arguments:
|
|
1250
|
+
group_ids: A list of consumer group IDs. These are typically the
|
|
1251
|
+
group names as strings.
|
|
1252
|
+
|
|
1253
|
+
Keyword Arguments:
|
|
1254
|
+
group_coordinator_id (int, optional): The node_id of the groups' coordinator
|
|
1255
|
+
broker. If set to None, it will query the cluster for each group to
|
|
1256
|
+
find that group's coordinator. Explicitly specifying this can be
|
|
1257
|
+
useful for avoiding extra network round trips if you already know
|
|
1258
|
+
the group coordinator. This is only useful when all the group_ids
|
|
1259
|
+
have the same coordinator, otherwise it will error. Default: None.
|
|
1260
|
+
include_authorized_operations (bool, optional): Whether or not to include
|
|
1261
|
+
information about the operations a group is allowed to perform.
|
|
1262
|
+
Only supported on API version >= v3. Default: False.
|
|
1263
|
+
|
|
1264
|
+
Returns:
|
|
1265
|
+
A list of group descriptions. For now the group descriptions
|
|
1081
1266
|
are the raw results from the DescribeGroupsResponse. Long-term, we
|
|
1082
1267
|
plan to change this to return namedtuples as well as decoding the
|
|
1083
1268
|
partition assignments.
|
|
@@ -1108,8 +1293,11 @@ class KafkaAdminClient(object):
|
|
|
1108
1293
|
def _list_consumer_groups_send_request(self, broker_id):
|
|
1109
1294
|
"""Send a ListGroupsRequest to a broker.
|
|
1110
1295
|
|
|
1111
|
-
:
|
|
1112
|
-
|
|
1296
|
+
Arguments:
|
|
1297
|
+
broker_id (int): The broker's node_id.
|
|
1298
|
+
|
|
1299
|
+
Returns:
|
|
1300
|
+
A message future
|
|
1113
1301
|
"""
|
|
1114
1302
|
version = self._matching_api_version(ListGroupsRequest)
|
|
1115
1303
|
if version <= 2:
|
|
@@ -1149,15 +1337,20 @@ class KafkaAdminClient(object):
|
|
|
1149
1337
|
|
|
1150
1338
|
As soon as any error is encountered, it is immediately raised.
|
|
1151
1339
|
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1340
|
+
Keyword Arguments:
|
|
1341
|
+
broker_ids ([int], optional): A list of broker node_ids to query for consumer
|
|
1342
|
+
groups. If set to None, will query all brokers in the cluster.
|
|
1343
|
+
Explicitly specifying broker(s) can be useful for determining which
|
|
1344
|
+
consumer groups are coordinated by those broker(s). Default: None
|
|
1345
|
+
|
|
1346
|
+
Returns:
|
|
1347
|
+
list: List of tuples of Consumer Groups.
|
|
1348
|
+
|
|
1349
|
+
Raises:
|
|
1350
|
+
GroupCoordinatorNotAvailableError: The coordinator is not
|
|
1351
|
+
available, so cannot process requests.
|
|
1352
|
+
GroupLoadInProgressError: The coordinator is loading and
|
|
1353
|
+
hence can't process requests.
|
|
1161
1354
|
"""
|
|
1162
1355
|
# While we return a list, internally use a set to prevent duplicates
|
|
1163
1356
|
# because if a group coordinator fails after being queried, and its
|
|
@@ -1177,10 +1370,17 @@ class KafkaAdminClient(object):
|
|
|
1177
1370
|
group_coordinator_id, partitions=None):
|
|
1178
1371
|
"""Send an OffsetFetchRequest to a broker.
|
|
1179
1372
|
|
|
1180
|
-
:
|
|
1181
|
-
|
|
1182
|
-
broker.
|
|
1183
|
-
|
|
1373
|
+
Arguments:
|
|
1374
|
+
group_id (str): The consumer group id name for which to fetch offsets.
|
|
1375
|
+
group_coordinator_id (int): The node_id of the group's coordinator broker.
|
|
1376
|
+
|
|
1377
|
+
Keyword Arguments:
|
|
1378
|
+
partitions: A list of TopicPartitions for which to fetch
|
|
1379
|
+
offsets. On brokers >= 0.10.2, this can be set to None to fetch all
|
|
1380
|
+
known offsets for the consumer group. Default: None.
|
|
1381
|
+
|
|
1382
|
+
Returns:
|
|
1383
|
+
A message future
|
|
1184
1384
|
"""
|
|
1185
1385
|
version = self._matching_api_version(OffsetFetchRequest)
|
|
1186
1386
|
if version <= 3:
|
|
@@ -1208,8 +1408,11 @@ class KafkaAdminClient(object):
|
|
|
1208
1408
|
def _list_consumer_group_offsets_process_response(self, response):
|
|
1209
1409
|
"""Process an OffsetFetchResponse.
|
|
1210
1410
|
|
|
1211
|
-
:
|
|
1212
|
-
|
|
1411
|
+
Arguments:
|
|
1412
|
+
response: an OffsetFetchResponse.
|
|
1413
|
+
|
|
1414
|
+
Returns:
|
|
1415
|
+
A dictionary composed of TopicPartition keys and
|
|
1213
1416
|
OffsetAndMetadata values.
|
|
1214
1417
|
"""
|
|
1215
1418
|
if response.API_VERSION <= 3:
|
|
@@ -1250,16 +1453,21 @@ class KafkaAdminClient(object):
|
|
|
1250
1453
|
|
|
1251
1454
|
As soon as any error is encountered, it is immediately raised.
|
|
1252
1455
|
|
|
1253
|
-
:
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1456
|
+
Arguments:
|
|
1457
|
+
group_id (str): The consumer group id name for which to fetch offsets.
|
|
1458
|
+
|
|
1459
|
+
Keyword Arguments:
|
|
1460
|
+
group_coordinator_id (int, optional): The node_id of the group's coordinator
|
|
1461
|
+
broker. If set to None, will query the cluster to find the group
|
|
1462
|
+
coordinator. Explicitly specifying this can be useful to prevent
|
|
1463
|
+
that extra network round trip if you already know the group
|
|
1464
|
+
coordinator. Default: None.
|
|
1465
|
+
partitions: A list of TopicPartitions for which to fetch
|
|
1466
|
+
offsets. On brokers >= 0.10.2, this can be set to None to fetch all
|
|
1467
|
+
known offsets for the consumer group. Default: None.
|
|
1468
|
+
|
|
1469
|
+
Returns:
|
|
1470
|
+
dictionary: A dictionary with TopicPartition keys and
|
|
1263
1471
|
OffsetAndMetada values. Partitions that are not specified and for
|
|
1264
1472
|
which the group_id does not have a recorded offset are omitted. An
|
|
1265
1473
|
offset value of `-1` indicates the group_id has no offset for that
|
|
@@ -1283,14 +1491,19 @@ class KafkaAdminClient(object):
|
|
|
1283
1491
|
|
|
1284
1492
|
The result needs checking for potential errors.
|
|
1285
1493
|
|
|
1286
|
-
:
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1494
|
+
Arguments:
|
|
1495
|
+
group_ids ([str]): The consumer group ids of the groups which are to be deleted.
|
|
1496
|
+
|
|
1497
|
+
Keyword Arguments:
|
|
1498
|
+
group_coordinator_id (int, optional): The node_id of the broker which is
|
|
1499
|
+
the coordinator for all the groups. Use only if all groups are coordinated
|
|
1500
|
+
by the same broker. If set to None, will query the cluster to find the coordinator
|
|
1501
|
+
for every single group. Explicitly specifying this can be useful to prevent
|
|
1502
|
+
that extra network round trips if you already know the group coordinator.
|
|
1503
|
+
Default: None.
|
|
1504
|
+
|
|
1505
|
+
Returns:
|
|
1506
|
+
A list of tuples (group_id, KafkaError)
|
|
1294
1507
|
"""
|
|
1295
1508
|
if group_coordinator_id is not None:
|
|
1296
1509
|
futures = [self._delete_consumer_groups_send_request(group_ids, group_coordinator_id)]
|
|
@@ -1311,6 +1524,14 @@ class KafkaAdminClient(object):
|
|
|
1311
1524
|
return results
|
|
1312
1525
|
|
|
1313
1526
|
def _convert_delete_groups_response(self, response):
|
|
1527
|
+
"""Parse the DeleteGroupsResponse, mapping group IDs to their respective errors.
|
|
1528
|
+
|
|
1529
|
+
Arguments:
|
|
1530
|
+
response: A DeleteGroupsResponse object from the broker.
|
|
1531
|
+
|
|
1532
|
+
Returns:
|
|
1533
|
+
A list of (group_id, KafkaError) for each deleted group.
|
|
1534
|
+
"""
|
|
1314
1535
|
if response.API_VERSION <= 1:
|
|
1315
1536
|
results = []
|
|
1316
1537
|
for group_id, error_code in response.results:
|
|
@@ -1322,12 +1543,14 @@ class KafkaAdminClient(object):
|
|
|
1322
1543
|
.format(response.API_VERSION))
|
|
1323
1544
|
|
|
1324
1545
|
def _delete_consumer_groups_send_request(self, group_ids, group_coordinator_id):
|
|
1325
|
-
"""Send a
|
|
1546
|
+
"""Send a DeleteGroupsRequest to the specified broker (the group coordinator).
|
|
1326
1547
|
|
|
1327
|
-
:
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
|
|
1548
|
+
Arguments:
|
|
1549
|
+
group_ids ([str]): A list of consumer group IDs to be deleted.
|
|
1550
|
+
group_coordinator_id (int): The node_id of the broker coordinating these groups.
|
|
1551
|
+
|
|
1552
|
+
Returns:
|
|
1553
|
+
A future representing the in-flight DeleteGroupsRequest.
|
|
1331
1554
|
"""
|
|
1332
1555
|
version = self._matching_api_version(DeleteGroupsRequest)
|
|
1333
1556
|
if version <= 1:
|
|
@@ -1339,6 +1562,14 @@ class KafkaAdminClient(object):
|
|
|
1339
1562
|
return self._send_request_to_node(group_coordinator_id, request)
|
|
1340
1563
|
|
|
1341
1564
|
def _wait_for_futures(self, futures):
|
|
1565
|
+
"""Block until all futures complete. If any fail, raise the encountered exception.
|
|
1566
|
+
|
|
1567
|
+
Arguments:
|
|
1568
|
+
futures: A list of Future objects awaiting results.
|
|
1569
|
+
|
|
1570
|
+
Raises:
|
|
1571
|
+
The first encountered exception if a future fails.
|
|
1572
|
+
"""
|
|
1342
1573
|
while not all(future.succeeded() for future in futures):
|
|
1343
1574
|
for future in futures:
|
|
1344
1575
|
self._client.poll(future=future)
|
|
@@ -1349,7 +1580,8 @@ class KafkaAdminClient(object):
|
|
|
1349
1580
|
def describe_log_dirs(self):
|
|
1350
1581
|
"""Send a DescribeLogDirsRequest request to a broker.
|
|
1351
1582
|
|
|
1352
|
-
:
|
|
1583
|
+
Returns:
|
|
1584
|
+
A message future
|
|
1353
1585
|
"""
|
|
1354
1586
|
version = self._matching_api_version(DescribeLogDirsRequest)
|
|
1355
1587
|
if version <= 0:
|