kafka-python 2.0.3__py2.py3-none-any.whl → 2.0.5__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- kafka/admin/client.py +370 -126
- kafka/client_async.py +81 -51
- kafka/cluster.py +1 -1
- kafka/codec.py +9 -2
- kafka/conn.py +38 -24
- kafka/consumer/group.py +2 -2
- kafka/coordinator/base.py +11 -1
- kafka/errors.py +587 -28
- kafka/producer/kafka.py +3 -3
- kafka/producer/sender.py +6 -6
- kafka/version.py +1 -1
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.5.dist-info}/METADATA +14 -14
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.5.dist-info}/RECORD +15 -16
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.5.dist-info}/WHEEL +1 -1
- kafka_python-2.0.3.dist-info/LICENSE +0 -202
- {kafka_python-2.0.3.dist-info → kafka_python-2.0.5.dist-info}/top_level.txt +0 -0
kafka/admin/client.py
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
from __future__ import absolute_import
|
|
1
|
+
from __future__ import absolute_import, division
|
|
2
2
|
|
|
3
3
|
from collections import defaultdict
|
|
4
4
|
import copy
|
|
5
5
|
import logging
|
|
6
6
|
import socket
|
|
7
|
+
import time
|
|
7
8
|
|
|
8
9
|
from . import ConfigResourceType
|
|
9
10
|
from kafka.vendor import six
|
|
@@ -72,7 +73,7 @@ class KafkaAdminClient(object):
|
|
|
72
73
|
reconnection attempts will continue periodically with this fixed
|
|
73
74
|
rate. To avoid connection storms, a randomization factor of 0.2
|
|
74
75
|
will be applied to the backoff resulting in a random range between
|
|
75
|
-
20% below and 20% above the computed value. Default:
|
|
76
|
+
20% below and 20% above the computed value. Default: 30000.
|
|
76
77
|
request_timeout_ms (int): Client request timeout in milliseconds.
|
|
77
78
|
Default: 30000.
|
|
78
79
|
connections_max_idle_ms: Close idle connections after the number of
|
|
@@ -156,7 +157,7 @@ class KafkaAdminClient(object):
|
|
|
156
157
|
'request_timeout_ms': 30000,
|
|
157
158
|
'connections_max_idle_ms': 9 * 60 * 1000,
|
|
158
159
|
'reconnect_backoff_ms': 50,
|
|
159
|
-
'reconnect_backoff_max_ms':
|
|
160
|
+
'reconnect_backoff_max_ms': 30000,
|
|
160
161
|
'max_in_flight_requests_per_connection': 5,
|
|
161
162
|
'receive_buffer_bytes': None,
|
|
162
163
|
'send_buffer_bytes': None,
|
|
@@ -212,11 +213,13 @@ class KafkaAdminClient(object):
|
|
|
212
213
|
metric_group_prefix='admin',
|
|
213
214
|
**self.config
|
|
214
215
|
)
|
|
215
|
-
self._client.check_version(timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
|
|
216
216
|
|
|
217
217
|
# Get auto-discovered version from client if necessary
|
|
218
218
|
if self.config['api_version'] is None:
|
|
219
219
|
self.config['api_version'] = self._client.config['api_version']
|
|
220
|
+
else:
|
|
221
|
+
# need to run check_version for get_api_versions()
|
|
222
|
+
self._client.check_version(timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
|
|
220
223
|
|
|
221
224
|
self._closed = False
|
|
222
225
|
self._refresh_controller_id()
|
|
@@ -240,8 +243,11 @@ class KafkaAdminClient(object):
|
|
|
240
243
|
This resolves to the lesser of either the latest api version this
|
|
241
244
|
library supports, or the max version supported by the broker.
|
|
242
245
|
|
|
243
|
-
:
|
|
244
|
-
|
|
246
|
+
Arguments:
|
|
247
|
+
operation: A list of protocol operation versions from kafka.protocol.
|
|
248
|
+
|
|
249
|
+
Returns:
|
|
250
|
+
int: The max matching version number between client and broker.
|
|
245
251
|
"""
|
|
246
252
|
broker_api_versions = self._client.get_api_versions()
|
|
247
253
|
api_key = operation[0].API_KEY
|
|
@@ -262,29 +268,41 @@ class KafkaAdminClient(object):
|
|
|
262
268
|
def _validate_timeout(self, timeout_ms):
|
|
263
269
|
"""Validate the timeout is set or use the configuration default.
|
|
264
270
|
|
|
265
|
-
:
|
|
266
|
-
|
|
271
|
+
Arguments:
|
|
272
|
+
timeout_ms: The timeout provided by api call, in milliseconds.
|
|
273
|
+
|
|
274
|
+
Returns:
|
|
275
|
+
The timeout to use for the operation.
|
|
267
276
|
"""
|
|
268
277
|
return timeout_ms or self.config['request_timeout_ms']
|
|
269
278
|
|
|
270
|
-
def _refresh_controller_id(self):
|
|
279
|
+
def _refresh_controller_id(self, timeout_ms=30000):
|
|
271
280
|
"""Determine the Kafka cluster controller."""
|
|
272
281
|
version = self._matching_api_version(MetadataRequest)
|
|
273
282
|
if 1 <= version <= 6:
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
283
|
+
timeout_at = time.time() + timeout_ms / 1000
|
|
284
|
+
while time.time() < timeout_at:
|
|
285
|
+
request = MetadataRequest[version]()
|
|
286
|
+
future = self._send_request_to_node(self._client.least_loaded_node(), request)
|
|
287
|
+
|
|
288
|
+
self._wait_for_futures([future])
|
|
289
|
+
|
|
290
|
+
response = future.value
|
|
291
|
+
controller_id = response.controller_id
|
|
292
|
+
if controller_id == -1:
|
|
293
|
+
log.warning("Controller ID not available, got -1")
|
|
294
|
+
time.sleep(1)
|
|
295
|
+
continue
|
|
296
|
+
# verify the controller is new enough to support our requests
|
|
297
|
+
controller_version = self._client.check_version(node_id=controller_id, timeout=(self.config['api_version_auto_timeout_ms'] / 1000))
|
|
298
|
+
if controller_version < (0, 10, 0):
|
|
299
|
+
raise IncompatibleBrokerVersion(
|
|
300
|
+
"The controller appears to be running Kafka {}. KafkaAdminClient requires brokers >= 0.10.0.0."
|
|
301
|
+
.format(controller_version))
|
|
302
|
+
self._controller_id = controller_id
|
|
303
|
+
return
|
|
304
|
+
else:
|
|
305
|
+
raise Errors.NodeNotAvailableError('controller')
|
|
288
306
|
else:
|
|
289
307
|
raise UnrecognizedBrokerVersion(
|
|
290
308
|
"Kafka Admin interface cannot determine the controller using MetadataRequest_v{}."
|
|
@@ -293,9 +311,12 @@ class KafkaAdminClient(object):
|
|
|
293
311
|
def _find_coordinator_id_send_request(self, group_id):
|
|
294
312
|
"""Send a FindCoordinatorRequest to a broker.
|
|
295
313
|
|
|
296
|
-
:
|
|
314
|
+
Arguments:
|
|
315
|
+
group_id: The consumer group ID. This is typically the group
|
|
297
316
|
name as a string.
|
|
298
|
-
|
|
317
|
+
|
|
318
|
+
Returns:
|
|
319
|
+
A message future
|
|
299
320
|
"""
|
|
300
321
|
# TODO add support for dynamically picking version of
|
|
301
322
|
# GroupCoordinatorRequest which was renamed to FindCoordinatorRequest.
|
|
@@ -315,8 +336,11 @@ class KafkaAdminClient(object):
|
|
|
315
336
|
def _find_coordinator_id_process_response(self, response):
|
|
316
337
|
"""Process a FindCoordinatorResponse.
|
|
317
338
|
|
|
318
|
-
:
|
|
319
|
-
|
|
339
|
+
Arguments:
|
|
340
|
+
response: a FindCoordinatorResponse.
|
|
341
|
+
|
|
342
|
+
Returns:
|
|
343
|
+
The node_id of the broker that is the coordinator.
|
|
320
344
|
"""
|
|
321
345
|
if response.API_VERSION <= 0:
|
|
322
346
|
error_type = Errors.for_code(response.error_code)
|
|
@@ -339,9 +363,12 @@ class KafkaAdminClient(object):
|
|
|
339
363
|
Will block until the FindCoordinatorResponse is received for all groups.
|
|
340
364
|
Any errors are immediately raised.
|
|
341
365
|
|
|
342
|
-
:
|
|
366
|
+
Arguments:
|
|
367
|
+
group_ids: A list of consumer group IDs. This is typically the group
|
|
343
368
|
name as a string.
|
|
344
|
-
|
|
369
|
+
|
|
370
|
+
Returns:
|
|
371
|
+
A dict of {group_id: node_id} where node_id is the id of the
|
|
345
372
|
broker that is the coordinator for the corresponding group.
|
|
346
373
|
"""
|
|
347
374
|
groups_futures = {
|
|
@@ -358,18 +385,24 @@ class KafkaAdminClient(object):
|
|
|
358
385
|
def _send_request_to_node(self, node_id, request, wakeup=True):
|
|
359
386
|
"""Send a Kafka protocol message to a specific broker.
|
|
360
387
|
|
|
361
|
-
|
|
388
|
+
Arguments:
|
|
389
|
+
node_id: The broker id to which to send the message.
|
|
390
|
+
request: The message to send.
|
|
391
|
+
|
|
362
392
|
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
:
|
|
367
|
-
|
|
393
|
+
Keyword Arguments:
|
|
394
|
+
wakeup (bool, optional): Optional flag to disable thread-wakeup.
|
|
395
|
+
|
|
396
|
+
Returns:
|
|
397
|
+
A future object that may be polled for status and results.
|
|
398
|
+
|
|
399
|
+
Raises:
|
|
400
|
+
The exception if the message could not be sent.
|
|
368
401
|
"""
|
|
369
402
|
while not self._client.ready(node_id):
|
|
370
403
|
# poll until the connection to broker is ready, otherwise send()
|
|
371
404
|
# will fail with NodeNotReadyError
|
|
372
|
-
self._client.poll()
|
|
405
|
+
self._client.poll(timeout_ms=200)
|
|
373
406
|
return self._client.send(node_id, request, wakeup)
|
|
374
407
|
|
|
375
408
|
def _send_request_to_controller(self, request):
|
|
@@ -377,8 +410,11 @@ class KafkaAdminClient(object):
|
|
|
377
410
|
|
|
378
411
|
Will block until the message result is received.
|
|
379
412
|
|
|
380
|
-
:
|
|
381
|
-
|
|
413
|
+
Arguments:
|
|
414
|
+
request: The message to send.
|
|
415
|
+
|
|
416
|
+
Returns:
|
|
417
|
+
The Kafka protocol response for the message.
|
|
382
418
|
"""
|
|
383
419
|
tries = 2 # in case our cached self._controller_id is outdated
|
|
384
420
|
while tries:
|
|
@@ -418,6 +454,18 @@ class KafkaAdminClient(object):
|
|
|
418
454
|
|
|
419
455
|
@staticmethod
|
|
420
456
|
def _convert_new_topic_request(new_topic):
|
|
457
|
+
"""
|
|
458
|
+
Build the tuple required by CreateTopicsRequest from a NewTopic object.
|
|
459
|
+
|
|
460
|
+
Arguments:
|
|
461
|
+
new_topic: A NewTopic instance containing name, partition count, replication factor,
|
|
462
|
+
replica assignments, and config entries.
|
|
463
|
+
|
|
464
|
+
Returns:
|
|
465
|
+
A tuple in the form:
|
|
466
|
+
(topic_name, num_partitions, replication_factor, [(partition_id, [replicas])...],
|
|
467
|
+
[(config_key, config_value)...])
|
|
468
|
+
"""
|
|
421
469
|
return (
|
|
422
470
|
new_topic.name,
|
|
423
471
|
new_topic.num_partitions,
|
|
@@ -433,12 +481,17 @@ class KafkaAdminClient(object):
|
|
|
433
481
|
def create_topics(self, new_topics, timeout_ms=None, validate_only=False):
|
|
434
482
|
"""Create new topics in the cluster.
|
|
435
483
|
|
|
436
|
-
:
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
484
|
+
Arguments:
|
|
485
|
+
new_topics: A list of NewTopic objects.
|
|
486
|
+
|
|
487
|
+
Keyword Arguments:
|
|
488
|
+
timeout_ms (numeric, optional): Milliseconds to wait for new topics to be created
|
|
489
|
+
before the broker returns.
|
|
490
|
+
validate_only (bool, optional): If True, don't actually create new topics.
|
|
491
|
+
Not supported by all versions. Default: False
|
|
492
|
+
|
|
493
|
+
Returns:
|
|
494
|
+
Appropriate version of CreateTopicResponse class.
|
|
442
495
|
"""
|
|
443
496
|
version = self._matching_api_version(CreateTopicsRequest)
|
|
444
497
|
timeout_ms = self._validate_timeout(timeout_ms)
|
|
@@ -468,10 +521,15 @@ class KafkaAdminClient(object):
|
|
|
468
521
|
def delete_topics(self, topics, timeout_ms=None):
|
|
469
522
|
"""Delete topics from the cluster.
|
|
470
523
|
|
|
471
|
-
:
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
:
|
|
524
|
+
Arguments:
|
|
525
|
+
topics ([str]): A list of topic name strings.
|
|
526
|
+
|
|
527
|
+
Keyword Arguments:
|
|
528
|
+
timeout_ms (numeric, optional): Milliseconds to wait for topics to be deleted
|
|
529
|
+
before the broker returns.
|
|
530
|
+
|
|
531
|
+
Returns:
|
|
532
|
+
Appropriate version of DeleteTopicsResponse class.
|
|
475
533
|
"""
|
|
476
534
|
version = self._matching_api_version(DeleteTopicsRequest)
|
|
477
535
|
timeout_ms = self._validate_timeout(timeout_ms)
|
|
@@ -515,16 +573,38 @@ class KafkaAdminClient(object):
|
|
|
515
573
|
return future.value
|
|
516
574
|
|
|
517
575
|
def list_topics(self):
|
|
576
|
+
"""Retrieve a list of all topic names in the cluster.
|
|
577
|
+
|
|
578
|
+
Returns:
|
|
579
|
+
A list of topic name strings.
|
|
580
|
+
"""
|
|
518
581
|
metadata = self._get_cluster_metadata(topics=None)
|
|
519
582
|
obj = metadata.to_object()
|
|
520
583
|
return [t['topic'] for t in obj['topics']]
|
|
521
584
|
|
|
522
585
|
def describe_topics(self, topics=None):
|
|
586
|
+
"""Fetch metadata for the specified topics or all topics if None.
|
|
587
|
+
|
|
588
|
+
Keyword Arguments:
|
|
589
|
+
topics ([str], optional) A list of topic names. If None, metadata for all
|
|
590
|
+
topics is retrieved.
|
|
591
|
+
|
|
592
|
+
Returns:
|
|
593
|
+
A list of dicts describing each topic (including partition info).
|
|
594
|
+
"""
|
|
523
595
|
metadata = self._get_cluster_metadata(topics=topics)
|
|
524
596
|
obj = metadata.to_object()
|
|
525
597
|
return obj['topics']
|
|
526
598
|
|
|
527
599
|
def describe_cluster(self):
|
|
600
|
+
"""
|
|
601
|
+
Fetch cluster-wide metadata such as the list of brokers, the controller ID,
|
|
602
|
+
and the cluster ID.
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
Returns:
|
|
606
|
+
A dict with cluster-wide metadata, excluding topic details.
|
|
607
|
+
"""
|
|
528
608
|
metadata = self._get_cluster_metadata()
|
|
529
609
|
obj = metadata.to_object()
|
|
530
610
|
obj.pop('topics') # We have 'describe_topics' for this
|
|
@@ -532,6 +612,15 @@ class KafkaAdminClient(object):
|
|
|
532
612
|
|
|
533
613
|
@staticmethod
|
|
534
614
|
def _convert_describe_acls_response_to_acls(describe_response):
|
|
615
|
+
"""Convert a DescribeAclsResponse into a list of ACL objects and a KafkaError.
|
|
616
|
+
|
|
617
|
+
Arguments:
|
|
618
|
+
describe_response: The response object from the DescribeAclsRequest.
|
|
619
|
+
|
|
620
|
+
Returns:
|
|
621
|
+
A tuple of (list_of_acl_objects, error) where error is an instance
|
|
622
|
+
of KafkaError (NoError if successful).
|
|
623
|
+
"""
|
|
535
624
|
version = describe_response.API_VERSION
|
|
536
625
|
|
|
537
626
|
error = Errors.for_code(describe_response.error_code)
|
|
@@ -571,8 +660,11 @@ class KafkaAdminClient(object):
|
|
|
571
660
|
The cluster must be configured with an authorizer for this to work, or
|
|
572
661
|
you will get a SecurityDisabledError
|
|
573
662
|
|
|
574
|
-
:
|
|
575
|
-
|
|
663
|
+
Arguments:
|
|
664
|
+
acl_filter: an ACLFilter object
|
|
665
|
+
|
|
666
|
+
Returns:
|
|
667
|
+
tuple of a list of matching ACL objects and a KafkaError (NoError if successful)
|
|
576
668
|
"""
|
|
577
669
|
|
|
578
670
|
version = self._matching_api_version(DescribeAclsRequest)
|
|
@@ -617,6 +709,14 @@ class KafkaAdminClient(object):
|
|
|
617
709
|
|
|
618
710
|
@staticmethod
|
|
619
711
|
def _convert_create_acls_resource_request_v0(acl):
|
|
712
|
+
"""Convert an ACL object into the CreateAclsRequest v0 format.
|
|
713
|
+
|
|
714
|
+
Arguments:
|
|
715
|
+
acl: An ACL object with resource pattern and permissions.
|
|
716
|
+
|
|
717
|
+
Returns:
|
|
718
|
+
A tuple: (resource_type, resource_name, principal, host, operation, permission_type).
|
|
719
|
+
"""
|
|
620
720
|
|
|
621
721
|
return (
|
|
622
722
|
acl.resource_pattern.resource_type,
|
|
@@ -629,7 +729,14 @@ class KafkaAdminClient(object):
|
|
|
629
729
|
|
|
630
730
|
@staticmethod
|
|
631
731
|
def _convert_create_acls_resource_request_v1(acl):
|
|
732
|
+
"""Convert an ACL object into the CreateAclsRequest v1 format.
|
|
632
733
|
|
|
734
|
+
Arguments:
|
|
735
|
+
acl: An ACL object with resource pattern and permissions.
|
|
736
|
+
|
|
737
|
+
Returns:
|
|
738
|
+
A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type).
|
|
739
|
+
"""
|
|
633
740
|
return (
|
|
634
741
|
acl.resource_pattern.resource_type,
|
|
635
742
|
acl.resource_pattern.resource_name,
|
|
@@ -642,6 +749,19 @@ class KafkaAdminClient(object):
|
|
|
642
749
|
|
|
643
750
|
@staticmethod
|
|
644
751
|
def _convert_create_acls_response_to_acls(acls, create_response):
|
|
752
|
+
"""Parse CreateAclsResponse and correlate success/failure with original ACL objects.
|
|
753
|
+
|
|
754
|
+
Arguments:
|
|
755
|
+
acls: A list of ACL objects that were requested for creation.
|
|
756
|
+
create_response: The broker's CreateAclsResponse object.
|
|
757
|
+
|
|
758
|
+
Returns:
|
|
759
|
+
A dict with:
|
|
760
|
+
{
|
|
761
|
+
'succeeded': [list of ACL objects successfully created],
|
|
762
|
+
'failed': [(acl_object, KafkaError), ...]
|
|
763
|
+
}
|
|
764
|
+
"""
|
|
645
765
|
version = create_response.API_VERSION
|
|
646
766
|
|
|
647
767
|
creations_error = []
|
|
@@ -670,8 +790,11 @@ class KafkaAdminClient(object):
|
|
|
670
790
|
This endpoint only accepts a list of concrete ACL objects, no ACLFilters.
|
|
671
791
|
Throws TopicAlreadyExistsError if topic is already present.
|
|
672
792
|
|
|
673
|
-
:
|
|
674
|
-
|
|
793
|
+
Arguments:
|
|
794
|
+
acls: a list of ACL objects
|
|
795
|
+
|
|
796
|
+
Returns:
|
|
797
|
+
dict of successes and failures
|
|
675
798
|
"""
|
|
676
799
|
|
|
677
800
|
for acl in acls:
|
|
@@ -701,6 +824,14 @@ class KafkaAdminClient(object):
|
|
|
701
824
|
|
|
702
825
|
@staticmethod
|
|
703
826
|
def _convert_delete_acls_resource_request_v0(acl):
|
|
827
|
+
"""Convert an ACLFilter object into the DeleteAclsRequest v0 format.
|
|
828
|
+
|
|
829
|
+
Arguments:
|
|
830
|
+
acl: An ACLFilter object identifying the ACLs to be deleted.
|
|
831
|
+
|
|
832
|
+
Returns:
|
|
833
|
+
A tuple: (resource_type, resource_name, principal, host, operation, permission_type).
|
|
834
|
+
"""
|
|
704
835
|
return (
|
|
705
836
|
acl.resource_pattern.resource_type,
|
|
706
837
|
acl.resource_pattern.resource_name,
|
|
@@ -712,6 +843,14 @@ class KafkaAdminClient(object):
|
|
|
712
843
|
|
|
713
844
|
@staticmethod
|
|
714
845
|
def _convert_delete_acls_resource_request_v1(acl):
|
|
846
|
+
"""Convert an ACLFilter object into the DeleteAclsRequest v1 format.
|
|
847
|
+
|
|
848
|
+
Arguments:
|
|
849
|
+
acl: An ACLFilter object identifying the ACLs to be deleted.
|
|
850
|
+
|
|
851
|
+
Returns:
|
|
852
|
+
A tuple: (resource_type, resource_name, pattern_type, principal, host, operation, permission_type).
|
|
853
|
+
"""
|
|
715
854
|
return (
|
|
716
855
|
acl.resource_pattern.resource_type,
|
|
717
856
|
acl.resource_pattern.resource_name,
|
|
@@ -724,6 +863,16 @@ class KafkaAdminClient(object):
|
|
|
724
863
|
|
|
725
864
|
@staticmethod
|
|
726
865
|
def _convert_delete_acls_response_to_matching_acls(acl_filters, delete_response):
|
|
866
|
+
"""Parse the DeleteAclsResponse and map the results back to each input ACLFilter.
|
|
867
|
+
|
|
868
|
+
Arguments:
|
|
869
|
+
acl_filters: A list of ACLFilter objects that were provided in the request.
|
|
870
|
+
delete_response: The response from the DeleteAclsRequest.
|
|
871
|
+
|
|
872
|
+
Returns:
|
|
873
|
+
A list of tuples of the form:
|
|
874
|
+
(acl_filter, [(matching_acl, KafkaError), ...], filter_level_error).
|
|
875
|
+
"""
|
|
727
876
|
version = delete_response.API_VERSION
|
|
728
877
|
filter_result_list = []
|
|
729
878
|
for i, filter_responses in enumerate(delete_response.filter_responses):
|
|
@@ -762,8 +911,11 @@ class KafkaAdminClient(object):
|
|
|
762
911
|
|
|
763
912
|
Deletes all ACLs matching the list of input ACLFilter
|
|
764
913
|
|
|
765
|
-
:
|
|
766
|
-
|
|
914
|
+
Arguments:
|
|
915
|
+
acl_filters: a list of ACLFilter
|
|
916
|
+
|
|
917
|
+
Returns:
|
|
918
|
+
a list of 3-tuples corresponding to the list of input filters.
|
|
767
919
|
The tuples hold (the input ACLFilter, list of affected ACLs, KafkaError instance)
|
|
768
920
|
"""
|
|
769
921
|
|
|
@@ -795,6 +947,14 @@ class KafkaAdminClient(object):
|
|
|
795
947
|
|
|
796
948
|
@staticmethod
|
|
797
949
|
def _convert_describe_config_resource_request(config_resource):
|
|
950
|
+
"""Convert a ConfigResource into the format required by DescribeConfigsRequest.
|
|
951
|
+
|
|
952
|
+
Arguments:
|
|
953
|
+
config_resource: A ConfigResource with resource_type, name, and optional config keys.
|
|
954
|
+
|
|
955
|
+
Returns:
|
|
956
|
+
A tuple: (resource_type, resource_name, [list_of_config_keys] or None).
|
|
957
|
+
"""
|
|
798
958
|
return (
|
|
799
959
|
config_resource.resource_type,
|
|
800
960
|
config_resource.name,
|
|
@@ -806,13 +966,18 @@ class KafkaAdminClient(object):
|
|
|
806
966
|
def describe_configs(self, config_resources, include_synonyms=False):
|
|
807
967
|
"""Fetch configuration parameters for one or more Kafka resources.
|
|
808
968
|
|
|
809
|
-
:
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
:
|
|
969
|
+
Arguments:
|
|
970
|
+
config_resources: An list of ConfigResource objects.
|
|
971
|
+
Any keys in ConfigResource.configs dict will be used to filter the
|
|
972
|
+
result. Setting the configs dict to None will get all values. An
|
|
973
|
+
empty dict will get zero values (as per Kafka protocol).
|
|
974
|
+
|
|
975
|
+
Keyword Arguments:
|
|
976
|
+
include_synonyms (bool, optional): If True, return synonyms in response. Not
|
|
977
|
+
supported by all versions. Default: False.
|
|
978
|
+
|
|
979
|
+
Returns:
|
|
980
|
+
Appropriate version of DescribeConfigsResponse class.
|
|
816
981
|
"""
|
|
817
982
|
|
|
818
983
|
# Break up requests by type - a broker config request must be sent to the specific broker.
|
|
@@ -881,6 +1046,14 @@ class KafkaAdminClient(object):
|
|
|
881
1046
|
|
|
882
1047
|
@staticmethod
|
|
883
1048
|
def _convert_alter_config_resource_request(config_resource):
|
|
1049
|
+
"""Convert a ConfigResource into the format required by AlterConfigsRequest.
|
|
1050
|
+
|
|
1051
|
+
Arguments:
|
|
1052
|
+
config_resource: A ConfigResource with resource_type, name, and config (key, value) pairs.
|
|
1053
|
+
|
|
1054
|
+
Returns:
|
|
1055
|
+
A tuple: (resource_type, resource_name, [(config_key, config_value), ...]).
|
|
1056
|
+
"""
|
|
884
1057
|
return (
|
|
885
1058
|
config_resource.resource_type,
|
|
886
1059
|
config_resource.name,
|
|
@@ -898,8 +1071,11 @@ class KafkaAdminClient(object):
|
|
|
898
1071
|
least-loaded node. See the comment in the source code for details.
|
|
899
1072
|
We would happily accept a PR fixing this.
|
|
900
1073
|
|
|
901
|
-
:
|
|
902
|
-
|
|
1074
|
+
Arguments:
|
|
1075
|
+
config_resources: A list of ConfigResource objects.
|
|
1076
|
+
|
|
1077
|
+
Returns:
|
|
1078
|
+
Appropriate version of AlterConfigsResponse class.
|
|
903
1079
|
"""
|
|
904
1080
|
version = self._matching_api_version(AlterConfigsRequest)
|
|
905
1081
|
if version <= 1:
|
|
@@ -930,6 +1106,15 @@ class KafkaAdminClient(object):
|
|
|
930
1106
|
|
|
931
1107
|
@staticmethod
|
|
932
1108
|
def _convert_create_partitions_request(topic_name, new_partitions):
|
|
1109
|
+
"""Convert a NewPartitions object into the tuple format for CreatePartitionsRequest.
|
|
1110
|
+
|
|
1111
|
+
Arguments:
|
|
1112
|
+
topic_name: The name of the existing topic.
|
|
1113
|
+
new_partitions: A NewPartitions instance with total_count and new_assignments.
|
|
1114
|
+
|
|
1115
|
+
Returns:
|
|
1116
|
+
A tuple: (topic_name, (total_count, [list_of_assignments])).
|
|
1117
|
+
"""
|
|
933
1118
|
return (
|
|
934
1119
|
topic_name,
|
|
935
1120
|
(
|
|
@@ -941,12 +1126,17 @@ class KafkaAdminClient(object):
|
|
|
941
1126
|
def create_partitions(self, topic_partitions, timeout_ms=None, validate_only=False):
|
|
942
1127
|
"""Create additional partitions for an existing topic.
|
|
943
1128
|
|
|
944
|
-
:
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
1129
|
+
Arguments:
|
|
1130
|
+
topic_partitions: A map of topic name strings to NewPartition objects.
|
|
1131
|
+
|
|
1132
|
+
Keyword Arguments:
|
|
1133
|
+
timeout_ms (numeric, optional): Milliseconds to wait for new partitions to be
|
|
1134
|
+
created before the broker returns.
|
|
1135
|
+
validate_only (bool, optional): If True, don't actually create new partitions.
|
|
1136
|
+
Default: False
|
|
1137
|
+
|
|
1138
|
+
Returns:
|
|
1139
|
+
Appropriate version of CreatePartitionsResponse class.
|
|
950
1140
|
"""
|
|
951
1141
|
version = self._matching_api_version(CreatePartitionsRequest)
|
|
952
1142
|
timeout_ms = self._validate_timeout(timeout_ms)
|
|
@@ -980,10 +1170,12 @@ class KafkaAdminClient(object):
|
|
|
980
1170
|
def _describe_consumer_groups_send_request(self, group_id, group_coordinator_id, include_authorized_operations=False):
|
|
981
1171
|
"""Send a DescribeGroupsRequest to the group's coordinator.
|
|
982
1172
|
|
|
983
|
-
:
|
|
984
|
-
|
|
985
|
-
broker.
|
|
986
|
-
|
|
1173
|
+
Arguments:
|
|
1174
|
+
group_id: The group name as a string
|
|
1175
|
+
group_coordinator_id: The node_id of the groups' coordinator broker.
|
|
1176
|
+
|
|
1177
|
+
Returns:
|
|
1178
|
+
A message future.
|
|
987
1179
|
"""
|
|
988
1180
|
version = self._matching_api_version(DescribeGroupsRequest)
|
|
989
1181
|
if version <= 2:
|
|
@@ -1066,18 +1258,23 @@ class KafkaAdminClient(object):
|
|
|
1066
1258
|
|
|
1067
1259
|
Any errors are immediately raised.
|
|
1068
1260
|
|
|
1069
|
-
:
|
|
1070
|
-
group
|
|
1071
|
-
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
|
|
1078
|
-
|
|
1079
|
-
|
|
1080
|
-
|
|
1261
|
+
Arguments:
|
|
1262
|
+
group_ids: A list of consumer group IDs. These are typically the
|
|
1263
|
+
group names as strings.
|
|
1264
|
+
|
|
1265
|
+
Keyword Arguments:
|
|
1266
|
+
group_coordinator_id (int, optional): The node_id of the groups' coordinator
|
|
1267
|
+
broker. If set to None, it will query the cluster for each group to
|
|
1268
|
+
find that group's coordinator. Explicitly specifying this can be
|
|
1269
|
+
useful for avoiding extra network round trips if you already know
|
|
1270
|
+
the group coordinator. This is only useful when all the group_ids
|
|
1271
|
+
have the same coordinator, otherwise it will error. Default: None.
|
|
1272
|
+
include_authorized_operations (bool, optional): Whether or not to include
|
|
1273
|
+
information about the operations a group is allowed to perform.
|
|
1274
|
+
Only supported on API version >= v3. Default: False.
|
|
1275
|
+
|
|
1276
|
+
Returns:
|
|
1277
|
+
A list of group descriptions. For now the group descriptions
|
|
1081
1278
|
are the raw results from the DescribeGroupsResponse. Long-term, we
|
|
1082
1279
|
plan to change this to return namedtuples as well as decoding the
|
|
1083
1280
|
partition assignments.
|
|
@@ -1108,8 +1305,11 @@ class KafkaAdminClient(object):
|
|
|
1108
1305
|
def _list_consumer_groups_send_request(self, broker_id):
|
|
1109
1306
|
"""Send a ListGroupsRequest to a broker.
|
|
1110
1307
|
|
|
1111
|
-
:
|
|
1112
|
-
|
|
1308
|
+
Arguments:
|
|
1309
|
+
broker_id (int): The broker's node_id.
|
|
1310
|
+
|
|
1311
|
+
Returns:
|
|
1312
|
+
A message future
|
|
1113
1313
|
"""
|
|
1114
1314
|
version = self._matching_api_version(ListGroupsRequest)
|
|
1115
1315
|
if version <= 2:
|
|
@@ -1149,15 +1349,20 @@ class KafkaAdminClient(object):
|
|
|
1149
1349
|
|
|
1150
1350
|
As soon as any error is encountered, it is immediately raised.
|
|
1151
1351
|
|
|
1152
|
-
|
|
1153
|
-
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1352
|
+
Keyword Arguments:
|
|
1353
|
+
broker_ids ([int], optional): A list of broker node_ids to query for consumer
|
|
1354
|
+
groups. If set to None, will query all brokers in the cluster.
|
|
1355
|
+
Explicitly specifying broker(s) can be useful for determining which
|
|
1356
|
+
consumer groups are coordinated by those broker(s). Default: None
|
|
1357
|
+
|
|
1358
|
+
Returns:
|
|
1359
|
+
list: List of tuples of Consumer Groups.
|
|
1360
|
+
|
|
1361
|
+
Raises:
|
|
1362
|
+
GroupCoordinatorNotAvailableError: The coordinator is not
|
|
1363
|
+
available, so cannot process requests.
|
|
1364
|
+
GroupLoadInProgressError: The coordinator is loading and
|
|
1365
|
+
hence can't process requests.
|
|
1161
1366
|
"""
|
|
1162
1367
|
# While we return a list, internally use a set to prevent duplicates
|
|
1163
1368
|
# because if a group coordinator fails after being queried, and its
|
|
@@ -1177,10 +1382,17 @@ class KafkaAdminClient(object):
|
|
|
1177
1382
|
group_coordinator_id, partitions=None):
|
|
1178
1383
|
"""Send an OffsetFetchRequest to a broker.
|
|
1179
1384
|
|
|
1180
|
-
:
|
|
1181
|
-
|
|
1182
|
-
broker.
|
|
1183
|
-
|
|
1385
|
+
Arguments:
|
|
1386
|
+
group_id (str): The consumer group id name for which to fetch offsets.
|
|
1387
|
+
group_coordinator_id (int): The node_id of the group's coordinator broker.
|
|
1388
|
+
|
|
1389
|
+
Keyword Arguments:
|
|
1390
|
+
partitions: A list of TopicPartitions for which to fetch
|
|
1391
|
+
offsets. On brokers >= 0.10.2, this can be set to None to fetch all
|
|
1392
|
+
known offsets for the consumer group. Default: None.
|
|
1393
|
+
|
|
1394
|
+
Returns:
|
|
1395
|
+
A message future
|
|
1184
1396
|
"""
|
|
1185
1397
|
version = self._matching_api_version(OffsetFetchRequest)
|
|
1186
1398
|
if version <= 3:
|
|
@@ -1208,8 +1420,11 @@ class KafkaAdminClient(object):
|
|
|
1208
1420
|
def _list_consumer_group_offsets_process_response(self, response):
|
|
1209
1421
|
"""Process an OffsetFetchResponse.
|
|
1210
1422
|
|
|
1211
|
-
:
|
|
1212
|
-
|
|
1423
|
+
Arguments:
|
|
1424
|
+
response: an OffsetFetchResponse.
|
|
1425
|
+
|
|
1426
|
+
Returns:
|
|
1427
|
+
A dictionary composed of TopicPartition keys and
|
|
1213
1428
|
OffsetAndMetadata values.
|
|
1214
1429
|
"""
|
|
1215
1430
|
if response.API_VERSION <= 3:
|
|
@@ -1250,16 +1465,21 @@ class KafkaAdminClient(object):
|
|
|
1250
1465
|
|
|
1251
1466
|
As soon as any error is encountered, it is immediately raised.
|
|
1252
1467
|
|
|
1253
|
-
:
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
|
|
1468
|
+
Arguments:
|
|
1469
|
+
group_id (str): The consumer group id name for which to fetch offsets.
|
|
1470
|
+
|
|
1471
|
+
Keyword Arguments:
|
|
1472
|
+
group_coordinator_id (int, optional): The node_id of the group's coordinator
|
|
1473
|
+
broker. If set to None, will query the cluster to find the group
|
|
1474
|
+
coordinator. Explicitly specifying this can be useful to prevent
|
|
1475
|
+
that extra network round trip if you already know the group
|
|
1476
|
+
coordinator. Default: None.
|
|
1477
|
+
partitions: A list of TopicPartitions for which to fetch
|
|
1478
|
+
offsets. On brokers >= 0.10.2, this can be set to None to fetch all
|
|
1479
|
+
known offsets for the consumer group. Default: None.
|
|
1480
|
+
|
|
1481
|
+
Returns:
|
|
1482
|
+
dictionary: A dictionary with TopicPartition keys and
|
|
1263
1483
|
OffsetAndMetada values. Partitions that are not specified and for
|
|
1264
1484
|
which the group_id does not have a recorded offset are omitted. An
|
|
1265
1485
|
offset value of `-1` indicates the group_id has no offset for that
|
|
@@ -1283,14 +1503,19 @@ class KafkaAdminClient(object):
|
|
|
1283
1503
|
|
|
1284
1504
|
The result needs checking for potential errors.
|
|
1285
1505
|
|
|
1286
|
-
:
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1506
|
+
Arguments:
|
|
1507
|
+
group_ids ([str]): The consumer group ids of the groups which are to be deleted.
|
|
1508
|
+
|
|
1509
|
+
Keyword Arguments:
|
|
1510
|
+
group_coordinator_id (int, optional): The node_id of the broker which is
|
|
1511
|
+
the coordinator for all the groups. Use only if all groups are coordinated
|
|
1512
|
+
by the same broker. If set to None, will query the cluster to find the coordinator
|
|
1513
|
+
for every single group. Explicitly specifying this can be useful to prevent
|
|
1514
|
+
that extra network round trips if you already know the group coordinator.
|
|
1515
|
+
Default: None.
|
|
1516
|
+
|
|
1517
|
+
Returns:
|
|
1518
|
+
A list of tuples (group_id, KafkaError)
|
|
1294
1519
|
"""
|
|
1295
1520
|
if group_coordinator_id is not None:
|
|
1296
1521
|
futures = [self._delete_consumer_groups_send_request(group_ids, group_coordinator_id)]
|
|
@@ -1311,6 +1536,14 @@ class KafkaAdminClient(object):
|
|
|
1311
1536
|
return results
|
|
1312
1537
|
|
|
1313
1538
|
def _convert_delete_groups_response(self, response):
|
|
1539
|
+
"""Parse the DeleteGroupsResponse, mapping group IDs to their respective errors.
|
|
1540
|
+
|
|
1541
|
+
Arguments:
|
|
1542
|
+
response: A DeleteGroupsResponse object from the broker.
|
|
1543
|
+
|
|
1544
|
+
Returns:
|
|
1545
|
+
A list of (group_id, KafkaError) for each deleted group.
|
|
1546
|
+
"""
|
|
1314
1547
|
if response.API_VERSION <= 1:
|
|
1315
1548
|
results = []
|
|
1316
1549
|
for group_id, error_code in response.results:
|
|
@@ -1322,12 +1555,14 @@ class KafkaAdminClient(object):
|
|
|
1322
1555
|
.format(response.API_VERSION))
|
|
1323
1556
|
|
|
1324
1557
|
def _delete_consumer_groups_send_request(self, group_ids, group_coordinator_id):
|
|
1325
|
-
"""Send a
|
|
1558
|
+
"""Send a DeleteGroupsRequest to the specified broker (the group coordinator).
|
|
1559
|
+
|
|
1560
|
+
Arguments:
|
|
1561
|
+
group_ids ([str]): A list of consumer group IDs to be deleted.
|
|
1562
|
+
group_coordinator_id (int): The node_id of the broker coordinating these groups.
|
|
1326
1563
|
|
|
1327
|
-
:
|
|
1328
|
-
|
|
1329
|
-
all the groups.
|
|
1330
|
-
:return: A message future
|
|
1564
|
+
Returns:
|
|
1565
|
+
A future representing the in-flight DeleteGroupsRequest.
|
|
1331
1566
|
"""
|
|
1332
1567
|
version = self._matching_api_version(DeleteGroupsRequest)
|
|
1333
1568
|
if version <= 1:
|
|
@@ -1339,6 +1574,14 @@ class KafkaAdminClient(object):
|
|
|
1339
1574
|
return self._send_request_to_node(group_coordinator_id, request)
|
|
1340
1575
|
|
|
1341
1576
|
def _wait_for_futures(self, futures):
|
|
1577
|
+
"""Block until all futures complete. If any fail, raise the encountered exception.
|
|
1578
|
+
|
|
1579
|
+
Arguments:
|
|
1580
|
+
futures: A list of Future objects awaiting results.
|
|
1581
|
+
|
|
1582
|
+
Raises:
|
|
1583
|
+
The first encountered exception if a future fails.
|
|
1584
|
+
"""
|
|
1342
1585
|
while not all(future.succeeded() for future in futures):
|
|
1343
1586
|
for future in futures:
|
|
1344
1587
|
self._client.poll(future=future)
|
|
@@ -1349,7 +1592,8 @@ class KafkaAdminClient(object):
|
|
|
1349
1592
|
def describe_log_dirs(self):
|
|
1350
1593
|
"""Send a DescribeLogDirsRequest request to a broker.
|
|
1351
1594
|
|
|
1352
|
-
:
|
|
1595
|
+
Returns:
|
|
1596
|
+
A message future
|
|
1353
1597
|
"""
|
|
1354
1598
|
version = self._matching_api_version(DescribeLogDirsRequest)
|
|
1355
1599
|
if version <= 0:
|