kafka-python 3.0.0__py3-none-any.whl

kafka/cli/admin/acls/delete.py

@@ -0,0 +1,23 @@
+from .common import add_acl_filter_args, acl_filter_from_args
+
+
+class DeleteACLs:
+    COMMAND = 'delete'
+    HELP = 'Delete Kafka ACLs'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        add_acl_filter_args(parser)
+
+    @classmethod
+    def command(cls, client, args):
+        acl_filter = acl_filter_from_args(args)
+        results = client.delete_acls([acl_filter])
+        output = []
+        for acl_filter, matching_acls, error in results:
+            output.append({
+                'filter': repr(acl_filter),
+                'deleted': [repr(acl) for acl in matching_acls],
+                'error': str(error),
+            })
+        return output

kafka/cli/admin/acls/describe.py

@@ -0,0 +1,16 @@
+from .common import add_acl_filter_args, acl_filter_from_args
+
+
+class DescribeACLs:
+    COMMAND = 'describe'
+    HELP = 'Describe Kafka ACLs'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        add_acl_filter_args(parser)
+
+    @classmethod
+    def command(cls, client, args):
+        acl_filter = acl_filter_from_args(args)
+        acls, error = client.describe_acls(acl_filter)
+        return [repr(acl) for acl in acls]

kafka/cli/admin/cluster/__init__.py

@@ -0,0 +1,14 @@
+from .describe import DescribeCluster
+from .describe_quorum import DescribeQuorum
+from .features import DescribeFeatures, UpdateFeatures
+from .log_dirs import DescribeLogDirs, AlterLogDirs
+from .versions import GetApiVersions, GetBrokerVersion
+
+
+class ClusterCommandGroup:
+    GROUP = 'cluster'
+    HELP = 'Manage Kafka Cluster'
+    COMMANDS = [DescribeCluster, DescribeQuorum,
+                GetApiVersions, GetBrokerVersion,
+                DescribeFeatures, UpdateFeatures,
+                DescribeLogDirs, AlterLogDirs]

kafka/cli/admin/cluster/describe.py

@@ -0,0 +1,11 @@
+class DescribeCluster:
+    COMMAND = 'describe'
+    HELP = 'Describe Kafka Cluster'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        pass
+
+    @classmethod
+    def command(cls, client, args):
+        return client.describe_cluster()

kafka/cli/admin/cluster/describe_quorum.py

@@ -0,0 +1,11 @@
+class DescribeQuorum:
+    COMMAND = 'describe-quorum'
+    HELP = 'Describe the KRaft metadata quorum'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        pass
+
+    @classmethod
+    def command(cls, client, args):
+        return client.describe_metadata_quorum()

kafka/cli/admin/cluster/features.py

@@ -0,0 +1,52 @@
+from kafka.admin import UpdateFeatureType
+
+
+class DescribeFeatures:
+    COMMAND = 'describe-features'
+    HELP = 'Describe Features of Kafka Cluster'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-f', '--feature', type=str, action='append', dest='features', default=[],
+                            help='Show one or more specific features. If not provided, returns all features.')
+
+    @classmethod
+    def command(cls, client, args):
+        result = client.describe_features()
+        if args.features:
+            return {k: v for k, v in result.items() if k in args.features}
+        else:
+            return result
+
+
+class UpdateFeatures:
+    COMMAND = 'update-features'
+    HELP = 'Update Features of Kafka Cluster'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-f', '--feature', type=str, action='append', dest='features', default=[], help='set feature=value')
+        parser.add_argument('--downgrade', action='store_true')
+        parser.add_argument('--unsafe', action='store_true')
+        parser.add_argument('--timeout', type=int, default=60)
+        parser.add_argument('--validate-only', action='store_true')
+
+    @staticmethod
+    def _feature_type(args):
+        if not args.downgrade:
+            return UpdateFeatureType.UPGRADE
+        elif args.unsafe:
+            return UpdateFeatureType.UNSAFE_DOWNGRADE
+        else:
+            return UpdateFeatureType.SAFE_DOWNGRADE
+
+    @classmethod
+    def command(cls, client, args):
+        feature_type = cls._feature_type(args)
+        feature_updates = {
+            feature_name: (feature_type, version)
+            for feature_name, version in [feature.split('=') for feature in args.features]
+        }
+        return client.update_features(feature_updates,
+                                      validate_only=args.validate_only,
+                                      timeout_ms=1000*args.timeout)

kafka/cli/admin/cluster/log_dirs.py

@@ -0,0 +1,43 @@
+from kafka.structs import TopicPartitionReplica
+
+
+class DescribeLogDirs:
+    COMMAND = 'describe-log-dirs'
+    HELP = 'Get topic log directories and stats'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('--broker', type=int, action='append', dest='brokers', help='Query specific broker(s)')
+        parser.add_argument('--topic', type=str, action='append', dest='topics', help='Get data about specific topic(s)')
+
+    @classmethod
+    def command(cls, client, args):
+        return client.describe_log_dirs(topic_partitions=args.topics, brokers=args.brokers)
+
+
+class AlterLogDirs:
+    COMMAND = 'alter-log-dirs'
+    HELP = 'Move replicas between log directories on their hosting brokers'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument(
+            '-a', '--assignment', type=str, action='append',
+            dest='assignments', default=[], required=True,
+            help='TOPIC:PARTITION:BROKER_ID=/absolute/log/dir/path (repeatable). '
+                 'Instructs BROKER_ID to move its replica of TOPIC:PARTITION '
+                 'into the given log directory.')
+
+    @classmethod
+    def command(cls, client, args):
+        assignments = {}
+        for spec in args.assignments:
+            tpr_str, log_dir = spec.rsplit('=', 1)
+            topic, partition, broker_id = tpr_str.rsplit(':', 2)
+            tpr = TopicPartitionReplica(topic, int(partition), int(broker_id))
+            assignments[tpr] = log_dir
+        result = client.alter_replica_log_dirs(assignments)
+        return {
+            f'{tpr.topic}:{tpr.partition}:{tpr.broker_id}': err.__name__
+            for tpr, err in result.items()
+        }

kafka/cli/admin/cluster/versions.py

@@ -0,0 +1,33 @@
+from kafka.protocol.api_key import ApiKey
+
+
+class GetApiVersions:
+    COMMAND = 'api-versions'
+    HELP = 'Get Supported Api Versions'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-k', '--api-key', type=str, action='append', dest='api_keys', default=None)
+        parser.add_argument('--raw', action='store_true')
+
+    @classmethod
+    def command(cls, client, args):
+        api_keys = set(ApiKey[k] for k in args.api_keys) if args.api_keys else set(ApiKey)
+        api_versions = client.api_versions()
+        return {(k.value if args.raw else k.name): v for k, v in api_versions.items()
+                if k in api_keys}
+
+
+class GetBrokerVersion:
+    COMMAND = 'broker-version'
+    HELP = 'Get Version for Broker'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('--broker', required=True)
+
+    @classmethod
+    def command(cls, client, args):
+        broker_id = int(args.broker)
+        bvd = client.get_broker_version_data(broker_id)
+        return {broker_id: '.'.join(map(str, bvd.broker_version))}

kafka/cli/admin/configs/__init__.py

@@ -0,0 +1,10 @@
+from .alter import AlterConfigs
+from .describe import DescribeConfigs
+from .list import ListConfigResources
+from .reset import ResetConfigs
+
+
+class ConfigsCommandGroup:
+    GROUP = 'configs'
+    HELP = 'Manage Kafka Configuration'
+    COMMANDS = [DescribeConfigs, AlterConfigs, ListConfigResources, ResetConfigs]

kafka/cli/admin/configs/alter.py

@@ -0,0 +1,43 @@
+import re
+
+from kafka.admin import AlterConfigOp
+from .common import add_resource_arguments, parse_resources
+
+
+class AlterConfigs:
+    COMMAND = 'alter'
+    HELP = 'Alter Kafka Configs'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        add_resource_arguments(parser)
+        parser.add_argument('-c', '--config', type=str, action='append', dest='configs', required=True, help='key=value to alter')
+        parser.add_argument('-v', '--validate-only', action='store_true', default=False)
+        parser.add_argument('--allow-unknown', action='store_false', dest='raise_on_unknown', default=True)
+        incremental = parser.add_mutually_exclusive_group()
+        incremental.add_argument('--force-incremental', action='store_true', dest='incremental', default=None)
+        incremental.add_argument('--force-alter', action='store_false', dest='incremental', default=None)
+
+    @classmethod
+    def command(cls, client, args):
+        try:
+            configs = dict(config.split('=') for config in args.configs)
+            regex = r'^(set|del|add|sub)\((.*)\)$'
+            ops = {
+                'set': AlterConfigOp.SET,
+                'del': AlterConfigOp.DELETE,
+                'add': AlterConfigOp.APPEND,
+                'sub': AlterConfigOp.SUBTRACT,
+            }
+            for key in configs:
+                match = re.match(regex, configs[key])
+                if match:
+                    op_str, val = match.groups()
+                    configs[key] = (ops[op_str], val)
+        except ValueError:
+            raise ValueError(f'Unable to parse configs! {args.configs}')
+        resources = parse_resources(args, configs=configs)
+        return client.alter_configs(resources,
+                                    validate_only=args.validate_only,
+                                    raise_on_unknown=args.raise_on_unknown,
+                                    incremental=args.incremental)

kafka/cli/admin/configs/common.py

@@ -0,0 +1,17 @@
+from kafka.admin import ConfigResource
+
+
+def add_resource_arguments(parser):
+    """Add arguments for specifying ConfigResource"""
+    parser.add_argument(
+        '-r', '--resource-type', type=str, required=True,
+        help='Type of resource to describe: topic, broker, broker_logger, '
+             'client_metrics, group.')
+    parser.add_argument(
+        '-n', '--resource-name', type=str, action='append', dest='resource_names', required=True,
+        help='Name of resource(s) to describe. May be repeated.')
+
+
+def parse_resources(args, configs=None):
+    return [ConfigResource(args.resource_type.upper().replace('-', '_'), resource_name, configs)
+            for resource_name in args.resource_names]

kafka/cli/admin/configs/describe.py

@@ -0,0 +1,30 @@
+from .common import add_resource_arguments, parse_resources
+
+
+class DescribeConfigs:
+    COMMAND = 'describe'
+    HELP = 'Describe Kafka Configs'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        add_resource_arguments(parser)
+        parser.add_argument('-c', '--config', type=str, action='append', dest='configs', default=None)
+        parser.add_argument('--dynamic', action='store_true', default=False)
+        parser.add_argument('--modified', action='store_true', default=False)
+        parser.add_argument('--static', action='store_true', default=False)
+        parser.add_argument('--default', action='store_true', default=False)
+
+    @classmethod
+    def command(cls, client, args):
+        resources = parse_resources(args, configs=args.configs)
+        if args.modified:
+            config_filter = 'modified'
+        elif args.dynamic:
+            config_filter = 'dynamic'
+        elif args.static:
+            config_filter = 'static'
+        elif args.default:
+            config_filter = 'default'
+        else:
+            config_filter = 'all'
+        return client.describe_configs(resources, config_filter=config_filter)

kafka/cli/admin/configs/list.py

@@ -0,0 +1,16 @@
+class ListConfigResources:
+    COMMAND = 'list'
+    HELP = 'List config resources known to the cluster (requires broker >= 4.1 for non client_metrics types)'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument(
+            '-r', '--resource-type', type=str, action='append', dest='resource_types', default=[],
+            help='Filter by resource type (repeatable): topic, broker, '
+                  'broker_logger, client_metrics, group. Omit to list all '
+                  'supported types.')
+
+    @classmethod
+    def command(cls, client, args):
+        return client.list_config_resources(
+            resource_types=args.resource_types or None)

kafka/cli/admin/configs/reset.py

@@ -0,0 +1,20 @@
+from .common import add_resource_arguments, parse_resources
+
+
+class ResetConfigs:
+    COMMAND = 'reset'
+    HELP = 'Reset Kafka Configs'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        add_resource_arguments(parser)
+        parser.add_argument('-c', '--config', type=str, action='append', dest='configs', default=[], help='key to reset')
+        parser.add_argument('-v', '--validate-only', action='store_true', default=False)
+        parser.add_argument('--allow-unknown', action='store_false', dest='raise_on_unknown', default=True)
+
+    @classmethod
+    def command(cls, client, args):
+        resources = parse_resources(args, configs=args.configs)
+        return client.reset_configs(resources,
+                                    validate_only=args.validate_only,
+                                    raise_on_unknown=args.raise_on_unknown)

kafka/cli/admin/groups/__init__.py

@@ -0,0 +1,16 @@
+from .alter_offsets import AlterGroupOffsets
+from .delete import DeleteGroups
+from .delete_offsets import DeleteGroupOffsets
+from .describe import DescribeGroups
+from .list import ListGroups
+from .list_offsets import ListGroupOffsets
+from .remove_members import RemoveGroupMembers
+from .reset_offsets import ResetGroupOffsets
+
+
+class GroupsCommandGroup:
+    GROUP = 'groups'
+    HELP = 'Manage Kafka Groups'
+    COMMANDS = [ListGroups, DescribeGroups, DeleteGroups,
+                ListGroupOffsets, AlterGroupOffsets, ResetGroupOffsets, DeleteGroupOffsets,
+                RemoveGroupMembers]

kafka/cli/admin/groups/alter_offsets.py

@@ -0,0 +1,30 @@
+from kafka.structs import OffsetAndMetadata, TopicPartition
+
+
+class AlterGroupOffsets:
+    COMMAND = 'alter-offsets'
+    HELP = 'Alter committed offsets for a consumer group'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-g', '--group-id', type=str, required=True)
+        parser.add_argument(
+            '-o', '--offset', type=str, action='append',
+            dest='offsets', default=[], required=True,
+            help='TOPIC:PARTITION:OFFSET triple (repeatable).')
+        parser.add_argument(
+            '--group-coordinator-id', type=int, default=None,
+            help='Send directly to this broker id, skipping coordinator lookup')
+
+    @classmethod
+    def command(cls, client, args):
+        offsets = {}
+        for spec in args.offsets:
+            topic, partition, offset = spec.rsplit(':', 2)
+            offsets[TopicPartition(topic, int(partition))] = \
+                OffsetAndMetadata(int(offset), '', None)
+        result = client.alter_group_offsets(
+            args.group_id, offsets,
+            group_coordinator_id=args.group_coordinator_id)
+        return {'%s:%d' % (tp.topic, tp.partition): err.__name__
+                for tp, err in result.items()}

kafka/cli/admin/groups/delete.py

@@ -0,0 +1,11 @@
+class DeleteGroups:
+    COMMAND = 'delete'
+    HELP = 'Delete Groups'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-g', '--group-id', type=str, action='append', dest='groups', required=True)
+
+    @classmethod
+    def command(cls, client, args):
+        return client.delete_groups(args.groups)

kafka/cli/admin/groups/delete_offsets.py

@@ -0,0 +1,29 @@
+from kafka.structs import TopicPartition
+
+
+class DeleteGroupOffsets:
+    COMMAND = 'delete-offsets'
+    HELP = 'Delete committed offsets for a consumer group'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-g', '--group-id', type=str, required=True)
+        parser.add_argument(
+            '-p', '--partition', type=str, action='append',
+            dest='partitions', default=[], required=True,
+            help='TOPIC:PARTITION pair (repeatable).')
+        parser.add_argument(
+            '--group-coordinator-id', type=int, default=None,
+            help='Send directly to this broker id, skipping coordinator lookup')
+
+    @classmethod
+    def command(cls, client, args):
+        partitions = []
+        for spec in args.partitions:
+            topic, partition = spec.rsplit(':', 1)
+            partitions.append(TopicPartition(topic, int(partition)))
+        result = client.delete_group_offsets(
+            args.group_id, partitions,
+            group_coordinator_id=args.group_coordinator_id)
+        return {'%s:%d' % (tp.topic, tp.partition): err.__name__
+                for tp, err in result.items()}

kafka/cli/admin/groups/describe.py

@@ -0,0 +1,11 @@
+class DescribeGroups:
+    COMMAND = 'describe'
+    HELP = 'Describe Groups'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-g', '--group-id', type=str, action='append', dest='groups', required=True)
+
+    @classmethod
+    def command(cls, client, args):
+        return client.describe_groups(args.groups)

kafka/cli/admin/groups/list.py

@@ -0,0 +1,28 @@
+from kafka.admin import GroupState, GroupType
+
+
+class ListGroups:
+    COMMAND = 'list'
+    HELP = 'List Groups'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        state_choices = sorted(s.value for s in GroupState)
+        type_choices = sorted(t.value for t in GroupType)
+        parser.add_argument(
+            '--state', type=str, action='append', dest='states_filter', default=[],
+            help='Filter by group state (repeatable). One of: '
+                 + ', '.join(state_choices)
+                 + '. Case-insensitive; names also accepted. '
+                   'Requires broker >= 3.0 (KIP-518).')
+        parser.add_argument(
+            '--type', type=str, action='append', dest='types_filter', default=[],
+            help='Filter by group type (repeatable). One of: '
+                 + ', '.join(type_choices)
+                 + '. Requires broker >= 4.0 (KIP-848).')
+
+    @classmethod
+    def command(cls, client, args):
+        return client.list_groups(
+            states_filter=args.states_filter or None,
+            types_filter=args.types_filter or None)

kafka/cli/admin/groups/list_offsets.py

@@ -0,0 +1,29 @@
+from collections import defaultdict
+
+from kafka.structs import TopicPartition
+from kafka.admin import OffsetSpec
+
+
+class ListGroupOffsets:
+    COMMAND = 'list-offsets'
+    HELP = 'List Offsets for Group'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-g', '--group-id', type=str, required=True)
+
+    @classmethod
+    def command(cls, client, args):
+        offsets = client.list_group_offsets(args.group_id)[args.group_id]
+        latest = client.list_partition_offsets({tp: OffsetSpec.LATEST for tp in offsets})
+        results = defaultdict(dict)
+        for tp in latest:
+            committed = offsets[tp]
+            results[tp.topic][tp.partition] = {
+                'offset': committed.offset,
+                'leader_epoch': committed.leader_epoch,
+                'metadata': committed.metadata,
+                'latest_offset': latest[tp].offset,
+                'lag': latest[tp].offset - committed.offset,
+            }
+        return dict(results)

kafka/cli/admin/groups/remove_members.py

@@ -0,0 +1,40 @@
+from kafka.admin import MemberToRemove
+
+
+class RemoveGroupMembers:
+    COMMAND = 'remove-members'
+    HELP = 'Remove members from a consumer group'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument('-g', '--group-id', type=str, required=True)
+        parser.add_argument(
+            '-m', '--member-id', type=str, action='append',
+            dest='member_ids', default=[],
+            help='Dynamic member id to remove (repeatable).')
+        parser.add_argument(
+            '-i', '--group-instance-id', type=str, action='append',
+            dest='group_instance_ids', default=[],
+            help='Static group.instance.id to remove (repeatable; '
+                 'requires broker LeaveGroup v3+).')
+        parser.add_argument(
+            '--reason', type=str, default=None,
+            help='Optional reason; sent to broker on LeaveGroup v5+ '
+                 '(KIP-800). Applied to all members.')
+        parser.add_argument(
+            '--group-coordinator-id', type=int, default=None,
+            help='Send directly to this broker id, skipping coordinator lookup')
+
+    @classmethod
+    def command(cls, client, args):
+        members = [MemberToRemove(member_id=mid, reason=args.reason)
+                   for mid in args.member_ids]
+        members.extend(MemberToRemove(group_instance_id=gid, reason=args.reason)
+                       for gid in args.group_instance_ids)
+        if not members:
+            raise ValueError(
+                'At least one --member-id or --group-instance-id is required.')
+        result = client.remove_group_members(
+            args.group_id, members,
+            group_coordinator_id=args.group_coordinator_id)
+        return {m: err.__name__ for m, err in result.items()}