kafka-python 3.0.0__py3-none-any.whl

kafka/cli/admin/transactions/find_hanging.py

@@ -0,0 +1,26 @@
+class FindHangingTransactions:
+    COMMAND = 'find-hanging'
+    HELP = 'Find transactions older than the broker timeout + 5 min'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument(
+            '--broker-id', type=int, action='append', dest='broker_ids',
+            help='Query only these brokers (repeatable). Default: all.')
+        parser.add_argument(
+            '--max-transaction-timeout-ms', type=int, default=900_000,
+            help="Broker's max-transaction-timeout (ms). "
+                 'Transactions older than this + 5 min are flagged as hanging. '
+                 'Default: 900000 (Kafka default).')
+
+    @classmethod
+    def command(cls, client, args):
+        results = client.find_hanging_transactions(
+            broker_ids=args.broker_ids,
+            max_transaction_timeout_ms=args.max_transaction_timeout_ms,
+        )
+        for row in results:
+            row['topic_partitions'] = [
+                {'topic': tp.topic, 'partition': tp.partition}
+                for tp in row['topic_partitions']]
+        return results

kafka/cli/admin/transactions/list.py

@@ -0,0 +1,37 @@
+class ListTransactions:
+    COMMAND = 'list'
+    HELP = 'List transactions across all brokers (broker >= 3.0)'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument(
+            '--broker-id', type=int, action='append', dest='broker_ids',
+            help='Query only these brokers (repeatable). Default: all brokers.')
+        parser.add_argument(
+            '--producer-id', type=int, action='append', dest='producer_id_filters',
+            help='Only list transactions for these producer ids (repeatable).')
+        parser.add_argument(
+            '--state', type=str, action='append', dest='state_filters',
+            help='Only list transactions in these states (repeatable). '
+                 'Valid values: Empty, Ongoing, PrepareCommit, PrepareAbort, '
+                 'CompleteCommit, CompleteAbort, Dead, PrepareEpochFence.')
+        parser.add_argument(
+            '--duration-filter-ms', type=int, default=None,
+            help='Only list transactions running longer than this. '
+                 'Requires broker >= 3.8 (ListTransactions v1+).')
+        parser.add_argument(
+            '--id-pattern', type=str, default=None, dest='transactional_id_pattern',
+            help='Only list transactions whose id matches this regex. '
+                 'Requires broker >= 4.1 (KIP-1152).')
+
+    @classmethod
+    def command(cls, client, args):
+        results = client.list_transactions(
+            broker_ids=args.broker_ids,
+            producer_id_filters=args.producer_id_filters,
+            state_filters=args.state_filters,
+            duration_filter_ms=args.duration_filter_ms,
+            transactional_id_pattern=args.transactional_id_pattern,
+        )
+        return {broker_id: [t._asdict() for t in listings]
+                for broker_id, listings in results.items()}

kafka/cli/admin/users/__init__.py

@@ -0,0 +1,8 @@
+from .alter_user_scram_credentials import AlterUserScramCredentials
+from .describe_user_scram_credentials import DescribeUserScramCredentials
+
+
+class UsersCommandGroup:
+    GROUP = 'users'
+    HELP = 'Manage Kafka Users'
+    COMMANDS = [DescribeUserScramCredentials, AlterUserScramCredentials]

kafka/cli/admin/users/alter_user_scram_credentials.py

@@ -0,0 +1,34 @@
+from kafka.admin import (
+    ScramMechanism,
+    UserScramCredentialDeletion,
+    UserScramCredentialUpsertion,
+)
+
+
+class AlterUserScramCredentials:
+    COMMAND = 'alter-scram-credentials'
+    HELP = 'Alter SCRAM credentials for Kafka users'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument(
+            '--delete', type=str, action='append', dest='deletions', default=[],
+            help='USER:MECHANISM pair to delete (e.g. alice:SCRAM-SHA-256)')
+        parser.add_argument(
+            '--upsert', type=str, action='append', dest='upsertions', default=[],
+            help='USER:MECHANISM:PASSWORD triple to insert or update')
+        parser.add_argument(
+            '--iterations', type=int, default=None,
+            help='PBKDF2 iteration count for upsertions (default: 4096)')
+
+    @classmethod
+    def command(cls, client, args):
+        alterations = []
+        for spec in args.deletions:
+            user, mechanism = spec.split(':', 1)
+            alterations.append(UserScramCredentialDeletion(user, mechanism))
+        for spec in args.upsertions:
+            user, mechanism, password = spec.split(':', 2)
+            alterations.append(UserScramCredentialUpsertion(
+                user, mechanism, password, iterations=args.iterations))
+        return client.alter_user_scram_credentials(alterations)

kafka/cli/admin/users/describe_user_scram_credentials.py

@@ -0,0 +1,15 @@
+class DescribeUserScramCredentials:
+    COMMAND = 'describe-scram-credentials'
+    HELP = 'Describe SCRAM credentials for Kafka users'
+
+    @classmethod
+    def add_arguments(cls, parser):
+        parser.add_argument(
+            '--user', type=str, action='append', dest='users', default=[],
+            help='User name to describe (repeatable). '
+                 'If omitted, describes all users.')
+
+    @classmethod
+    def command(cls, client, args):
+        users = args.users if args.users else None
+        return client.describe_user_scram_credentials(users)

kafka/cli/common.py

@@ -0,0 +1,95 @@
+import logging
+
+
+def add_connect_cli_args(parser, bootstrap_required=True):
+    connect_group = parser.add_argument_group('connection')
+    connect_group.add_argument(
+        '-b', '--bootstrap-servers', type=str, action='append', required=bootstrap_required,
+        help='host:port for cluster bootstrap server. Can be provided multiple times.')
+    connect_group.add_argument(
+        '-S', '--security-protocol', type=str, default='PLAINTEXT', help='PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL')
+    connect_group.add_argument(
+        '-M', '--sasl-mechanism', type=str, default='PLAIN', help='PLAIN, GSSAPI, OAUTHBEARER, SCRAM-SHA-256, SCRAM-SHA-512')
+    connect_group.add_argument(
+        '-U', '--sasl-user', type=str)
+    connect_group.add_argument(
+        '-P', '--sasl-password', type=str)
+
+
+def build_kwargs(props):
+    kwargs = {}
+    for prop in props or []:
+        k, v = prop.split('=')
+        try:
+            v = int(v)
+        except ValueError:
+            pass
+        if v == 'None':
+            v = None
+        elif v == 'False':
+            v = False
+        elif v == 'True':
+            v = True
+        kwargs[k] = v
+    return kwargs
+
+
+def build_connect_kwargs(config):
+    if not config.bootstrap_servers:
+        raise ValueError('python -m kafka: error: the following arguments are required: -b/--bootstrap-servers')
+    kwargs = build_kwargs(config.extra_config)
+    kwargs.update({
+        'bootstrap_servers': config.bootstrap_servers,
+        'security_protocol': config.security_protocol,
+        'sasl_mechanism': config.sasl_mechanism,
+        'sasl_plain_username': config.sasl_user,
+        'sasl_plain_password': config.sasl_password,
+    })
+    return kwargs
+
+
+def add_logging_cli_args(parser):
+    logging_group = parser.add_argument_group('logging')
+    logging_group.add_argument(
+        '-l', '--log-level', type=str, default='CRITICAL',
+        help='logging level, passed to logging.basicConfig')
+    logging_group.add_argument(
+        '-L', '--enable-logger', type=str, action='append',
+        help='enable a specific logger. Can be provided multiple times. If not provided, all loggers are enabled')
+    logging_group.add_argument(
+        '-D', '--disable-logger', type=str, action='append',
+        help='disable a specific logger. Can be provided multiple times.')
+    extended_group = parser.add_argument_group('extended')
+    extended_group.add_argument(
+        '-C', '--extra-config', type=str, action='append',
+        help='additional configuration properties for client in "key=val" format. Can be provided multiple times.')
+
+
+def configure_logging(config):
+    _LOGGING_LEVELS = {
+        'NOTSET': 0,
+        'DEBUG': 10,
+        'INFO': 20,
+        'WARNING': 30,
+        'ERROR': 40,
+        'CRITICAL': 50,
+    }
+    if config.enable_logger is not None:
+        log_level = _LOGGING_LEVELS[config.log_level.upper()]
+        handler = logging.StreamHandler()
+        handler.setLevel(log_level)
+        handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
+        for name in config.enable_logger:
+            logger = logging.getLogger(name)
+            logger.setLevel(log_level)
+            logger.addHandler(handler)
+    else:
+        logging.basicConfig(level=_LOGGING_LEVELS[config.log_level.upper()])
+    if config.disable_logger is not None:
+        for name in config.disable_logger:
+            logging.getLogger(name).setLevel(logging.CRITICAL + 1)
+
+
+def add_common_cli_args(parser, bootstrap_required=True):
+    add_connect_cli_args(parser, bootstrap_required)
+    add_logging_cli_args(parser)

kafka/cli/consumer/__init__.py

@@ -0,0 +1,63 @@
+import argparse
+import logging
+
+from kafka import KafkaConsumer
+from kafka.errors import KafkaError
+from ..common import add_common_cli_args, configure_logging, build_connect_kwargs
+
+
+def main_parser(prog=None):
+    parser = argparse.ArgumentParser(
+        prog=prog or 'python -m kafka.consumer',
+        description='Kafka console consumer',
+    )
+    add_common_cli_args(parser)
+    options = parser.add_argument_group('consumer options')
+    options.add_argument(
+        '-t', '--topic', type=str, action='append', dest='topics', required=True,
+        help='subscribe to topic')
+    options.add_argument(
+        '-g', '--group', type=str,
+        help='consumer group')
+    options.add_argument(
+        '-i', '--group-instance-id', type=str,
+        help='static group membership identifier')
+    options.add_argument(
+        '-f', '--format', type=str, default='str',
+        help='output format: str|raw|full')
+    options.add_argument(
+        '--encoding', type=str, default='utf-8', help='encoding to use for str output decode()')
+    return parser
+
+
+def run_cli(args=None, prog=None):
+    parser = main_parser(prog=prog)
+    config = parser.parse_args(args)
+
+    if config.format not in ('str', 'raw', 'full'):
+        raise ValueError('Unrecognized format: %s' % config.format)
+    configure_logging(config)
+    logger = logging.getLogger(__name__)
+
+    kwargs = build_connect_kwargs(config)
+    consumer = KafkaConsumer(
+        group_id=config.group,
+        group_instance_id=config.group_instance_id,
+        **kwargs)
+    consumer.subscribe(config.topics)
+    try:
+        for m in consumer:
+            if config.format == 'str':
+                print(m.value.decode(config.encoding))
+            elif config.format == 'full':
+                print(m)
+            else:
+                print(m.value)
+    except KeyboardInterrupt:
+        logger.info('Bye!')
+        return 0
+    except Exception:
+        logger.critical('Error!', exc_info=True)
+        return 1
+    finally:
+        consumer.close()

kafka/cli/producer/__init__.py

@@ -0,0 +1,57 @@
+import argparse
+import logging
+import sys
+
+from kafka import KafkaProducer
+from ..common import add_common_cli_args, configure_logging, build_connect_kwargs
+
+
+def main_parser(prog=None):
+    parser = argparse.ArgumentParser(
+        prog=prog or 'python -m kafka.producer',
+        description='Kafka console producer',
+    )
+    add_common_cli_args(parser)
+    options = parser.add_argument_group('producer options')
+    options.add_argument(
+        '-t', '--topic', type=str, required=True,
+        help='publish to topic')
+    options.add_argument(
+        '--encoding', type=str, default='utf-8',
+        help='byte encoding for produced messages')
+    return parser
+
+
+def run_cli(args=None, prog=None):
+    parser = main_parser(prog=prog)
+    config = parser.parse_args(args)
+
+    configure_logging(config)
+    logger = logging.getLogger(__name__)
+
+    kwargs = build_connect_kwargs(config)
+    producer = KafkaProducer(**kwargs)
+
+    def log_result(res_or_err):
+        if isinstance(res_or_err, Exception):
+            logger.error("Error producing message", exc_info=res_or_err)
+        else:
+            logger.info("Message produced: %s", res_or_err)
+
+    try:
+        while True:
+            try:
+                value = input()
+            except EOFError:
+                value = sys.stdin.read().rstrip('\n')
+                if not value:
+                    return 0
+            producer.send(config.topic, value=value.encode(config.encoding)).add_both(log_result)
+    except KeyboardInterrupt:
+        logger.info('Bye!')
+        return 0
+    except Exception:
+        logger.critical('Error!', exc_info=True)
+        return 1
+    finally:
+        producer.close()