kafka-python 3.0.0__py3-none-any.whl

kafka/consumer/group.py

@@ -0,0 +1,1347 @@
+import copy
+import logging
+import re
+import selectors
+import socket
+import time
+
+import kafka.errors as Errors
+from kafka.errors import KafkaConfigurationError, UnsupportedVersionError
+
+from kafka.consumer.fetcher import Fetcher
+from kafka.consumer.subscription_state import SubscriptionState
+from kafka.coordinator.consumer import ConsumerCoordinator
+from kafka.coordinator.assignors.range import RangePartitionAssignor
+from kafka.coordinator.assignors.roundrobin import RoundRobinPartitionAssignor
+from kafka.metrics import MetricConfig, Metrics
+from kafka.net.compat import KafkaNetClient
+from kafka.protocol.consumer import OffsetResetStrategy
+from kafka.structs import ConsumerGroupMetadata, OffsetAndMetadata, TopicPartition
+from kafka.util import Timer
+from kafka.version import __version__
+
+log = logging.getLogger(__name__)
+
+
+class KafkaConsumer:
+    """Consume records from a Kafka cluster.
+
+    The consumer will transparently handle the failure of servers in the Kafka
+    cluster, and adapt as topic-partitions are created or migrate between
+    brokers. It also interacts with the assigned kafka Group Coordinator node
+    to allow multiple consumers to load balance consumption of topics (requires
+    kafka >= 0.9.0.0).
+
+    The consumer is not thread safe and should not be shared across threads.
+
+    Arguments:
+        *topics (str): optional list of topics to subscribe to. If not set,
+            call :meth:`~kafka.KafkaConsumer.subscribe` or
+            :meth:`~kafka.KafkaConsumer.assign` before consuming records.
+
+    Keyword Arguments:
+        bootstrap_servers: 'host[:port]' string (or list of 'host[:port]'
+            strings) that the consumer should contact to bootstrap initial
+            cluster metadata. This does not have to be the full node list.
+            It just needs to have at least one broker that will respond to a
+            Metadata API Request. Default port is 9092. If no servers are
+            specified, will default to localhost:9092.
+        client_id (str): A name for this client. This string is passed in
+            each request to servers and can be used to identify specific
+            server-side log entries that correspond to this client. Also
+            submitted to GroupCoordinator for logging with respect to
+            consumer group administration. Default: 'kafka-python-{version}'
+        client_rack (str): A rack identifier for this client. Sent to brokers
+            on FetchRequest v11+ (KIP-392, requires Kafka 2.4+ brokers with
+            ``replica.selector.class`` configured server-side). When set, the
+            broker may route fetches to a follower replica in the same rack
+            instead of the leader, reducing cross-rack traffic. Leave as ''
+            (default) to always fetch from the leader.
+        group_id (str or None): The name of the consumer group to join for dynamic
+            partition assignment (if enabled), and to use for fetching and
+            committing offsets. If None, auto-partition assignment (via
+            group coordinator) and offset commits are disabled.
+            Default: None
+        group_instance_id (str): A unique identifier of the consumer instance
+            provided by end user. Only non-empty strings are permitted. If set,
+            the consumer is treated as a static member, which means that only
+            one instance with this ID is allowed in the consumer group at any
+            time. This can be used in combination with a larger session timeout
+            to avoid group rebalances caused by transient unavailability (e.g.
+            process restarts). If not set, the consumer will join the group as
+            a dynamic member, which is the traditional behavior. Default: None
+        key_deserializer (kafka.serializer.Deserializer): Takes a
+            raw message key and returns a deserialized key.
+            Default: None.
+        value_deserializer (kafka.serializer.Deserializer): Takes a
+            raw message value and returns a deserialized value.
+            Default: None.
+        enable_incremental_fetch_sessions: (bool): Use incremental fetch sessions
+            when available / supported by kafka broker. See KIP-227. Default: True.
+        fetch_min_bytes (int): Minimum amount of data the server should
+            return for a fetch request, otherwise wait up to
+            fetch_max_wait_ms for more data to accumulate. Default: 1.
+        fetch_max_wait_ms (int): The maximum amount of time in milliseconds
+            the server will block before answering the fetch request if
+            there isn't sufficient data to immediately satisfy the
+            requirement given by fetch_min_bytes. Default: 500.
+        fetch_max_bytes (int): The maximum amount of data the server should
+            return for a fetch request. This is not an absolute maximum, if the
+            first message in the first non-empty partition of the fetch is
+            larger than this value, the message will still be returned to
+            ensure that the consumer can make progress. NOTE: consumer performs
+            fetches to multiple brokers in parallel so memory usage will depend
+            on the number of brokers containing partitions for the topic.
+            Supported Kafka version >= 0.10.1.0. Default: 52428800 (50 MB).
+        max_partition_fetch_bytes (int): The maximum amount of data
+            per-partition the server will return. The maximum total memory
+            used for a request = #partitions * max_partition_fetch_bytes.
+            This size must be at least as large as the maximum message size
+            the server allows or else it is possible for the producer to
+            send messages larger than the consumer can fetch. If that
+            happens, the consumer can get stuck trying to fetch a large
+            message on a certain partition. Default: 1048576.
+        request_timeout_ms (int): Client request timeout in milliseconds.
+            Default: 305000.
+        retry_backoff_ms (int): Milliseconds to backoff when retrying on
+            errors. Default: 100.
+        reconnect_backoff_ms (int): The amount of time in milliseconds to
+            wait before attempting to reconnect to a given host.
+            Default: 50.
+        reconnect_backoff_max_ms (int): The maximum amount of time in
+            milliseconds to backoff/wait when reconnecting to a broker that has
+            repeatedly failed to connect. If provided, the backoff per host
+            will increase exponentially for each consecutive connection
+            failure, up to this maximum. Once the maximum is reached,
+            reconnection attempts will continue periodically with this fixed
+            rate. To avoid connection storms, a randomization factor of 0.2
+            will be applied to the backoff resulting in a random range between
+            20% below and 20% above the computed value. Default: 30000.
+        max_in_flight_requests_per_connection (int): Requests are pipelined
+            to kafka brokers up to this number of maximum requests per
+            broker connection. Default: 5.
+        auto_offset_reset (str): A policy for resetting offsets on
+            OffsetOutOfRange errors: 'earliest' will move to the oldest
+            available message, 'latest' will move to the most recent. Any
+            other value will raise the exception. Default: 'latest'.
+        enable_auto_commit (bool): If True , the consumer's offset will be
+            periodically committed in the background. Default: True.
+        auto_commit_interval_ms (int): Number of milliseconds between automatic
+            offset commits, if enable_auto_commit is True. Default: 5000.
+        default_offset_commit_callback (callable): Called as
+            callback(offsets, response) response will be either an Exception
+            or an OffsetCommitResponse struct. This callback can be used to
+            trigger custom actions when a commit request completes.
+        check_crcs (bool): Automatically check the CRC32 of the records
+            consumed. This ensures no on-the-wire or on-disk corruption to
+            the messages occurred. This check adds some overhead, so it may
+            be disabled in cases seeking extreme performance. Default: True
+        isolation_level (str): Configure KIP-98 transactional consumer by
+            setting to 'read_committed'. This will cause the consumer to
+            skip records from aborted transactions. Default: 'read_uncommitted'
+        allow_auto_create_topics (bool): Enable/disable auto topic creation
+            on metadata request. Only available with api_version >= (0, 11).
+            Default: True
+        metadata_max_age_ms (int): The period of time in milliseconds after
+            which we force a refresh of metadata, even if we haven't seen any
+            partition leadership changes to proactively discover any new
+            brokers or partitions. Default: 300000
+        partition_assignment_strategy (list): List of objects to use to
+            distribute partition ownership amongst consumer instances when
+            group management is used.
+            Default: [RangePartitionAssignor, RoundRobinPartitionAssignor]
+        max_poll_records (int): The maximum number of records returned in a
+            single call to :meth:`~kafka.KafkaConsumer.poll`. Default: 500
+        max_poll_interval_ms (int): The maximum delay between invocations of
+            :meth:`~kafka.KafkaConsumer.poll` when using consumer group
+            management. This places an upper bound on the amount of time that
+            the consumer can be idle before fetching more records. If
+            :meth:`~kafka.KafkaConsumer.poll` is not called before expiration
+            of this timeout, then the consumer is considered failed and the
+            group will rebalance in order to reassign the partitions to another
+            member. Default 300000
+        session_timeout_ms (int): The timeout used to detect failures when
+            using Kafka's group management facilities. The consumer sends
+            periodic heartbeats to indicate its liveness to the broker. If
+            no heartbeats are received by the broker before the expiration of
+            this session timeout, then the broker will remove this consumer
+            from the group and initiate a rebalance. Note that the value must
+            be in the allowable range as configured in the broker configuration
+            by group.min.session.timeout.ms and group.max.session.timeout.ms.
+            Default: 45000 for brokers 3.0+, otherwise 30000.
+        heartbeat_interval_ms (int): The expected time in milliseconds
+            between heartbeats to the consumer coordinator when using
+            Kafka's group management facilities. Heartbeats are used to ensure
+            that the consumer's session stays active and to facilitate
+            rebalancing when new consumers join or leave the group. The
+            value must be set lower than session_timeout_ms, but typically
+            should be set no higher than 1/3 of that value. It can be
+            adjusted even lower to control the expected time for normal
+            rebalances. Default: 3000
+        receive_message_max_bytes (int): Maximum allowed network frame size.
+            Used to avoid OOM when decoding malformed network message header.
+            Default: 1000000.
+        receive_buffer_bytes (int): The size of the TCP receive buffer
+            (SO_RCVBUF) to use when reading data. Default: None (relies on
+            system defaults). The java client defaults to 32768.
+        send_buffer_bytes (int): The size of the TCP send buffer
+            (SO_SNDBUF) to use when sending data. Default: None (relies on
+            system defaults). The java client defaults to 131072.
+        socket_options (list): List of tuple-arguments to socket.setsockopt
+            to apply to broker connection sockets. Default:
+            [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]
+        consumer_timeout_ms (int): number of milliseconds to block during
+            message iteration before raising StopIteration (i.e., ending the
+            iterator). Default block forever [float('inf')].
+        security_protocol (str): Protocol used to communicate with brokers.
+            Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL.
+            Default: PLAINTEXT.
+        ssl_context (ssl.SSLContext): Pre-configured SSLContext for wrapping
+            socket connections. If provided, all other ssl_* configurations
+            will be ignored. Default: None.
+        ssl_check_hostname (bool): Flag to configure whether ssl handshake
+            should verify that the certificate matches the brokers hostname.
+            Default: True.
+        ssl_cafile (str): Optional filename of ca file to use in certificate
+            verification. Default: None.
+        ssl_certfile (str): Optional filename of file in pem format containing
+            the client certificate, as well as any ca certificates needed to
+            establish the certificate's authenticity. Default: None.
+        ssl_keyfile (str): Optional filename containing the client private key.
+            Default: None.
+        ssl_password (str): Optional password to be used when loading the
+            certificate chain. Default: None.
+        ssl_crlfile (str): Optional filename containing the CRL to check for
+            certificate expiration. By default, no CRL check is done. When
+            providing a file, only the leaf certificate will be checked against
+            this CRL. Default: None.
+        ssl_ciphers (str): optionally set the available ciphers for ssl
+            connections. It should be a string in the OpenSSL cipher list
+            format. If no cipher can be selected (because compile-time options
+            or other configuration forbids use of all the specified ciphers),
+            an ssl.SSLError will be raised. See ssl.SSLContext.set_ciphers
+        api_version (tuple): Specify which Kafka API version to use. If set to
+            None, the client will infer the broker version from the results of
+            ApiVersionsRequest API. For brokers earlier than 0.10, which do not
+            support the ApiVersionsRequest API, api_version is required.
+            Note: Dynamic version checking is performed eagerly during __init__
+            and can raise KafkaTimeoutError if no connection can be made before
+            timeout (see bootstrap_timeout_ms below).
+            Different versions enable different functionality.
+
+            Examples::
+
+                (4, 3) most recent broker release, enable all supported features
+                (0, 11) enables message format v2 (internal)
+                (0, 10, 0) enables sasl authentication and message format v1
+                (0, 9) enables full group coordination features with automatic
+                    partition assignment and rebalancing,
+                (0, 8, 2) enables kafka-storage offset commits with manual
+                    partition assignment only,
+                (0, 8, 1) enables zookeeper-storage offset commits with manual
+                    partition assignment only,
+                (0, 8, 0) enables basic functionality but requires manual
+                    partition assignment and offset management.
+
+            Default: None
+        bootstrap_timeout_ms (int): number of milliseconds to wait for first
+            successful cluster bootstrap. If provided, an attempt to bootstrap
+            will raise KafkaTimeoutError if it is unable to fetch cluster
+            metadata before the configured timeout. Note that bootstrap will
+            be called eagerly from __init__() if api_version is None.
+            Default: 30000
+        connections_max_idle_ms: Close idle connections after the number of
+            milliseconds specified by this config. The broker closes idle
+            connections after connections.max.idle.ms, so this avoids hitting
+            unexpected socket disconnected errors on the client.
+            Default: 540000
+        metric_reporters (list): A list of classes to use as metrics reporters.
+            Implementing the AbstractMetricsReporter interface allows plugging
+            in classes that will be notified of new metric creation. Default: []
+        metrics_enabled (bool): Whether to track metrics on this instance. Default True.
+        metrics_num_samples (int): The number of samples maintained to compute
+            metrics. Default: 2
+        metrics_sample_window_ms (int): The maximum age in milliseconds of
+            samples used to compute metrics. Default: 30000
+        selector (selectors.BaseSelector): Provide a specific selector
+            implementation to use for I/O multiplexing.
+            Default: selectors.DefaultSelector
+        exclude_internal_topics (bool): Whether records from internal topics
+            (such as offsets) should be exposed to the consumer. If set to True
+            the only way to receive records from an internal topic is
+            subscribing to it. Requires 0.10+ Default: True
+        sasl_mechanism (str): Authentication mechanism when security_protocol
+            is configured for SASL_PLAINTEXT or SASL_SSL. Valid values are:
+            PLAIN, GSSAPI, OAUTHBEARER, SCRAM-SHA-256, SCRAM-SHA-512.
+        sasl_plain_username (str): username for sasl PLAIN and SCRAM authentication.
+            Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms.
+        sasl_plain_password (str): password for sasl PLAIN and SCRAM authentication.
+            Required if sasl_mechanism is PLAIN or one of the SCRAM mechanisms.
+        sasl_kerberos_name (str or gssapi.Name): Constructed gssapi.Name for use with
+            sasl mechanism handshake. If provided, sasl_kerberos_service_name and
+            sasl_kerberos_domain name are ignored. Default: None.
+        sasl_kerberos_service_name (str): Service name to include in GSSAPI
+            sasl mechanism handshake. Default: 'kafka'
+        sasl_kerberos_domain_name (str): kerberos domain name to use in GSSAPI
+            sasl mechanism handshake. Default: one of bootstrap servers
+        sasl_oauth_token_provider (kafka.net.sasl.oauth.AbstractTokenProvider): OAuthBearer
+            token provider instance. Default: None
+        proxy_url (str): URL to proxy socket connections through. Supports SOCKS5 only.
+            Requires scheme:// (e.g., socks5://foo.bar/). Default: None
+        kafka_client (callable): Custom class / callable for creating KafkaNetClient instances
+
+    Note:
+        Configuration parameters are described in more detail at
+        https://kafka.apache.org/documentation/#consumerconfigs
+    """
+    DEFAULT_CONFIG = {
+        'bootstrap_servers': 'localhost',
+        'client_id': 'kafka-python-' + __version__,
+        'client_rack': '',
+        'group_id': None,
+        'group_instance_id': None,
+        'key_deserializer': None,
+        'value_deserializer': None,
+        'enable_incremental_fetch_sessions': True,
+        'fetch_max_wait_ms': 500,
+        'fetch_min_bytes': 1,
+        'fetch_max_bytes': 52428800,
+        'max_partition_fetch_bytes': 1 * 1024 * 1024,
+        'request_timeout_ms': 30000,
+        'retry_backoff_ms': 100,
+        'reconnect_backoff_ms': 50,
+        'reconnect_backoff_max_ms': 30000,
+        'max_in_flight_requests_per_connection': 5,
+        'auto_offset_reset': 'latest',
+        'enable_auto_commit': True,
+        'auto_commit_interval_ms': 5000,
+        'default_offset_commit_callback': lambda offsets, response: True,
+        'check_crcs': True,
+        'isolation_level': 'read_uncommitted',
+        'allow_auto_create_topics': True,
+        'metadata_max_age_ms': 5 * 60 * 1000,
+        'client_dns_lookup': 'use_all_dns_ips',
+        'partition_assignment_strategy': (RangePartitionAssignor, RoundRobinPartitionAssignor),
+        'max_poll_records': 500,
+        'max_poll_interval_ms': 300000,
+        'session_timeout_ms': 45000,
+        'heartbeat_interval_ms': 3000,
+        'receive_buffer_bytes': None,
+        'send_buffer_bytes': None,
+        'receive_message_max_bytes': 1000000,
+        'socket_options': [(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)],
+        'consumer_timeout_ms': float('inf'),
+        'security_protocol': 'PLAINTEXT',
+        'ssl_context': None,
+        'ssl_check_hostname': True,
+        'ssl_cafile': None,
+        'ssl_certfile': None,
+        'ssl_keyfile': None,
+        'ssl_crlfile': None,
+        'ssl_password': None,
+        'ssl_ciphers': None,
+        'api_version': None,
+        'bootstrap_timeout_ms': 30000,
+        'connections_max_idle_ms': 9 * 60 * 1000,
+        'metric_reporters': [],
+        'metrics_enabled': True,
+        'metrics_num_samples': 2,
+        'metrics_sample_window_ms': 30000,
+        'metric_group_prefix': 'consumer',
+        'selector': selectors.DefaultSelector,
+        'exclude_internal_topics': True,
+        'sasl_mechanism': None,
+        'sasl_plain_username': None,
+        'sasl_plain_password': None,
+        'sasl_kerberos_name': None,
+        'sasl_kerberos_service_name': 'kafka',
+        'sasl_kerberos_domain_name': None,
+        'sasl_oauth_token_provider': None,
+        'proxy_url': None,
+        'socks5_proxy': None,  # deprecated
+        'kafka_client': KafkaNetClient,
+    }
+    DEFAULT_SESSION_TIMEOUT_MS_PRE_KIP_735 = 30000
+
+    def __init__(self, *topics, **configs):
+        # Only check for extra config keys in top-level class
+        extra_configs = set(configs).difference(self.DEFAULT_CONFIG)
+        if extra_configs:
+            raise KafkaConfigurationError("Unrecognized configs: %s" % (extra_configs,))
+
+        self.config = copy.copy(self.DEFAULT_CONFIG)
+        self.config.update(configs)
+
+        deprecated = {'smallest': 'earliest', 'largest': 'latest'}
+        if self.config['auto_offset_reset'] in deprecated:
+            new_config = deprecated[self.config['auto_offset_reset']]
+            log.warning('use auto_offset_reset=%s (%s is deprecated)',
+                        new_config, self.config['auto_offset_reset'])
+            self.config['auto_offset_reset'] = new_config
+
+        connections_max_idle_ms = self.config['connections_max_idle_ms']
+        request_timeout_ms = self.config['request_timeout_ms']
+        fetch_max_wait_ms = self.config['fetch_max_wait_ms']
+        if not (fetch_max_wait_ms < request_timeout_ms < connections_max_idle_ms):
+            raise KafkaConfigurationError(
+                "connections_max_idle_ms ({}) must be larger than "
+                "request_timeout_ms ({}) which must be larger than "
+                "fetch_max_wait_ms ({})."
+                .format(connections_max_idle_ms, request_timeout_ms, fetch_max_wait_ms))
+
+        if self.config['metrics_enabled']:
+            metrics_tags = {'client-id': self.config['client_id']}
+            metric_config = MetricConfig(samples=self.config['metrics_num_samples'],
+                                         time_window_ms=self.config['metrics_sample_window_ms'],
+                                         tags=metrics_tags)
+            reporters = [reporter() for reporter in self.config['metric_reporters']]
+            self._metrics = Metrics(metric_config, reporters)
+        else:
+            self._metrics = None
+
+        self._client = self.config['kafka_client'](metrics=self._metrics, **self.config)
+        self._manager = self._client._manager
+        self._cluster = self._manager.cluster
+        self._net = self._manager._net
+
+        # Drive the IO loop on a background thread so coroutines (heartbeat,
+        # metadata refresh) make progress without depending on user-thread
+        # poll() timing.
+        self._net.start()
+
+        # We currently depend on eager-resolution of api_version.
+        # If it wasn't provided as a config option, we need to bootstrap
+        # to get it.
+        if self._manager.broker_version_data is None:
+            self._manager.bootstrap(self.config['bootstrap_timeout_ms'])
+        self.config['api_version'] = self._manager.broker_version
+
+        # Coordinator configurations are different for older brokers
+        # max_poll_interval_ms is not supported directly -- it must the be
+        # the same as session_timeout_ms. If the user provides one of them,
+        # use it for both.
+        user_supplied_session_timeout = 'session_timeout_ms' in configs
+        user_supplied_max_poll_interval = 'max_poll_interval_ms' in configs
+
+        if not user_supplied_session_timeout:
+            if self.config['api_version'] < (0, 10, 1) and user_supplied_max_poll_interval:
+                self.config['session_timeout_ms'] = self.config['max_poll_interval_ms']
+
+            elif self.config['api_version'] < (3, 0):
+                # Prior to 3.0 the broker-side default max session timeout was 30000
+                self.config['session_timeout_ms'] = self.DEFAULT_SESSION_TIMEOUT_MS_PRE_KIP_735
+
+        if not user_supplied_max_poll_interval:
+            if self.config['api_version'] < (0, 10, 1):
+                self.config['max_poll_interval_ms'] = self.config['session_timeout_ms']
+
+        if self.config['group_instance_id'] is not None:
+            if self.config['group_id'] is None:
+                raise KafkaConfigurationError("group_instance_id requires group_id")
+
+        self._subscription = SubscriptionState(self.config['auto_offset_reset'])
+        self._fetcher = Fetcher(
+            self._client, self._subscription, metrics=self._metrics, **self.config)
+        self._coordinator = ConsumerCoordinator(
+            self._client, self._subscription, metrics=self._metrics,
+            assignors=self.config['partition_assignment_strategy'],
+            **self.config)
+        self._closed = False
+        self._iterator = None
+        self._consumer_timeout = float('inf')
+
+        if topics:
+            self._subscription.subscribe(topics=topics)
+            self._cluster.set_topics(topics)
+
+    def _validate_group_instance_id(self, group_instance_id):
+        # See also kafka.util.ensure_valid_topic_name
+        if not group_instance_id or not isinstance(group_instance_id, str):
+            raise KafkaConfigurationError("group_instance_id must be non-empty string")
+        if group_instance_id in (".", ".."):
+            raise KafkaConfigurationError("group_instance_id cannot be \".\" or \"..\"")
+        if len(group_instance_id) > 249:
+            raise KafkaConfigurationError("group_instance_id can't be longer than 249 characters")
+        if not re.match(r'^[A-Za-z0-9\.\_\-]+$', group_instance_id):
+            raise KafkaConfigurationError("group_instance_id is illegal: it contains a character other than ASCII alphanumerics, '.', '_' and '-'")
+
+    def bootstrap_connected(self):
+        """Return True if the bootstrap is connected."""
+        return self._client.bootstrap_connected()
+
+    def bootstrap(self, timeout_ms=None, refresh=False):
+        """Block until the consumer has bootstrapped cluster metadata.
+
+        Bootstrap is otherwise driven lazily by the IO thread when
+        metadata is first needed (e.g. on the first ``poll()`` or
+        ``topics()`` call). Call this to deterministically wait for
+        ``cluster.brokers()`` to be populated -- useful in tests and at
+        startup when the caller wants metadata available before issuing
+        the first request.
+
+        Arguments:
+            timeout_ms (int, optional): Maximum time to wait, in
+                milliseconds. ``None`` waits indefinitely. Default: None.
+            refresh (bool): If True, force a new bootstrap even when one
+                has already succeeded. Default: False.
+
+        Raises:
+            KafkaTimeoutError: If bootstrap does not complete within
+                ``timeout_ms``.
+        """
+        self._manager.bootstrap(timeout_ms=timeout_ms, refresh=refresh)
+
+    def assign(self, partitions):
+        """Manually assign a list of TopicPartitions to this consumer.
+
+        Arguments:
+            partitions (list of TopicPartition): Assignment for this instance.
+
+        Raises:
+            IllegalStateError: If consumer has already called
+            :meth:`~kafka.KafkaConsumer.subscribe`.
+
+        Warning:
+            It is not possible to use both manual partition assignment with
+            :meth:`~kafka.KafkaConsumer.assign` and group assignment with
+            :meth:`~kafka.KafkaConsumer.subscribe`.
+
+        Note:
+            This interface does not support incremental assignment and will
+            replace the previous assignment (if there was one).
+
+        Note:
+            Manual topic assignment through this method does not use the
+            consumer's group management functionality. As such, there will be
+            no rebalance operation triggered when group membership or cluster
+            and topic metadata change.
+        """
+        if not partitions:
+            self.unsubscribe()
+        else:
+            # make sure the offsets of topic partitions the consumer is unsubscribing from
+            # are committed since there will be no following rebalance
+            self._coordinator.maybe_auto_commit_offsets_now()
+            self._subscription.assign_from_user(partitions)
+            self._cluster.set_topics([tp.topic for tp in partitions])
+            log.debug("Subscribed to partition(s): %s", partitions)
+
+    def assignment(self):
+        """Get the TopicPartitions currently assigned to this consumer.
+
+        If partitions were directly assigned using
+        :meth:`~kafka.KafkaConsumer.assign`, then this will simply return the
+        same partitions that were previously assigned.  If topics were
+        subscribed using :meth:`~kafka.KafkaConsumer.subscribe`, then this will
+        give the set of topic partitions currently assigned to the consumer
+        (which may be None if the assignment hasn't happened yet, or if the
+        partitions are in the process of being reassigned).
+
+        Returns:
+            set: {TopicPartition, ...}
+        """
+        return self._subscription.assigned_partitions()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        # Skip autocommit when the `with` block exited via exception: we
+        # can't know whether the in-flight record was successfully
+        # processed, so committing its offset would risk silently dropping
+        # an unprocessed message on next start. Clean exits commit normally.
+        self.close(autocommit=exc_type is None)
+
+    def close(self, autocommit=True, timeout_ms=None):
+        """Close the consumer, attempting any needed cleanup within a bounded
+        wall-clock budget.
+
+        Keyword Arguments:
+            autocommit (bool): If auto-commit is configured for this consumer,
+                this optional flag causes the consumer to attempt to commit any
+                pending consumed offsets prior to close. Default: True
+            timeout_ms (num, optional): Milliseconds to wait for auto-commit
+                and other in-flight close work. ``None`` falls back to
+                ``request_timeout_ms`` so close() can never hang indefinitely
+                even if the coordinator is unreachable. Default: None
+        """
+        if self._closed:
+            return
+        log.debug("Closing the KafkaConsumer.")
+        self._closed = True
+        # Cap the auto-commit retry loop -- ``_commit_offsets_sync_async``
+        # retries forever when given ``None`` (Timer(None).expired is always
+        # False), so a coordinator that's down or stuck would otherwise hang
+        # close() until process exit.
+        if timeout_ms is None:
+            timeout_ms = self.config['request_timeout_ms']
+        self._coordinator.close(autocommit=autocommit, timeout_ms=timeout_ms)
+        if self._metrics:
+            self._metrics.close()
+        self._client.close()
+        try:
+            self.config['key_deserializer'].close()
+        except AttributeError:
+            pass
+        try:
+            self.config['value_deserializer'].close()
+        except AttributeError:
+            pass
+        log.debug("The KafkaConsumer has closed.")
+
+    def commit_async(self, offsets=None, callback=None):
+        """Commit offsets to kafka asynchronously, optionally firing callback.
+
+        This commits offsets only to Kafka. The offsets committed using this API
+        will be used on the first fetch after every rebalance and also on
+        startup. As such, if you need to store offsets in anything other than
+        Kafka, this API should not be used. To avoid re-processing the last
+        message read if a consumer is restarted, the committed offset should be
+        the next message your application should consume, i.e.: last_offset + 1.
+
+        This is an asynchronous call and will not block. Any errors encountered
+        are either passed to the callback (if provided) or discarded.
+
+        Arguments:
+            offsets (dict, optional): {TopicPartition: OffsetAndMetadata} dict
+                to commit with the configured group_id. Defaults to currently
+                consumed offsets for all subscribed partitions.
+            callback (callable, optional): Called as callback(offsets, response)
+                with response as either an Exception or an OffsetCommitResponse
+                struct. This callback can be used to trigger custom actions when
+                a commit request completes.
+
+        Raises:
+            IncompatibleBrokerVersion: if broker version < 0.8.1
+            IllegalStateError: if group_id is None
+
+        Returns:
+            kafka.future.Future
+        """
+        if self.config['api_version'] < (0, 8, 1):
+            raise Errors.IncompatibleBrokerVersion('Requires >= Kafka 0.8.1')
+        if self.config['group_id'] is None:
+            raise Errors.IllegalStateError('Requires group_id')
+        if offsets is None:
+            offsets = self._subscription.all_consumed_offsets()
+        log.debug("Committing offsets: %s", offsets)
+        future = self._coordinator.commit_offsets_async(
+            offsets, callback=callback)
+        return future
+
+    def commit(self, offsets=None, timeout_ms=None):
+        """Commit offsets to kafka, blocking until success or error.
+
+        This commits offsets only to Kafka. The offsets committed using this API
+        will be used on the first fetch after every rebalance and also on
+        startup. As such, if you need to store offsets in anything other than
+        Kafka, this API should not be used. To avoid re-processing the last
+        message read if a consumer is restarted, the committed offset should be
+        the next message your application should consume, i.e.: last_offset + 1.
+
+        Blocks until either the commit succeeds or an unrecoverable error is
+        encountered (in which case it is thrown to the caller).
+
+        Currently only supports kafka-topic offset storage (not zookeeper).
+
+        Arguments:
+            offsets (dict, optional): {TopicPartition: OffsetAndMetadata} dict
+                to commit with the configured group_id. Defaults to currently
+                consumed offsets for all subscribed partitions.
+
+        Raises:
+            IncompatibleBrokerVersion: if broker version < 0.8.1
+            IllegalStateError: if group_id is None
+        """
+        if self.config['api_version'] < (0, 8, 1):
+            raise Errors.IncompatibleBrokerVersion('Requires >= Kafka 0.8.1')
+        if self.config['group_id'] is None:
+            raise Errors.IllegalStateError('Requires group_id')
+        if offsets is None:
+            offsets = self._subscription.all_consumed_offsets()
+        self._coordinator.commit_offsets_sync(offsets, timeout_ms=timeout_ms)
+
+    def group_metadata(self):
+        """Return a snapshot of this consumer's group membership (KIP-447).
+
+        Pass the result to KafkaProducer.send_offsets_to_transaction() so the
+        broker can fence stale instances of this consumer when committing
+        offsets inside a transaction. The snapshot is always safe to call:
+        if no group_id is configured (manual assignment) the returned
+        ConsumerGroupMetadata has group_id=None.
+
+        Returns:
+            ConsumerGroupMetadata
+        """
+        if self.config['group_id'] is None:
+            return ConsumerGroupMetadata(group_id=None)
+        return self._coordinator.group_metadata()
+
+    def committed(self, partition, metadata=False, timeout_ms=None):
+        """Get the last committed offset for the given partition.
+
+        This offset will be used as the position for the consumer
+        in the event of a failure.
+
+        This call will block to do a remote call to get the latest committed
+        offsets from the server.
+
+        Arguments:
+            partition (TopicPartition): The partition to check.
+            metadata (bool, optional): If True, return OffsetAndMetadata struct
+                instead of offset int. Default: False.
+
+        Returns:
+            The last committed offset (int or OffsetAndMetadata), or None if there was no prior commit.
+
+        Raises:
+            IncompatibleBrokerVersion: if broker version < 0.8.1
+            IllegalStateError: if group_id is None
+            TypeError: if partition is not TopicPartition
+            KafkaTimeoutError: if timeout_ms provided
+            BrokerResponseError: if OffsetFetchRequest raises an error.
+        """
+        if self.config['api_version'] < (0, 8, 1):
+            raise Errors.IncompatibleBrokerVersion('Requires >= Kafka 0.8.1')
+        if self.config['group_id'] is None:
+            raise Errors.IllegalStateError('Requires group_id')
+        if not isinstance(partition, TopicPartition):
+            raise TypeError('partition must be a TopicPartition namedtuple')
+        committed = self._coordinator.fetch_committed_offsets([partition], timeout_ms=timeout_ms)
+        if partition not in committed:
+            return None
+        return committed[partition] if metadata else committed[partition].offset
+
+    def _fetch_all_topic_metadata(self):
+        """A blocking call that fetches topic metadata for all topics in the
+        cluster that the user is authorized to view.
+        """
+        if self._cluster.metadata_refresh_in_progress:
+            future = self._cluster.request_update()
+            self._net.run(self._manager.wait_for, future, None)
+        stash = self._cluster.need_all_topic_metadata
+        self._cluster.need_all_topic_metadata = True
+        future = self._cluster.request_update()
+        self._net.run(self._manager.wait_for, future, None)
+        self._cluster.need_all_topic_metadata = stash
+
+    def topics(self):
+        """Get all topics the user is authorized to view.
+        This will always issue a remote call to the cluster to fetch the latest
+        information.
+
+        Returns:
+            set: topics
+        """
+        self._fetch_all_topic_metadata()
+        return self._cluster.topics()
+
+    def partitions_for_topic(self, topic):
+        """This method first checks the local metadata cache for information
+        about the topic. If the topic is not found (either because the topic
+        does not exist, the user is not authorized to view the topic, or the
+        metadata cache is not populated), then it will issue a metadata update
+        call to the cluster.
+
+        Arguments:
+            topic (str): Topic to check.
+
+        Returns:
+            set: Partition ids
+        """
+        partitions = self._cluster.partitions_for_topic(topic)
+        if partitions is None:
+            self._fetch_all_topic_metadata()
+            partitions = self._cluster.partitions_for_topic(topic)
+        return partitions or set()
+
+    def poll(self, timeout_ms=0, max_records=None, update_offsets=True):
+        """Fetch data from assigned topics / partitions.
+
+        Records are fetched and returned in batches by topic-partition.
+        On each poll, consumer will try to use the last consumed offset as the
+        starting offset and fetch sequentially. The last consumed offset can be
+        manually set through :meth:`~kafka.KafkaConsumer.seek` or automatically
+        set as the last committed offset for the subscribed list of partitions.
+
+        Incompatible with iterator interface -- use one or the other, not both.
+
+        Arguments:
+            timeout_ms (int, optional): Milliseconds spent waiting in poll if
+                data is not available in the buffer. If 0, returns immediately
+                with any records that are available currently in the buffer,
+                else returns empty. Must not be negative. Default: 0
+            max_records (int, optional): The maximum number of records returned
+                in a single call to :meth:`~kafka.KafkaConsumer.poll`.
+                Default: Inherit value from max_poll_records.
+
+        Raises:
+            ValueError: if timeout is < 0 or max_records <= 0.
+            TypeError: if max_records is not int.
+            IllegalStateError: if consumer already closed.
+
+        Returns:
+            dict[TopicPartition, list[ConsumerRecord]]: records since the last
+                fetch for the subscribed list of topics and partitions.
+        """
+        # Note: update_offsets is an internal-use only argument. It is used to
+        # support the python iterator interface, and which wraps consumer.poll()
+        # and requires that the partition offsets tracked by the fetcher are not
+        # updated until the iterator returns each record to the user. As such,
+        # the argument is not documented and should not be relied on by library
+        # users to not break in the future.
+        if timeout_ms < 0:
+            raise ValueError('Timeout must not be negative')
+        if max_records is None:
+            max_records = self.config['max_poll_records']
+        if not isinstance(max_records, int):
+            raise TypeError('max_records must be an integer')
+        if max_records <= 0:
+            raise ValueError('max_records must be positive')
+        if self._closed:
+            raise Errors.IllegalStateError('KafkaConsumer is closed')
+
+        # Poll for new data until the timeout expires
+        timer = Timer(timeout_ms)
+        while not self._closed:
+            records = self._poll_once(timer, max_records, update_offsets=update_offsets)
+            if records:
+                return records
+            elif timer.expired:
+                break
+        return {}
+
+    def _poll_once(self, timer, max_records, update_offsets=True):
+        """Do one round of polling. In addition to checking for new data, this does
+        any needed heart-beating, auto-commits, and offset updates.
+
+        Arguments:
+            timer (Timer): The maximum time in milliseconds to block.
+
+        Returns:
+            dict: Map of topic to list of records (may be empty).
+        """
+        if not self._coordinator.poll(timeout_ms=timer.timeout_ms):
+            log.debug('poll: timeout during coordinator.poll(); returning early')
+            return {}
+
+        self._refresh_committed_offsets(timeout_ms=timer.timeout_ms)
+        # Fire-and-forget: kicks ListOffsets reset for any remaining partitions.
+        # _reset_offsets_async self-drives metadata refresh + retry-backoff
+        # within request_timeout_ms.
+        self._fetcher.reset_offsets_if_needed()
+        # KIP-320: mark any positions whose cluster leader epoch has advanced
+        # beyond the position's epoch, then drive OffsetForLeaderEpoch
+        # validation in the background. Truncation surfaces on the next call.
+        self._fetcher.maybe_validate_positions()
+        self._fetcher.validate_offsets_if_needed()
+
+        # Cap the fetch wait by the heartbeat deadline so we don't block past
+        # when the coordinator wants to send the next heartbeat.
+        poll_timeout_ms = timer.timeout_ms
+        if self.config['group_id'] is not None:
+            poll_timeout_ms = min(poll_timeout_ms, self._coordinator.time_to_next_poll() * 1000)
+
+        records, idle = self._fetcher.fetch_records(
+            max_records, update_offsets=update_offsets, timeout_ms=poll_timeout_ms)
+        if not records and idle and poll_timeout_ms > 0:
+            # Nothing fetchable and nothing in flight (no assignment, all
+            # paused, or no resolvable positions). Sleep up to poll_timeout_ms
+            # to avoid busy-looping; poll_timeout_ms is already capped by the
+            # coordinator's next-action deadline.
+            poll_timeout_ms = min(poll_timeout_ms, self.config['retry_backoff_ms'])
+            time.sleep(poll_timeout_ms / 1000)
+        return records
+
+    def position(self, partition, timeout_ms=None):
+        """Get the offset of the next record that will be fetched
+
+        Arguments:
+            partition (TopicPartition): Partition to check
+
+        Raises:
+            TypeError: if partition is not a TopicPartition.
+            IllegalStateError: if partition is not assigned.
+
+        Returns:
+            int: Offset or None
+        """
+        if not isinstance(partition, TopicPartition):
+            raise TypeError('partition must be a TopicPartition namedtuple')
+        if not self._subscription.is_assigned(partition):
+            raise Errors.IllegalStateError('Partition is not assigned')
+
+        timer = Timer(timeout_ms)
+        # Phase 1: blocking refresh of committed offsets (network round-trip
+        # to the coordinator) and CPU-only marking of remaining partitions
+        # for reset by the configured policy.
+        self._refresh_committed_offsets(timeout_ms=timer.timeout_ms)
+        # Phase 2: ListOffsets reset is async; await its in-flight Task. The
+        # task's own loop is bounded by timer.timeout_ms so it doesn't run
+        # past the user's deadline.
+        reset_task = self._fetcher.reset_offsets_if_needed(timeout_ms=timer.timeout_ms)
+        if reset_task is not None and not timer.expired:
+            try:
+                self._net.run(self._manager.wait_for, reset_task, timer.timeout_ms)
+            except Errors.KafkaTimeoutError:
+                pass
+        # Phase 3 (KIP-320): mark any positions whose cluster leader epoch
+        # has advanced beyond the position's epoch and await the validation
+        # RPC. Surfaces LogTruncationError to the caller if truncation is
+        # detected (and auto_offset_reset is NONE).
+        self._fetcher.maybe_validate_positions()
+        validation_task = self._fetcher.validate_offsets_if_needed(
+            timeout_ms=timer.timeout_ms)
+        if validation_task is not None and not timer.expired:
+            try:
+                self._net.run(self._manager.wait_for, validation_task, timer.timeout_ms)
+            except Errors.KafkaTimeoutError:
+                pass
+        position = self._subscription.assignment[partition].position
+        if position is not None:
+            return position.offset
+
+    def highwater(self, partition):
+        """Last known highwater offset for a partition.
+
+        A highwater offset is the offset that will be assigned to the next
+        message that is produced. It may be useful for calculating lag, by
+        comparing with the reported position. Note that both position and
+        highwater refer to the *next* offset -- i.e., highwater offset is
+        one greater than the newest available message.
+
+        Highwater offsets are returned in FetchResponse messages, so will
+        not be available if no FetchRequests have been sent for this partition
+        yet.
+
+        Arguments:
+            partition (TopicPartition): Partition to check
+
+        Raises:
+            TypeError: if partition is not a TopicPartition.
+            IllegalStateError: if partition is not assigned.
+
+        Returns:
+            int or None: Offset if available
+        """
+        if not isinstance(partition, TopicPartition):
+            raise TypeError('partition must be a TopicPartition namedtuple')
+        if not self._subscription.is_assigned(partition):
+            raise Errors.IllegalStateError('Partition is not assigned')
+        return self._subscription.assignment[partition].highwater
+
+    def pause(self, *partitions):
+        """Suspend fetching from the requested partitions.
+
+        Future calls to :meth:`~kafka.KafkaConsumer.poll` will not return any
+        records from these partitions until they have been resumed using
+        :meth:`~kafka.KafkaConsumer.resume`.
+
+        Note: This method does not affect partition subscription. In particular,
+        it does not cause a group rebalance when automatic assignment is used.
+
+        Arguments:
+            *partitions (TopicPartition): Partitions to pause.
+        """
+        if not all([isinstance(p, TopicPartition) for p in partitions]):
+            raise TypeError('partitions must be TopicPartition namedtuples')
+        for partition in partitions:
+            log.debug("Pausing partition %s", partition)
+            self._subscription.pause(partition)
+        # Because the iterator checks is_fetchable() on each iteration
+        # we expect pauses to get handled automatically and therefore
+        # we do not need to reset the full iterator (forcing a full refetch)
+
+    def paused(self):
+        """Get the partitions that were previously paused using
+        :meth:`~kafka.KafkaConsumer.pause`.
+
+        Returns:
+            set: {partition (TopicPartition), ...}
+        """
+        return self._subscription.paused_partitions()
+
+    def resume(self, *partitions):
+        """Resume fetching from the specified (paused) partitions.
+
+        Arguments:
+            *partitions (TopicPartition): Partitions to resume.
+        """
+        if not all([isinstance(p, TopicPartition) for p in partitions]):
+            raise TypeError('partitions must be TopicPartition namedtuples')
+        for partition in partitions:
+            log.debug("Resuming partition %s", partition)
+            self._subscription.resume(partition)
+
+    def seek(self, partition, offset):
+        """Manually specify the fetch offset for a TopicPartition.
+
+        Overrides the fetch offsets that the consumer will use on the next
+        :meth:`~kafka.KafkaConsumer.poll`. If this API is invoked for the same
+        partition more than once, the latest offset will be used on the next
+        :meth:`~kafka.KafkaConsumer.poll`.
+
+        Note: You may lose data if this API is arbitrarily used in the middle of
+        consumption to reset the fetch offsets.
+
+        Arguments:
+            partition (TopicPartition): Partition for seek operation
+            offset (int): Message offset in partition
+
+        Raises:
+            ValueError: If offset is not an int >= 0; or if partition is not
+                currently assigned.
+        """
+        if not isinstance(partition, TopicPartition):
+            raise TypeError('partition must be a TopicPartition namedtuple')
+        if not isinstance(offset, int) or offset < 0:
+            raise ValueError('Offset must be int >= 0')
+        if partition not in self._subscription.assigned_partitions():
+            raise ValueError('Unassigned partition')
+        log.debug("Seeking to offset %s for partition %s", offset, partition)
+        self._subscription.assignment[partition].seek(offset)
+        self._iterator = None
+
+    def seek_to_beginning(self, *partitions):
+        """Seek to the oldest available offset for partitions.
+
+        Arguments:
+            *partitions: Optionally provide specific TopicPartitions, otherwise
+                default to all assigned partitions.
+
+        Raises:
+            ValueError: If any partition is not currently assigned, or if
+                no partitions are assigned.
+        """
+        if not all([isinstance(p, TopicPartition) for p in partitions]):
+            raise TypeError('partitions must be TopicPartition namedtuples')
+        if not partitions:
+            partitions = self._subscription.assigned_partitions()
+            if not partitions:
+                raise ValueError('No partitions are currently assigned')
+        else:
+            for p in partitions:
+                if p not in self._subscription.assigned_partitions():
+                    raise ValueError('Unassigned partition: %s' % (p,))
+
+        for tp in partitions:
+            log.debug("Seeking to beginning of partition %s", tp)
+            self._subscription.request_offset_reset(tp, OffsetResetStrategy.EARLIEST)
+        self._iterator = None
+
+    def seek_to_end(self, *partitions):
+        """Seek to the most recent available offset for partitions.
+
+        Arguments:
+            *partitions: Optionally provide specific TopicPartitions, otherwise
+                default to all assigned partitions.
+
+        Raises:
+            ValueError: If any partition is not currently assigned, or if
+                no partitions are assigned.
+        """
+        if not all([isinstance(p, TopicPartition) for p in partitions]):
+            raise TypeError('partitions must be TopicPartition namedtuples')
+        if not partitions:
+            partitions = self._subscription.assigned_partitions()
+            if not partitions:
+                raise ValueError('No partitions are currently assigned')
+        else:
+            for p in partitions:
+                if p not in self._subscription.assigned_partitions():
+                    raise ValueError('Unassigned partition: %s' % (p,))
+
+        for tp in partitions:
+            log.debug("Seeking to end of partition %s", tp)
+            self._subscription.request_offset_reset(tp, OffsetResetStrategy.LATEST)
+        self._iterator = None
+
+    def subscribe(self, topics=(), pattern=None, listener=None):
+        """Subscribe to a list of topics, or a topic regex pattern.
+
+        Partitions will be dynamically assigned via a group coordinator.
+        Topic subscriptions are not incremental: this list will replace the
+        current assignment (if there is one).
+
+        This method is incompatible with :meth:`~kafka.KafkaConsumer.assign`.
+
+        Arguments:
+            topics (list): List of topics for subscription.
+            pattern (str): Pattern to match available topics. You must provide
+                either topics or pattern, but not both.
+            listener (ConsumerRebalanceListener): Optionally include listener
+                callback, which will be called before and after each rebalance
+                operation.
+
+                As part of group management, the consumer will keep track of the
+                list of consumers that belong to a particular group and will
+                trigger a rebalance operation if one of the following events
+                trigger:
+
+                * Number of partitions change for any of the subscribed topics
+                * Topic is created or deleted
+                * An existing member of the consumer group dies
+                * A new member is added to the consumer group
+
+                When any of these events are triggered, the provided listener
+                will be invoked first to indicate that the consumer's assignment
+                has been revoked, and then again when the new assignment has
+                been received. Note that this listener will immediately override
+                any listener set in a previous call to subscribe. It is
+                guaranteed, however, that the partitions revoked/assigned
+                through this interface are from topics subscribed in this call.
+
+        Raises:
+            IllegalStateError: If called after previously calling
+                :meth:`~kafka.KafkaConsumer.assign`.
+            ValueError: If neither topics or pattern is provided.
+            TypeError: If listener is not a ConsumerRebalanceListener.
+        """
+        # SubscriptionState handles error checking
+        self._subscription.subscribe(topics=topics,
+                                     pattern=pattern,
+                                     listener=listener)
+
+        # Regex will need all topic metadata
+        if pattern is not None:
+            self._cluster.need_all_topic_metadata = True
+            self._cluster.set_topics([])
+            self._cluster.request_update()
+            log.debug("Subscribed to topic pattern: %s", pattern)
+        else:
+            self._cluster.need_all_topic_metadata = False
+            self._cluster.set_topics(self._subscription.group_subscription())
+            log.debug("Subscribed to topic(s): %s", topics)
+
+    def subscription(self):
+        """Get the current topic subscription.
+
+        Returns:
+            set: {topic, ...}
+        """
+        if self._subscription.subscription is None:
+            return None
+        return self._subscription.subscription.copy()
+
+    def unsubscribe(self):
+        """Unsubscribe from all topics and clear all assigned partitions."""
+        # make sure the offsets of topic partitions the consumer is unsubscribing from
+        # are committed since there will be no following rebalance
+        self._coordinator.maybe_auto_commit_offsets_now()
+        self._subscription.unsubscribe()
+        if self.config['api_version'] >= (0, 9):
+            self._coordinator.maybe_leave_group()
+        self._cluster.need_all_topic_metadata = False
+        self._cluster.set_topics([])
+        log.debug("Unsubscribed all topics or patterns and assigned partitions")
+        self._iterator = None
+
+    def metrics(self, raw=False):
+        """Get metrics on consumer performance.
+
+        This is ported from the Java Consumer, for details see:
+        https://kafka.apache.org/documentation/#consumer_monitoring
+
+        Warning:
+            This is an unstable interface. It may change in future
+            releases without warning.
+        """
+        if not self._metrics:
+            return
+        if raw:
+            return self._metrics.metrics.copy()
+
+        metrics = {}
+        for k, v in self._metrics.metrics.copy().items():
+            if k.group not in metrics:
+                metrics[k.group] = {}
+            if k.name not in metrics[k.group]:
+                metrics[k.group][k.name] = {}
+            metrics[k.group][k.name] = v.value()
+        return metrics
+
+    def offsets_for_times(self, timestamps, timeout_ms=None):
+        """Look up the offsets for the given partitions by timestamp. The
+        returned offset for each partition is the earliest offset whose
+        timestamp is greater than or equal to the given timestamp in the
+        corresponding partition.
+
+        This is a blocking call. The consumer does not have to be assigned the
+        partitions.
+
+        If the message format version in a partition is before 0.10.0, i.e.
+        the messages do not have timestamps, ``None`` will be returned for that
+        partition. ``None`` will also be returned for the partition if there
+        are no messages in it.
+
+        Note: This method may block indefinitely if the partition does not exist
+            and no timeout_ms provided.
+
+        Arguments:
+            timestamps (dict): ``{TopicPartition: int}`` mapping from partition
+                to the timestamp to look up. Unit should be milliseconds since
+                beginning of the epoch (midnight Jan 1, 1970 (UTC))
+            timeout_ms (int, optional): Milliseconds to block fetching offsets.
+
+        Returns:
+            ``{TopicPartition: OffsetAndTimestamp}``: mapping from partition
+            to the offset and timestamp of the first message with timestamp
+            greater than or equal to the target timestamp.
+
+        Raises:
+            ValueError: If the target timestamp is negative
+            UnsupportedVersionError: If the broker does not support looking
+                up the offsets by timestamp.
+            KafkaTimeoutError: If fetch failed in request_timeout_ms
+        """
+        timeout_ms = self.config['request_timeout_ms'] if timeout_ms is None else timeout_ms
+        for tp, ts in timestamps.items():
+            timestamps[tp] = int(ts)
+            if ts < 0:
+                raise ValueError(
+                    "The target time for partition {} is {}. The target time "
+                    "cannot be negative.".format(tp, ts))
+        return self._fetcher.offsets_by_times(timestamps, timeout_ms)
+
+    def beginning_offsets(self, partitions, timeout_ms=None):
+        """Get the first offset for the given partitions.
+
+        This method does not change the current consumer position of the
+        partitions.
+
+        Note: This method may block indefinitely if the partition does not exist
+            and no timeout_ms provided.
+
+        Arguments:
+            partitions (list): List of TopicPartition instances to fetch
+                offsets for.
+            timeout_ms (int, optional): Milliseconds to block fetching offsets.
+
+        Returns:
+            ``{TopicPartition: int}``: The earliest available offsets for the
+            given partitions.
+
+        Raises:
+            UnsupportedVersionError: If the broker does not support looking
+                up the offsets by timestamp.
+            KafkaTimeoutError: If fetch failed in timeout_ms.
+        """
+        timeout_ms = self.config['request_timeout_ms'] if timeout_ms is None else timeout_ms
+        offsets = self._fetcher.beginning_offsets(partitions, timeout_ms)
+        return offsets
+
+    def end_offsets(self, partitions, timeout_ms=None):
+        """Get the last offset for the given partitions. The last offset of a
+        partition is the offset of the upcoming message, i.e. the offset of the
+        last available message + 1.
+
+        This method does not change the current consumer position of the
+        partitions.
+
+        Note: This method may block indefinitely if the partition does not exist
+            and no timeout_ms provided.
+
+        Arguments:
+            partitions (list): List of TopicPartition instances to fetch
+                offsets for.
+            timeout_ms (int, optional): Milliseconds to block fetching offsets.
+
+        Returns:
+            ``{TopicPartition: int}``: The end offsets for the given partitions.
+
+        Raises:
+            UnsupportedVersionError: If the broker does not support looking
+                up the offsets by timestamp.
+            KafkaTimeoutError: If fetch failed in timeout_ms
+        """
+        timeout_ms = self.config['request_timeout_ms'] if timeout_ms is None else timeout_ms
+        offsets = self._fetcher.end_offsets(partitions, timeout_ms)
+        return offsets
+
+    def _refresh_committed_offsets(self, timeout_ms=None):
+        """Refresh committed offsets for partitions still needing a position
+        and mark any remaining partitions for reset by the configured policy.
+
+        This is the synchronous half of position resolution: a network
+        round-trip to the coordinator (timer-bounded), followed by
+        ``reset_missing_positions`` which is CPU-only. Partitions whose
+        position can be filled from a committed offset are filled here;
+        partitions without a committed offset are flagged for reset.
+
+        Callers that also want the reset to complete should follow up with
+        ``self._fetcher.reset_offsets_if_needed()`` and either await the
+        returned Task (e.g. via ``manager.wait_for``) or fire-and-forget.
+
+        Arguments:
+            timeout_ms (int, optional): Milliseconds to block refreshing
+                committed offsets.
+
+        Returns:
+            bool: True if all assigned partitions now have a fetch position
+                (no further reset needed); False if a reset is still
+                pending or the committed-offset refresh timed out.
+
+        Raises:
+            NoOffsetForPartitionError: If no offset is stored for a given
+                partition and no offset reset policy is defined.
+        """
+        if self._subscription.has_all_fetch_positions():
+            return True
+
+        if (self.config['api_version'] >= (0, 8, 1) and
+            self.config['group_id'] is not None):
+            # If there are any partitions which do not have a valid position and are not
+            # awaiting reset, then we need to fetch committed offsets. We will only do a
+            # coordinator lookup if there are partitions which have missing positions, so
+            # a consumer with manually assigned partitions can avoid a coordinator dependence
+            # by always ensuring that assigned partitions have an initial position.
+            if not self._coordinator.refresh_committed_offsets_if_needed(timeout_ms=timeout_ms):
+                return False
+
+        # If there are partitions still needing a position and a reset policy is defined,
+        # mark them for reset using the default policy. If no reset strategy is defined and
+        # there are partitions with a missing position, then we will raise an exception.
+        self._subscription.reset_missing_positions()
+        return self._subscription.has_all_fetch_positions()
+
+    def _message_generator_v2(self):
+        timeout_ms = 1000 * max(0, self._consumer_timeout - time.monotonic())
+        record_map = self.poll(timeout_ms=timeout_ms, update_offsets=False)
+        for tp, records in record_map.items():
+            # Generators are stateful, and it is possible that the tp / records
+            # here may become stale during iteration -- i.e., we seek to a
+            # different offset, pause consumption, or lose assignment.
+            for record in records:
+                # is_fetchable(tp) should handle assignment changes and offset
+                # resets; for all other changes (e.g., seeks) we'll rely on the
+                # outer function destroying the existing iterator/generator
+                # via self._iterator = None
+                if not self._subscription.is_fetchable(tp):
+                    log.debug("Not returning fetched records for partition %s"
+                              " since it is no longer fetchable", tp)
+                    break
+                self._subscription.assignment[tp].position = OffsetAndMetadata(
+                    record.offset + 1, '', record.leader_epoch)
+                yield record
+
+    def __iter__(self):  # pylint: disable=non-iterator-returned
+        return self
+
+    def __next__(self):
+        if self._closed:
+            raise StopIteration('KafkaConsumer closed')
+        self._set_consumer_timeout()
+        while time.monotonic() < self._consumer_timeout:
+            if not self._iterator:
+                self._iterator = self._message_generator_v2()
+            try:
+                return next(self._iterator)
+            except StopIteration:
+                self._iterator = None
+        raise StopIteration()
+
+    def _set_consumer_timeout(self):
+        # consumer_timeout_ms can be used to stop iteration early
+        if self.config['consumer_timeout_ms'] >= 0:
+            self._consumer_timeout = time.monotonic() + (
+                self.config['consumer_timeout_ms'] / 1000.0)