kafka-python 3.0.2__py3-none-any.whl → 3.0.5__py3-none-any.whl

kafka/cli/admin/cluster/versions.py

@@ -30,4 +30,4 @@ class GetBrokerVersion:
     def command(cls, client, args):
         broker_id = int(args.broker)
         bvd = client.get_broker_version_data(broker_id)
-        return {broker_id: '.'.join(map(str, bvd.broker_version))}
+        return {broker_id: bvd.broker_version_str}

kafka/cli/common.py

@@ -1,4 +1,5 @@
 import logging
+import logging.config
 
 
 def add_connect_cli_args(parser, bootstrap_required=True):
@@ -37,9 +38,13 @@ def build_kwargs(props):
 def build_connect_kwargs(config):
     if not config.bootstrap_servers:
         raise ValueError('python -m kafka: error: the following arguments are required: -b/--bootstrap-servers')
+    # Accept both repeated -b flags and comma-separated lists within a single flag
+    bootstrap_servers = []
+    for entry in config.bootstrap_servers:
+        bootstrap_servers.extend(s.strip() for s in entry.split(',') if s.strip())
     kwargs = build_kwargs(config.extra_config)
     kwargs.update({
-        'bootstrap_servers': config.bootstrap_servers,
+        'bootstrap_servers': bootstrap_servers,
         'security_protocol': config.security_protocol,
         'sasl_mechanism': config.sasl_mechanism,
         'sasl_plain_username': config.sasl_user,
@@ -59,12 +64,60 @@ def add_logging_cli_args(parser):
     logging_group.add_argument(
         '-D', '--disable-logger', type=str, action='append',
         help='disable a specific logger. Can be provided multiple times.')
+    logging_group.add_argument(
+        '--log-format', type=str, default=None,
+        help='log message format string, passed to logging.Formatter')
+    logging_group.add_argument(
+        '--log-date-format', type=str, default=None,
+        help='log date format string, passed to logging.Formatter')
+    logging_group.add_argument(
+        '--log-file', type=str, default=None,
+        help='write logs to this file instead of stderr')
+    logging_group.add_argument(
+        '--log-config', type=str, default=None,
+        help='path to a logging configuration file for full control over handlers, '
+             'formatters, etc. A .json (or .yaml/.yml, if PyYAML is installed) file is '
+             'loaded as a logging.config.dictConfig; any other extension is loaded as a '
+             'logging.config.fileConfig. The file owns handlers/formatters, so --log-format, '
+             '--log-date-format and --log-file are ignored, but --enable-logger and '
+             '--disable-logger still apply as logger level adjustments.')
+
+
+def add_extended_cli_args(parser):
     extended_group = parser.add_argument_group('extended')
     extended_group.add_argument(
         '-C', '--extra-config', type=str, action='append',
         help='additional configuration properties for client in "key=val" format. Can be provided multiple times.')
 
 
+def _load_log_config(path):
+    """Configure logging from a dictConfig (.json/.yaml) or fileConfig (.ini) file."""
+    if path.endswith(('.yaml', '.yml')):
+        try:
+            import yaml
+        except ImportError:
+            raise ValueError('PyYAML is required to load a YAML logging config: %s' % (path,))
+        with open(path) as f:
+            _dict_config(yaml.safe_load(f))
+    elif path.endswith('.json'):
+        import json
+        with open(path) as f:
+            _dict_config(json.load(f))
+    else:
+        # disable_existing_loggers defaults to True, which would silence loggers
+        # configured before this call (e.g. the loggers we are about to enable);
+        # keep them around so --enable-logger still works.
+        logging.config.fileConfig(path, disable_existing_loggers=False)
+
+
+def _dict_config(cfg):
+    # Default disable_existing_loggers to False (dictConfig defaults to True), so a
+    # config that does not mention an already-created logger does not silence it.
+    # A config may still set it explicitly to opt into the stdlib default.
+    cfg.setdefault('disable_existing_loggers', False)
+    logging.config.dictConfig(cfg)
+
+
 def configure_logging(config):
     _LOGGING_LEVELS = {
         'NOTSET': 0,
@@ -74,22 +127,43 @@ def configure_logging(config):
         'ERROR': 40,
         'CRITICAL': 50,
     }
-    if config.enable_logger is not None:
-        log_level = _LOGGING_LEVELS[config.log_level.upper()]
-        handler = logging.StreamHandler()
-        handler.setLevel(log_level)
-        handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
-        for name in config.enable_logger:
-            logger = logging.getLogger(name)
-            logger.setLevel(log_level)
-            logger.addHandler(handler)
+    log_level = _LOGGING_LEVELS[config.log_level.upper()]
+    if getattr(config, 'log_config', None):
+        _load_log_config(config.log_config)
+        if config.enable_logger is not None:
+            # Preserve the no-config behavior of --enable-logger: ONLY the named
+            # loggers emit. The config file owns the handlers/formatters, so rather
+            # than attach our own we reuse them: silence everything else by raising
+            # the root level, then let each enabled logger opt back in and propagate
+            # its records up to the config's handlers.
+            logging.getLogger().setLevel(logging.CRITICAL + 1)
+            for name in config.enable_logger:
+                logger = logging.getLogger(name)
+                logger.disabled = False
+                logger.setLevel(log_level)
     else:
-        logging.basicConfig(level=_LOGGING_LEVELS[config.log_level.upper()])
-    if config.disable_logger is not None:
-        for name in config.disable_logger:
-            logging.getLogger(name).setLevel(logging.CRITICAL + 1)
+        log_format = config.log_format or logging.BASIC_FORMAT
+        if config.enable_logger is not None:
+            if config.log_file:
+                handler = logging.FileHandler(config.log_file)
+            else:
+                handler = logging.StreamHandler()
+            handler.setLevel(log_level)
+            handler.setFormatter(logging.Formatter(log_format, datefmt=config.log_date_format))
+            for name in config.enable_logger:
+                logger = logging.getLogger(name)
+                logger.setLevel(log_level)
+                logger.addHandler(handler)
+        else:
+            logging.basicConfig(
+                level=log_level, format=log_format,
+                datefmt=config.log_date_format, filename=config.log_file)
+    # --disable-logger silences a named logger in either mode by raising its level.
+    for name in config.disable_logger or []:
+        logging.getLogger(name).setLevel(logging.CRITICAL + 1)
 
 
 def add_common_cli_args(parser, bootstrap_required=True):
     add_connect_cli_args(parser, bootstrap_required)
     add_logging_cli_args(parser)
+    add_extended_cli_args(parser)

kafka/cluster.py

@@ -125,9 +125,8 @@ class ClusterMetadata:
             if ttl_ms == 0:
                 try:
                     await self.refresh_metadata()
-                except Errors.KafkaError as exc:
-                    log.debug('Metadata refresh failed: %s', exc)
-                    log.exception(exc)
+                except Errors.KafkaError:
+                    log.debug('Metadata refresh failed', exc_info=True)
                 continue
             try:
                 log.debug('Sleeping %s for next Metadata refresh', ttl_ms / 1000)
@@ -168,7 +167,6 @@ class ClusterMetadata:
             raise Errors.NodeNotReadyError('metadata')
         else:
             self._manager.reset_backoff('metadata')
-        log.info(f'Metadata refresh (node_id={node_id})')
         try:
             request = self.metadata_request()
             log.debug("Sending metadata request %s to node %s", request, node_id)
@@ -177,7 +175,7 @@ class ClusterMetadata:
             log.error('Metadata refresh: failed %s', exc)
             self.failed_update(exc)
             raise
-        log.debug('Metadata refresh: success')
+        log.debug(f'Metadata refresh: success (node_id={node_id})')
         self.update_metadata(response)
 
     def _generate_bootstrap_brokers(self):

kafka/consumer/fetcher.py

@@ -232,9 +232,18 @@ class Fetcher:
         # No records yet. Block until either an in-flight fetch
         # completes (records may have arrived) or a pending offset-reset
         # task completes (positions become available, enabling a fetch
-        # on the next caller iteration). add_both fires synchronously on
-        # already-done futures, closing the race where a future resolves
-        # between scheduling and the wait setup.
+        # on the next caller iteration).
+        #
+        # add_both fires synchronously on an already-done future: if a fetch
+        # response lands between the drain above and this wait setup, _wake
+        # fires immediately so we re-drain instead of stalling for the full
+        # timeout.
+        #
+        # This relies on _fetch_futures holding only *recent* completions.
+        # otherwise a fetch that completed and was already drained iterations
+        # ago lingers behind a slow broker's in-flight fetch and re-fires
+        # _wake on every call, busy-looping the poll loop until that slow
+        # fetch finally returns.
         waited_on = list(self._fetch_futures)
         if self._reset_task is not None and not self._reset_task.is_done:
             waited_on.append(self._reset_task)
@@ -277,21 +286,66 @@ class Fetcher:
             future.add_both(self._clear_pending_fetch_request, node_id)
             futures.append(future)
         self._fetch_futures.extend(futures)
-        self._clean_done_fetch_futures()
+        await self._clean_done_fetch_futures()
         return futures
 
-    def _clean_done_fetch_futures(self):
-        while True:
-            if not self._fetch_futures:
-                break
-            if not self._fetch_futures[0].is_done:
-                break
-            self._fetch_futures.popleft()
+    async def _clean_done_fetch_futures(self):
+        # Drop every completed fetch future. With multiple brokers, fetches
+        # may complete out of order. fetch_records() relies on _fetch_futures
+        # holding only recent completions (it fires _wake synchronously on any
+        # done future to avoid stalling -- see the wait setup there); a
+        # lingering stale completion re-fires that wake on every call and busy-
+        # loops the poll loop until the slow broker's in-flight fetch returns.
+        #
+        # Threading: this REBINDS self._fetch_futures, which must happen on the
+        # IO thread so it never races the foreground's list(self._fetch_futures)
+        # read in fetch_records(). Defined async to enforce that -- the body
+        # can only run by being driven on the IO loop (awaited from another
+        # coroutine, or scheduled via manager.run/call_soon), so the rebind
+        # always executes on the IO thread regardless of who initiates it.
+        # The rebind is a single atomic attribute store, so a foreground reader
+        # always sees either the old or the new deque, never a half-cleaned one.
+        #
+        # Two alternate designs we considered (either would remove the need for
+        # this "evict every done future + rebind" dance):
+        #
+        #   1. Wakeup flag (Apache Kafka Java client, FetchBuffer). Instead of
+        #      waiting on the fetch-future objects, wait on a single consumable
+        #      signal: the IO thread sets a flag (wokenup) when it buffers a
+        #      completed fetch; the foreground's wait loops `while not woken:
+        #      await` and consumes the flag (compareAndSet true->false) on each
+        #      pass. Because the signal is cleared on consumption and is not
+        #      re-derived from lingering future objects, a stale/drained
+        #      completion cannot re-trigger it -- so no busy-loop and no
+        #      per-call cleanup of a future list at all. This is the most
+        #      faithful port of the threaded Java consumer's design.
+        #
+        #   2. Per-node fetch tracking. Key fetches by broker: dict[node_id,
+        #      deque] (or just dict[node_id, Future], since _create_fetch_-
+        #      requests keeps at most one in-flight fetch per node). Within a
+        #      single connection responses return in request order, so each
+        #      per-node deque completes in order and the simple head-only
+        #      popleft cleanup is correct again -- no out-of-order stranding,
+        #      and cleanup is an in-place popleft (atomic, no rebind, so the
+        #      threading note above goes away). This structure could also
+        #      subsume _nodes_with_pending_fetch_requests entirely ("pending"
+        #      == the node's last future is not done), collapsing two
+        #      structures into one source of truth.
+        if not self._fetch_futures:
+            return
+        self._fetch_futures = collections.deque(
+            fut for fut in self._fetch_futures if not fut.is_done)
 
     def in_flight_fetches(self):
-        """Return True if there are any unprocessed FetchRequests in flight."""
-        self._clean_done_fetch_futures()
-        return bool(self._fetch_futures)
+        """Return True if there are any unprocessed (incomplete) FetchRequests
+        in flight."""
+
+        # Read-only on purpose: this may be called from the foreground thread,
+        # which must not mutate _fetch_futures (see _clean_done_fetch_futures --
+        # cleanup is IO-thread-only). Snapshot first so we never iterate the
+        # deque while the IO thread extends it, and check is_done directly
+        # rather than relying on a prior cleanup pass.
+        return any(not fut.is_done for fut in list(self._fetch_futures))
 
     def reset_offsets_if_needed(self, timeout_ms=None):
         """Schedule pending offset resets and return the in-flight Task.

kafka/coordinator/base.py

@@ -1117,10 +1117,11 @@ class BaseCoordinator(ABC):
         try:
             send_time = time.monotonic()
             response = await self._manager.send(request, node_id=self.coordinator_id)
-            return self._handle_heartbeat_response(response, send_time)
         except Errors.KafkaError as exc:
             self._failed_request(self.coordinator_id, request, exc)
             raise
+        else:
+            return self._handle_heartbeat_response(response, send_time)
 
     def _handle_heartbeat_response(self, response, send_time):
         if self._sensors:
@@ -1138,8 +1139,7 @@ class BaseCoordinator(ABC):
                         self.coordinator_id)
             self.coordinator_dead(error)
         elif error_type is Errors.RebalanceInProgressError:
-            heartbeat_log.warning("Heartbeat failed for group %s because it is"
-                                  " rebalancing", self.group_id)
+            heartbeat_log.info("Group %s is rebalancing; rejoining.", self.group_id)
             self.request_rejoin()
         elif error_type is Errors.IllegalGenerationError:
             heartbeat_log.warning("Heartbeat failed for group %s: generation id is not "
@@ -1158,7 +1158,6 @@ class BaseCoordinator(ABC):
             heartbeat_log.error("Heartbeat failed: authorization error: %s", error)
         else:
             heartbeat_log.error("Heartbeat failed: Unhandled error: %s", error)
-
         raise error
 
 

kafka/net/connection.py

@@ -96,7 +96,7 @@ class KafkaConnection:
         return self._init_future
 
     def __await__(self):
-        yield self.init_future
+        yield from self.init_future.__await__()   # == await self.init_future; raises on failure
         return self
 
     @property
@@ -203,7 +203,7 @@ class KafkaConnection:
             if req_correlation_id != resp_correlation_id:
                 return self.close(Errors.KafkaConnectionError('Received unrecognized correlation id'))
 
-            self.net.unschedule(timeout_task)
+            self.net.cancel(timeout_task)
             latency_ms = (time.monotonic() - sent_time) * 1000
             if self._sensors:
                 self._sensors.request_time.record(latency_ms)
@@ -239,8 +239,10 @@ class KafkaConnection:
             self._init_future.failure(error)
         if not self._close_future.is_done:
             if exc is None:
+                log.info('%s: Connection closed', self)
                 self._close_future.success(None)
             else:
+                log.error('%s: Connection lost: %s', self, exc)
                 self._close_future.failure(exc)
 
     def fail_in_flight_requests(self, error):
@@ -252,7 +254,7 @@ class KafkaConnection:
             future.failure(error)
         while self.in_flight_requests:
             _, future, _, _, timeout_task = self.in_flight_requests.popleft()
-            self.net.unschedule(timeout_task)
+            self.net.cancel(timeout_task)
             future.failure(error)
 
     def connection_made(self, transport):
@@ -262,6 +264,13 @@ class KafkaConnection:
         To receive data, wait for data_received() calls.
         When the connection is closed, connection_lost() is called.
         """
+        if self.closed:
+            # A concurrent close() may have torn the connection down while the
+            # transport was still being built. Setting initializing=True below
+            # would resurrect an already-closed connection mid-teardown and
+            # break the fail_in_flight_requests invariant; refuse instead. The
+            # caller (manager._connect) closes the orphaned transport.
+            raise Errors.KafkaConnectionError('Connection closed during connect')
         self.transport = transport
         if self.transport.get_protocol() != self:
             self.transport.set_protocol(self)
@@ -276,6 +285,7 @@ class KafkaConnection:
             client_id=self.config['client_id'],
             receive_message_max_bytes=self.config['receive_message_max_bytes'],
             ident=log_prefix)
+        log.debug('%s: Connection made', self)
 
     def pause(self, v):
         self.paused.add(v)
@@ -362,6 +372,7 @@ class KafkaConnection:
             self.close(error)
         else:
             self._init_complete()
+            log.info('%s: Connected', self)
 
     async def _get_api_versions(self, timeout_at=None):
         if timeout_at is None:
@@ -400,11 +411,15 @@ class KafkaConnection:
         api_versions = {api_version.api_key: (api_version.min_version, api_version.max_version)
                         for api_version in response.api_keys}
         bvd = BrokerVersionData(api_versions=api_versions)
-        log.info('%s: Broker version identified as %s', self, '.'.join(map(str, bvd.broker_version)))
-        if self.broker_version_data is None or self.broker_version_data > bvd:
+        if self.broker_version_data is None:
+            log.info('%s: Broker version identified as %s', self, bvd.broker_version_str)
             self.broker_version_data = bvd
-        else:
-            log.info('%s: Clamping client to user-supplied broker version %s', self, '.'.join(map(str, self.broker_version)))
+        elif self.broker_version_data > bvd:
+            log.info('%s: Broker version identified as %s (lower than user-supplied %s)', self, bvd.broker_version_str, self.broker_version_data.broker_version_str)
+            self.broker_version_data = bvd
+        elif self.broker_version_data is not None and self.broker_version_data < bvd:
+            log.info('%s: Broker version identified as %s; clamping to user-supplied %s', self, bvd.broker_version_str, self.broker_version_data.broker_version_str)
+        # No log if user-supplied api_version is the same as broker-identified version
 
     @property
     def sasl_enabled(self):
@@ -534,10 +549,7 @@ class SaslReauthenticator:
         """Cancel any pending re-auth and fail the drain awaiter if present.
         Called from KafkaConnection.connection_lost."""
         if self._task is not None:
-            try:
-                self._conn.net.unschedule(self._task)
-            except (ValueError, KeyError):
-                pass
+            self._conn.net.cancel(self._task)
             self._task = None
         if self._drain_future is not None and not self._drain_future.is_done:
             self._drain_future.failure(Errors.KafkaConnectionError())

kafka/net/manager.py

@@ -73,7 +73,7 @@ class KafkaConnectionManager:
                 "client_dns_lookup must be one of %s; got %r"
                 % (self._VALID_DNS_LOOKUP_MODES, self.config['client_dns_lookup']))
 
-        if 'socks5_proxy' in configs:
+        if configs.get('socks5_proxy') is not None:
             if self.config['proxy_url'] is None:
                 log.warning('socks5_proxy is deprecated, use proxy_url instead')
                 self.config['proxy_url'] = configs['socks5_proxy']
@@ -117,7 +117,9 @@ class KafkaConnectionManager:
     async def _do_bootstrap(self, deadline):
         while not self.closed and (deadline is None or time.monotonic() < deadline):
             bootstrap_broker = random.choice(self.cluster.bootstrap_brokers())
-            log.debug('Attempting bootstrap with %s', bootstrap_broker)
+            log.info('Attempting bootstrap to %s at %s:%s (rack %s)',
+                     bootstrap_broker.node_id, bootstrap_broker.host,
+                     bootstrap_broker.port, bootstrap_broker.rack)
             try:
                 timeout_ms = (deadline - time.monotonic()) * 1000 if deadline is not None else None
                 conn = self.get_connection(bootstrap_broker.node_id,
@@ -129,7 +131,7 @@ class KafkaConnectionManager:
                 delay = self.connection_delay(bootstrap_broker.node_id)
                 if deadline is not None:
                     delay = min(delay, max(0, deadline - time.monotonic()))
-                log.debug('Bootstrap %s NodeNotReadyError: backoff %s', bootstrap_broker, delay)
+                log.warning('Bootstrap %s not ready; waiting %.2f secs', bootstrap_broker.node_id, delay)
                 await self._bootstrap_wakeup(delay)
                 continue
 
@@ -140,17 +142,20 @@ class KafkaConnectionManager:
                 raise
             except Exception as exc:
                 self._conns.pop(bootstrap_broker.node_id, conn).close(exc)
+                backoff_ms = self.update_backoff(bootstrap_broker.node_id)
+                log.warning('Bootstrap connection to %s failed: %s (backoff %.2f secs)',
+                            bootstrap_broker.node_id, exc, backoff_ms / 1000)
                 continue
 
             try:
                 await self.cluster.refresh_metadata(bootstrap_broker.node_id)
                 if not self.cluster.brokers():
-                    log.warning('Bootstrap metadata response has no brokers. Retrying.')
-                    self.update_backoff(bootstrap_broker.node_id)
+                    backoff_ms = self.update_backoff(bootstrap_broker.node_id)
+                    log.warning('Bootstrap metadata response has no brokers. Retrying in %.2f secs.', backoff_ms / 1000)
                     continue
             except Exception as exc:
-                log.error(f'Bootstrap attempt to {bootstrap_broker.node_id} failed: {exc}')
-                self.update_backoff(bootstrap_broker.node_id)
+                backoff_ms = self.update_backoff(bootstrap_broker.node_id)
+                log.error(f'Bootstrap attempt to {bootstrap_broker.node_id} failed: {exc} (backoff {(backoff_ms / 1000):.2f} secs)')
                 continue
             else:
                 self.reset_backoff(bootstrap_broker.node_id)
@@ -158,6 +163,7 @@ class KafkaConnectionManager:
                 log.info('Bootstrap complete: %s', self.cluster)
                 return True
             finally:
+                log.info('Closing bootstrap connection %s', bootstrap_broker.node_id)
                 self._conns.pop(bootstrap_broker.node_id, conn).close()
         else:
             raise Errors.KafkaTimeoutError(
@@ -239,9 +245,24 @@ class KafkaConnectionManager:
             return transport
 
     async def _connect(self, node, conn, reset_backoff_on_connect=True, timeout_at=None):
+        # Tracks ownership of the freshly built transport: while non-None it is
+        # ours to clean up (the connection hasn't taken it over yet), so the
+        # finally clause closes it. Cleared once connection_made() succeeds.
+        transport = None
         try:
             transport = await self._build_transport(node, timeout_at=timeout_at)
+            # The connection (or the whole manager) may have been closed while
+            # we were building the transport. Handing it to connection_made()
+            # would flip the conn back to `initializing` and resurrect a
+            # connection that is already being torn down. Discard
+            # the new transport instead of reviving a dead connection.
+            if conn.closed or self.closed:
+                log.debug('%s: closed during connect; discarding new transport', conn)
+                return
             conn.connection_made(transport)
+            transport = None  # conn owns cleanup now; skip finally: transport.close()
+            # Note: conn.initialize does not currently raise on error;
+            # errors are pushed to conn.init_future and raised on await conn
             await conn.initialize(timeout_at=timeout_at)
         except Exception as exc:
             log.error('Connection failed: %s', exc)
@@ -251,6 +272,9 @@ class KafkaConnectionManager:
                                 Errors.AuthorizationError)):
                 self._auth_failures[node.node_id] = exc
             return
+        finally:
+            if transport is not None:
+                transport.close()
 
         if self._sensors:
             self._sensors.connection_created.record()
@@ -275,6 +299,7 @@ class KafkaConnectionManager:
         node = self.cluster.broker_metadata(node_id)
         if node is None:
             raise Errors.UnknownBrokerIdError(node_id)
+        log.info('Initializing connection for node_id %s at %s:%s (rack=%s)', node_id, node.host, node.port, node.rack)
         conn = KafkaConnection(self._net, node_id=node_id, broker_version_data=self.broker_version_data, **self.config)
         if pop_on_close:
             conn.close_future.add_both(lambda _: self._conns.pop(node.node_id, None))
@@ -359,6 +384,7 @@ class KafkaConnectionManager:
                   node_id, backoff_ms, connect_ms, failures)
         backoff_until_time = time.monotonic() + (backoff_ms / 1000)
         self._backoff[node_id] = (failures, backoff_until_time, connect_ms)
+        return backoff_ms
 
     def connection_delay(self, node_id):
         """Connection delay in seconds.
@@ -423,11 +449,7 @@ class KafkaConnectionManager:
         try:
             return await wrapper
         finally:
-            if not timer.is_done:
-                try:
-                    self._net.unschedule(timer)
-                except ValueError:
-                    pass
+            self._net.cancel(timer)
 
     def call_soon(self, coro, *args):
         """Accepts a coroutine / awaitable / function and schedules it on the event loop.