kafka-python 3.0.0__py3-none-any.whl

kafka/metrics/stats/rate.py

@@ -0,0 +1,118 @@
+from kafka.metrics.measurable_stat import AbstractMeasurableStat
+from kafka.metrics.stats.sampled_stat import AbstractSampledStat
+
+
+class TimeUnit:
+    _names = {
+        'nanosecond': 0,
+        'microsecond': 1,
+        'millisecond': 2,
+        'second': 3,
+        'minute': 4,
+        'hour':  5,
+        'day': 6,
+    }
+
+    NANOSECONDS = _names['nanosecond']
+    MICROSECONDS = _names['microsecond']
+    MILLISECONDS = _names['millisecond']
+    SECONDS = _names['second']
+    MINUTES = _names['minute']
+    HOURS = _names['hour']
+    DAYS = _names['day']
+
+    @staticmethod
+    def get_name(time_unit):
+        return TimeUnit._names[time_unit]
+
+
+class Rate(AbstractMeasurableStat):
+    """
+    The rate of the given quantity. By default this is the total observed
+    over a set of samples from a sampled statistic divided by the elapsed
+    time over the sample windows. Alternative AbstractSampledStat
+    implementations can be provided, however, to record the rate of
+    occurrences (e.g. the count of values measured over the time interval)
+    or other such values.
+    """
+    __slots__ = ('_stat', '_unit')
+
+    def __init__(self, time_unit=TimeUnit.SECONDS, sampled_stat=None):
+        self._stat = sampled_stat or SampledTotal()
+        self._unit = time_unit
+
+    def unit_name(self):
+        return TimeUnit.get_name(self._unit)
+
+    def record(self, config, value, time_ms):
+        self._stat.record(config, value, time_ms)
+
+    def measure(self, config, now):
+        value = self._stat.measure(config, now)
+        return float(value) / self.convert(self.window_size(config, now))
+
+    def window_size(self, config, now):
+        # purge old samples before we compute the window size
+        self._stat.purge_obsolete_samples(config, now)
+
+        """
+        Here we check the total amount of time elapsed since the oldest
+        non-obsolete window. This give the total window_size of the batch
+        which is the time used for Rate computation. However, there is
+        an issue if we do not have sufficient data for e.g. if only
+        1 second has elapsed in a 30 second window, the measured rate
+        will be very high. Hence we assume that the elapsed time is
+        always N-1 complete windows plus whatever fraction of the final
+        window is complete.
+
+        Note that we could simply count the amount of time elapsed in
+        the current window and add n-1 windows to get the total time,
+        but this approach does not account for sleeps. AbstractSampledStat
+        only creates samples whenever record is called, if no record is
+        called for a period of time that time is not accounted for in
+        window_size and produces incorrect results.
+        """
+        total_elapsed_time_ms = now - self._stat.oldest(now).last_window_ms
+        # Check how many full windows of data we have currently retained
+        num_full_windows = int(total_elapsed_time_ms / config.time_window_ms)
+        min_full_windows = config.samples - 1
+
+        # If the available windows are less than the minimum required,
+        # add the difference to the totalElapsedTime
+        if num_full_windows < min_full_windows:
+            total_elapsed_time_ms += ((min_full_windows - num_full_windows) *
+                                      config.time_window_ms)
+
+        return total_elapsed_time_ms
+
+    def convert(self, time_ms):
+        if self._unit == TimeUnit.NANOSECONDS:
+            return time_ms * 1000.0 * 1000.0
+        elif self._unit == TimeUnit.MICROSECONDS:
+            return time_ms * 1000.0
+        elif self._unit == TimeUnit.MILLISECONDS:
+            return time_ms
+        elif self._unit == TimeUnit.SECONDS:
+            return time_ms / 1000.0
+        elif self._unit == TimeUnit.MINUTES:
+            return time_ms / (60.0 * 1000.0)
+        elif self._unit == TimeUnit.HOURS:
+            return time_ms / (60.0 * 60.0 * 1000.0)
+        elif self._unit == TimeUnit.DAYS:
+            return time_ms / (24.0 * 60.0 * 60.0 * 1000.0)
+        else:
+            raise ValueError('Unknown unit: %s' % (self._unit,))
+
+
+class SampledTotal(AbstractSampledStat):
+    __slots__ = ('_initial_value', '_samples', '_current')
+    def __init__(self, initial_value=None):
+        if initial_value is not None:
+            raise ValueError('initial_value cannot be set on SampledTotal')
+        super().__init__(0.0)
+
+    def update(self, sample, config, value, time_ms):
+        sample.value += value
+
+    def combine(self, samples, config, now):
+        return float(sum(sample.value for sample in samples))

kafka/metrics/stats/sampled_stat.py

@@ -0,0 +1,99 @@
+from abc import ABC, abstractmethod
+
+from kafka.metrics.measurable_stat import AbstractMeasurableStat
+
+
+class AbstractSampledStat(AbstractMeasurableStat, ABC):
+    """
+    An AbstractSampledStat records a single scalar value measured over
+    one or more samples. Each sample is recorded over a configurable
+    window. The window can be defined by number of events or elapsed
+    time (or both, if both are given the window is complete when
+    *either* the event count or elapsed time criterion is met).
+
+    All the samples are combined to produce the measurement. When a
+    window is complete the oldest sample is cleared and recycled to
+    begin recording the next sample.
+
+    Subclasses of this class define different statistics measured
+    using this basic pattern.
+    """
+    __slots__ = ('_initial_value', '_samples', '_current')
+
+    def __init__(self, initial_value):
+        self._initial_value = initial_value
+        self._samples = []
+        self._current = 0
+
+    @abstractmethod
+    def update(self, sample, config, value, time_ms):
+        pass
+
+    @abstractmethod
+    def combine(self, samples, config, now):
+        pass
+
+    def record(self, config, value, time_ms):
+        sample = self.current(time_ms)
+        if sample.is_complete(time_ms, config):
+            sample = self._advance(config, time_ms)
+        self.update(sample, config, float(value), time_ms)
+        sample.event_count += 1
+
+    def new_sample(self, time_ms):
+        return self.Sample(self._initial_value, time_ms)
+
+    def measure(self, config, now):
+        self.purge_obsolete_samples(config, now)
+        return float(self.combine(self._samples, config, now))
+
+    def current(self, time_ms):
+        if not self._samples:
+            self._samples.append(self.new_sample(time_ms))
+        return self._samples[self._current]
+
+    def oldest(self, now):
+        if not self._samples:
+            self._samples.append(self.new_sample(now))
+        oldest = self._samples[0]
+        for sample in self._samples[1:]:
+            if sample.last_window_ms < oldest.last_window_ms:
+                oldest = sample
+        return oldest
+
+    def purge_obsolete_samples(self, config, now):
+        """
+        Timeout any windows that have expired in the absence of any events
+        """
+        expire_age = config.samples * config.time_window_ms
+        for sample in self._samples:
+            if now - sample.last_window_ms >= expire_age:
+                sample.reset(now)
+
+    def _advance(self, config, time_ms):
+        self._current = (self._current + 1) % config.samples
+        if self._current >= len(self._samples):
+            sample = self.new_sample(time_ms)
+            self._samples.append(sample)
+            return sample
+        else:
+            sample = self.current(time_ms)
+            sample.reset(time_ms)
+            return sample
+
+    class Sample:
+
+        def __init__(self, initial_value, now):
+            self.initial_value = initial_value
+            self.event_count = 0
+            self.last_window_ms = now
+            self.value = initial_value
+
+        def reset(self, now):
+            self.event_count = 0
+            self.last_window_ms = now
+            self.value = self.initial_value
+
+        def is_complete(self, time_ms, config):
+            return (time_ms - self.last_window_ms >= config.time_window_ms or
+                    self.event_count >= config.event_window)

kafka/metrics/stats/sensor.py

@@ -0,0 +1,136 @@
+import threading
+import time
+
+from kafka.errors import QuotaViolationError
+from kafka.metrics import KafkaMetric
+
+
+class Sensor:
+    """
+    A sensor applies a continuous sequence of numerical values
+    to a set of associated metrics. For example a sensor on
+    message size would record a sequence of message sizes using
+    the `record(double)` api and would maintain a set
+    of metrics about request sizes such as the average or max.
+    """
+    __slots__ = ('_lock', '_registry', '_name', '_parents', '_metrics',
+                 '_stats', '_config', '_inactive_sensor_expiration_time_ms',
+                 '_last_record_time')
+
+    def __init__(self, registry, name, parents, config,
+                 inactive_sensor_expiration_time_seconds):
+        if not name:
+            raise ValueError('name must be non-empty')
+        self._lock = threading.RLock()
+        self._registry = registry
+        self._name = name
+        self._parents = parents or []
+        self._metrics = []
+        self._stats = []
+        self._config = config
+        self._inactive_sensor_expiration_time_ms = (
+            inactive_sensor_expiration_time_seconds * 1000)
+        self._last_record_time = time.monotonic() * 1000
+        self._check_forest(set())
+
+    def _check_forest(self, sensors):
+        """Validate that this sensor doesn't end up referencing itself."""
+        if self in sensors:
+            raise ValueError('Circular dependency in sensors: %s is its own'
+                             'parent.' % (self.name,))
+        sensors.add(self)
+        for parent in self._parents:
+            parent._check_forest(sensors)
+
+    @property
+    def name(self):
+        """
+        The name this sensor is registered with.
+        This name will be unique among all registered sensors.
+        """
+        return self._name
+
+    @property
+    def metrics(self):
+        return tuple(self._metrics)
+
+    def record(self, value=1.0, time_ms=None):
+        """
+        Record a value at a known time.
+        Arguments:
+            value (double): The value we are recording
+            time_ms (int): A POSIX timestamp in milliseconds.
+                Default: The time when record() is evaluated (now)
+
+        Raises:
+            QuotaViolationException: if recording this value moves a
+                metric beyond its configured maximum or minimum bound
+        """
+        if time_ms is None:
+            time_ms = time.monotonic() * 1000
+        self._last_record_time = time_ms
+        with self._lock:  # XXX high volume, might be performance issue
+            # increment all the stats
+            for stat in self._stats:
+                stat.record(self._config, value, time_ms)
+            self._check_quotas(time_ms)
+        for parent in self._parents:
+            parent.record(value, time_ms)
+
+    def _check_quotas(self, time_ms):
+        """
+        Check if we have violated our quota for any metric that
+        has a configured quota
+        """
+        for metric in self._metrics:
+            if metric.config and metric.config.quota:
+                value = metric.value(time_ms)
+                if not metric.config.quota.is_acceptable(value):
+                    raise QuotaViolationError("'%s' violated quota. Actual: "
+                                              "%d, Threshold: %d" %
+                                              (metric.metric_name,
+                                               value,
+                                               metric.config.quota.bound))
+
+    def add_compound(self, compound_stat, config=None):
+        """
+        Register a compound statistic with this sensor which
+        yields multiple measurable quantities (like a histogram)
+
+        Arguments:
+            stat (AbstractCompoundStat): The stat to register
+            config (MetricConfig): The configuration for this stat.
+                If None then the stat will use the default configuration
+                for this sensor.
+        """
+        if not compound_stat:
+            raise ValueError('compound stat must be non-empty')
+        self._stats.append(compound_stat)
+        for named_measurable in compound_stat.stats():
+            metric = KafkaMetric(named_measurable.name, named_measurable.stat,
+                                 config or self._config)
+            self._registry.register_metric(metric)
+            self._metrics.append(metric)
+
+    def add(self, metric_name, stat, config=None):
+        """
+        Register a metric with this sensor
+
+        Arguments:
+            metric_name (MetricName): The name of the metric
+            stat (AbstractMeasurableStat): The statistic to keep
+            config (MetricConfig): A special configuration for this metric.
+                If None use the sensor default configuration.
+        """
+        with self._lock:
+            metric = KafkaMetric(metric_name, stat, config or self._config)
+            self._registry.register_metric(metric)
+            self._metrics.append(metric)
+            self._stats.append(stat)
+
+    def has_expired(self):
+        """
+        Return True if the Sensor is eligible for removal due to inactivity.
+        """
+        return ((time.monotonic() * 1000 - self._last_record_time) >
+                self._inactive_sensor_expiration_time_ms)

kafka/metrics/stats/total.py

@@ -0,0 +1,15 @@
+from kafka.metrics.measurable_stat import AbstractMeasurableStat
+
+
+class Total(AbstractMeasurableStat):
+    """An un-windowed cumulative total maintained over all time."""
+    __slots__ = ('_total',)
+
+    def __init__(self, value=0.0):
+        self._total = value
+
+    def record(self, config, value, now):
+        self._total += value
+
+    def measure(self, config, now):
+        return float(self._total)

kafka/net/__init__.py

@@ -0,0 +1,19 @@
+from .connection import KafkaConnection
+from .inet import create_connection, KafkaNetSocket
+from .manager import KafkaConnectionManager
+from .metrics import KafkaConnectionMetrics, KafkaManagerMetrics
+from .selector import NetworkSelector
+from .http_connect import HttpConnectProxy
+from .socks5 import Socks5Proxy
+from .transport import KafkaTCPTransport, KafkaSSLTransport
+from .wakeup_notifier import WakeupNotifier
+
+from .compat import KafkaNetClient
+
+
+__all__ = [
+    'KafkaConnection', 'create_connection', 'KafkaNetSocket',
+    'KafkaConnectionManager', 'KafkaConnectionMetrics', 'KafkaManagerMetrics',
+    'NetworkSelector', 'HttpConnectProxy', 'Socks5Proxy', 'KafkaTCPTransport', 'KafkaSSLTransport',
+    'WakeupNotifier', 'KafkaNetClient',
+]

kafka/net/compat.py

@@ -0,0 +1,165 @@
+import logging
+import random
+import threading
+import time
+
+import kafka.errors as Errors
+from kafka.net.manager import KafkaConnectionManager
+from kafka.net.selector import NetworkSelector
+
+
+log = logging.getLogger(__name__)
+
+
+class KafkaNetClient:
+    """Drop-in replacement for KafkaClient backed by KafkaConnectionManager.
+
+    Provides the KafkaClient API surface that existing consumer/producer/admin
+    code depends on. Goal: shrink over time as components transition to using
+    KafkaConnectionManager directly (fire-and-forget via _request_buffer).
+    """
+    def __init__(self, net=None, manager=None, **configs):
+        # _lock is still used by the legacy Coordinator (kafka/coordinator/base.py).
+        # Remove once Coordinator moves to the IO thread (Phase D).
+        self._lock = threading.RLock()
+        self._net = NetworkSelector(**configs) if net is None else net
+        self._manager = KafkaConnectionManager(self._net, **configs) if manager is None else manager
+
+    @property
+    def cluster(self):
+        return self._manager.cluster
+
+    # Connection state queries
+
+    def connected(self, node_id):
+        conn = self._manager._conns.get(node_id)
+        return conn is not None and conn.connected
+
+    def is_disconnected(self, node_id):
+        return not self.connected(node_id)
+
+    def is_ready(self, node_id):
+        conn = self._manager._conns.get(node_id)
+        return conn is not None and conn.connected and not conn.paused
+
+    def ready(self, node_id, **kwargs):
+        if self.is_ready(node_id):
+            return True
+        try:
+            self._manager.get_connection(node_id)
+        except Errors.NodeNotReadyError:
+            pass
+        return False
+
+    def maybe_connect(self, node_id, **kwargs):
+        try:
+            self._manager.get_connection(node_id)
+        except Errors.NodeNotReadyError:
+            pass
+
+    def await_ready(self, node_id, timeout_ms=30000):
+        if self.is_ready(node_id):
+            return True
+        self.maybe_connect(node_id)
+        conn = self._manager._conns.get(node_id)
+        if conn is not None and not conn.init_future.is_done:
+            self._net.poll(timeout_ms=timeout_ms, future=conn.init_future)
+        # Connection may be initialized but paused (e.g. max_in_flight reached).
+        # Poll briefly to drain in-flight responses and unpause.
+        if conn is not None and conn.connected and conn.paused:
+            self._net.poll(timeout_ms=min(timeout_ms, self._manager.config['request_timeout_ms']))
+        if not self.is_ready(node_id):
+            raise Errors.KafkaConnectionError('Node %s not ready after %s ms' % (node_id, timeout_ms))
+        return True
+
+    # In-flight request tracking
+
+    def in_flight_request_count(self, node_id=None):
+        if node_id is not None:
+            conn = self._manager._conns.get(node_id)
+            return len(conn.in_flight_requests) if conn is not None else 0
+        return sum(len(c.in_flight_requests) for c in self._manager._conns.values())
+
+    def throttle_delay(self, node_id):
+        conn = self._manager._conns.get(node_id)
+        if conn is None:
+            return 0
+        remaining = conn._throttle_time - time.monotonic()
+        return max(0, remaining) * 1000
+
+    # Bootstrap / version
+
+    def bootstrap_connected(self):
+        bootstrap_future = self._manager._bootstrap_future
+        return bootstrap_future is not None and not bootstrap_future.is_done
+
+    def get_broker_version(self, timeout_ms=None):
+        if self._manager.broker_version is None:
+            self._manager.bootstrap(timeout_ms)
+        return self._manager.broker_version
+
+    def check_version(self, node_id=None, timeout_ms=10000):
+        if not self._manager.bootstrapped:
+            self._manager.bootstrap(timeout_ms)
+        if node_id is None:
+            return self._manager.broker_version
+        async def _check_version(broker_id, timeout_ms):
+            conn = await self._manager.get_connection(broker_id, timeout_ms=timeout_ms)
+            return conn.broker_version
+        return self._net.run(_check_version, node_id, timeout_ms)
+
+    # Request sending
+
+    def send(self, node_id, request, **kwargs):
+        return self._manager.send(request, node_id=node_id)
+
+    def send_and_receive(self, node_id, request, timeout_ms=30000):
+        self.await_ready(node_id, timeout_ms=timeout_ms)
+        f = self.send(node_id, request)
+        self._net.poll(timeout_ms=timeout_ms, future=f)
+        if f.succeeded():
+            return f.value
+        elif f.failed():
+            raise f.exception
+        raise Errors.KafkaTimeoutError('Request timed out')
+
+    # Delegation
+
+    def poll(self, timeout_ms=None, future=None):
+        # _lock serializes with HeartbeatThread, which also drives poll()
+        # while holding this lock. Without it, both threads would call
+        # _net.poll() concurrently and race on selector / task state.
+        # The lock goes away once HeartbeatThread does (Phase D).
+        with self._lock:
+            return self._net.poll(timeout_ms=timeout_ms, future=future)
+
+    def close(self, node_id=None):
+        self._manager.close(node_id=node_id)
+        if node_id is None:
+            self._net.close()
+
+    def least_loaded_node(self, bootstrap_fallback=False):
+        node_id = self._manager.least_loaded_node()
+        if node_id is None and bootstrap_fallback:
+            node_id = random.choice(self._manager.cluster.bootstrap_brokers()).node_id
+        return node_id
+
+    def least_loaded_node_refresh_ms(self, bootstrap_fallback=False):
+        brokers = self._manager.cluster.brokers()
+        if not brokers and bootstrap_fallback:
+            brokers = self._manager.cluster.bootstrap_brokers()
+        if not brokers:
+            return self._manager.config['reconnect_backoff_ms']
+        delays = [self._manager.connection_delay(broker.node_id) for broker in brokers]
+        return min(delays) * 1000
+
+    def connection_delay(self, node_id):
+        return self._manager.connection_delay(node_id)
+
+    def wakeup(self):
+        self._net.wakeup()
+
+    def api_version(self, operation, max_version=None):
+        if self._manager.broker_version_data is None:
+            raise Errors.IllegalStateError('broker_version_data is not available: have we bootstrapped yet?')
+        return self._manager.broker_version_data.api_version(operation, max_version=max_version)