kafka-python 3.0.0__py3-none-any.whl

kafka/future.py

@@ -0,0 +1,166 @@
+import functools
+import logging
+import threading
+
+from kafka.errors import RetriableError
+
+log = logging.getLogger(__name__)
+
+
+class Future:
+    __slots__ = ('is_done', 'value', 'exception', '_callbacks', '_errbacks', '_lock')
+    error_on_callbacks = False # and errbacks
+
+    def __init__(self):
+        self.is_done = False
+        self.value = None
+        self.exception = None
+        self._callbacks = []
+        self._errbacks = []
+        self._lock = threading.Lock()
+
+    def succeeded(self):
+        return self.is_done and self.exception is None
+
+    def failed(self):
+        return self.is_done and self.exception is not None
+
+    def retriable(self):
+        return self.is_done and isinstance(self.exception, RetriableError)
+
+    def success(self, value):
+        # Hot path: called once per produced record via the sender thread's
+        # batch-completion callback chain. Kept tight: explicit acquire/
+        # release (cheaper than `with`), callbacks snapshot under the lock,
+        # dispatch outside the lock, inlined callback loop (avoids an extra
+        # Python frame per completion).
+        # Clearing the lists releases any reference cycle held via stored
+        # bound methods (e.g. FutureProduceResult<->FutureRecordMetadata).
+        lock = self._lock
+        lock.acquire()
+        self.value = value
+        self.is_done = True
+        callbacks = self._callbacks
+        self._callbacks = None
+        self._errbacks = None
+        lock.release()
+        if callbacks:
+            error_on_callbacks = self.error_on_callbacks
+            for f in callbacks:
+                try:
+                    f(value)
+                except Exception as e:
+                    log.exception('Error processing callback')
+                    if error_on_callbacks:
+                        raise e
+        return self
+
+    def failure(self, e):
+        exception = e if type(e) is not type else e()
+        if not isinstance(exception, BaseException):
+            raise TypeError('future failed without an exception')
+        lock = self._lock
+        lock.acquire()
+        self.exception = exception
+        self.is_done = True
+        errbacks = self._errbacks
+        self._callbacks = None
+        self._errbacks = None
+        lock.release()
+        if errbacks:
+            error_on_callbacks = self.error_on_callbacks
+            for f in errbacks:
+                try:
+                    f(exception)
+                except Exception as err:
+                    log.exception('Error processing errback')
+                    if error_on_callbacks:
+                        raise err
+        return self
+
+    def add_callback(self, f, *args, **kwargs):
+        if args or kwargs:
+            f = functools.partial(f, *args, **kwargs)
+        lock = self._lock
+        lock.acquire()
+        if not self.is_done:
+            self._callbacks.append(f)
+            lock.release()
+            return self
+        lock.release()
+        if self.exception is None:
+            try:
+                f(self.value)
+            except Exception as e:
+                log.exception('Error processing callback')
+                if self.error_on_callbacks:
+                    raise e
+        return self
+
+    def add_errback(self, f, *args, **kwargs):
+        if args or kwargs:
+            f = functools.partial(f, *args, **kwargs)
+        lock = self._lock
+        lock.acquire()
+        if not self.is_done:
+            self._errbacks.append(f)
+            lock.release()
+            return self
+        lock.release()
+        if self.exception is not None:
+            try:
+                f(self.exception)
+            except Exception as e:
+                log.exception('Error processing errback')
+                if self.error_on_callbacks:
+                    raise e
+        return self
+
+    def _add_cb_eb(self, cb, eb):
+        """Register a (callback, errback) pair under a single lock acquire.
+
+        Fast path for call sites that always register both a plain callback
+        and errback with no ``*args``/``**kwargs``. Used on the producer hot
+        path (``FutureRecordMetadata`` -> ``FutureProduceResult``) to halve
+        the per-record lock-acquire count vs. calling ``add_callback()`` +
+        ``add_errback()`` separately.
+        """
+        lock = self._lock
+        lock.acquire()
+        if not self.is_done:
+            self._callbacks.append(cb)
+            self._errbacks.append(eb)
+            lock.release()
+            return self
+        lock.release()
+        if self.exception is None:
+            try:
+                cb(self.value)
+            except Exception as e:
+                log.exception('Error processing callback')
+                if self.error_on_callbacks:
+                    raise e
+        else:
+            try:
+                eb(self.exception)
+            except Exception as e:
+                log.exception('Error processing errback')
+                if self.error_on_callbacks:
+                    raise e
+        return self
+
+    def add_both(self, f, *args, **kwargs):
+        self.add_callback(f, *args, **kwargs)
+        self.add_errback(f, *args, **kwargs)
+        return self
+
+    def chain(self, future):
+        self._add_cb_eb(future.success, future.failure)
+        return self
+
+    def __await__(self):
+        if not self.is_done:
+            yield self
+        if self.exception:
+            raise self.exception
+        return self.value

kafka/metrics/__init__.py

@@ -0,0 +1,13 @@
+from kafka.metrics.compound_stat import NamedMeasurable
+from kafka.metrics.dict_reporter import DictReporter
+from kafka.metrics.kafka_metric import KafkaMetric
+from kafka.metrics.measurable import AnonMeasurable
+from kafka.metrics.metric_config import MetricConfig
+from kafka.metrics.metric_name import MetricName
+from kafka.metrics.metrics import Metrics
+from kafka.metrics.quota import Quota
+
+__all__ = [
+    'AnonMeasurable', 'DictReporter', 'KafkaMetric', 'MetricConfig',
+    'MetricName', 'Metrics', 'NamedMeasurable', 'Quota'
+]

kafka/metrics/compound_stat.py

@@ -0,0 +1,33 @@
+from abc import ABC, abstractmethod
+
+from kafka.metrics.stat import AbstractStat
+
+
+class AbstractCompoundStat(AbstractStat, ABC):
+    """
+    A compound stat is a stat where a single measurement and associated
+    data structure feeds many metrics. This is the example for a
+    histogram which has many associated percentiles.
+    """
+    @abstractmethod
+    def stats(self):
+        """
+        Return list of NamedMeasurable
+        """
+        pass
+
+
+class NamedMeasurable:
+    __slots__ = ('_name', '_stat')
+
+    def __init__(self, metric_name, measurable_stat):
+        self._name = metric_name
+        self._stat = measurable_stat
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def stat(self):
+        return self._stat

kafka/metrics/dict_reporter.py

@@ -0,0 +1,81 @@
+import logging
+import threading
+
+from kafka.metrics.metrics_reporter import AbstractMetricsReporter
+
+logger = logging.getLogger(__name__)
+
+
+class DictReporter(AbstractMetricsReporter):
+    """A basic dictionary based metrics reporter.
+
+    Store all metrics in a two level dictionary of category > name > metric.
+    """
+    def __init__(self, prefix=''):
+        self._lock = threading.Lock()
+        self._prefix = prefix if prefix else ''  # never allow None
+        self._store = {}
+
+    def snapshot(self):
+        """
+        Return a nested dictionary snapshot of all metrics and their
+        values at this time. Example:
+        {
+            'category': {
+                'metric1_name': 42.0,
+                'metric2_name': 'foo'
+            }
+        }
+        """
+        return dict((category, dict((name, metric.value())
+                                    for name, metric in list(metrics.items())))
+                    for category, metrics in
+                    list(self._store.items()))
+
+    def init(self, metrics):
+        for metric in metrics:
+            self.metric_change(metric)
+
+    def metric_change(self, metric):
+        with self._lock:
+            category = self.get_category(metric)
+            if category not in self._store:
+                self._store[category] = {}
+            self._store[category][metric.metric_name.name] = metric
+
+    def metric_removal(self, metric):
+        with self._lock:
+            category = self.get_category(metric)
+            metrics = self._store.get(category, {})
+            removed = metrics.pop(metric.metric_name.name, None)
+            if not metrics:
+                self._store.pop(category, None)
+            return removed
+
+    def get_category(self, metric):
+        """
+        Return a string category for the metric.
+
+        The category is made up of this reporter's prefix and the
+        metric's group and tags.
+
+        Examples:
+            prefix = 'foo', group = 'bar', tags = {'a': 1, 'b': 2}
+            returns: 'foo.bar.a=1,b=2'
+
+            prefix = 'foo', group = 'bar', tags = None
+            returns: 'foo.bar'
+
+            prefix = None, group = 'bar', tags = None
+            returns: 'bar'
+        """
+        tags = ','.join('%s=%s' % (k, v) for k, v in
+                        sorted(metric.metric_name.tags.items()))
+        return '.'.join(x for x in
+                        [self._prefix, metric.metric_name.group, tags] if x)
+
+    def configure(self, configs):
+        pass
+
+    def close(self):
+        pass

kafka/metrics/kafka_metric.py

@@ -0,0 +1,36 @@
+import time
+
+
+class KafkaMetric:
+    __slots__ = ('_metric_name', '_measurable', '_config')
+
+    # NOTE java constructor takes a lock instance
+    def __init__(self, metric_name, measurable, config):
+        if not metric_name:
+            raise ValueError('metric_name must be non-empty')
+        if not measurable:
+            raise ValueError('measurable must be non-empty')
+        self._metric_name = metric_name
+        self._measurable = measurable
+        self._config = config
+
+    @property
+    def metric_name(self):
+        return self._metric_name
+
+    @property
+    def measurable(self):
+        return self._measurable
+
+    @property
+    def config(self):
+        return self._config
+
+    @config.setter
+    def config(self, config):
+        self._config = config
+
+    def value(self, time_ms=None):
+        if time_ms is None:
+            time_ms = time.monotonic() * 1000
+        return self._measurable.measure(self._config, time_ms)

kafka/metrics/measurable.py

@@ -0,0 +1,27 @@
+from abc import ABC, abstractmethod
+
+
+class AbstractMeasurable(ABC):
+    """A measurable quantity that can be registered as a metric"""
+    @abstractmethod
+    def measure(self, config, now):
+        """
+        Measure this quantity and return the result
+
+        Arguments:
+            config (MetricConfig): The configuration for this metric
+            now (int): The POSIX time in milliseconds the measurement
+                is being taken
+
+        Returns:
+            The measured value
+        """
+        pass
+
+
+class AnonMeasurable(AbstractMeasurable):
+    def __init__(self, measure_fn):
+        self._measure_fn = measure_fn
+
+    def measure(self, config, now):
+        return float(self._measure_fn(config, now))

kafka/metrics/measurable_stat.py

@@ -0,0 +1,13 @@
+from abc import ABC
+
+from kafka.metrics.measurable import AbstractMeasurable
+from kafka.metrics.stat import AbstractStat
+
+
+class AbstractMeasurableStat(AbstractStat, AbstractMeasurable, ABC):
+    """
+    An AbstractMeasurableStat is an AbstractStat that is also
+    an AbstractMeasurable (i.e. can produce a single floating point value).
+    This is the interface used for most of the simple statistics such
+    as Avg, Max, Count, etc.
+    """

kafka/metrics/metric_config.py

@@ -0,0 +1,33 @@
+import sys
+
+
+class MetricConfig:
+    """Configuration values for metrics"""
+    __slots__ = ('quota', '_samples', 'event_window', 'time_window_ms', 'tags')
+
+    def __init__(self, quota=None, samples=2, event_window=sys.maxsize,
+                 time_window_ms=30 * 1000, tags=None):
+        """
+        Arguments:
+            quota (Quota, optional): Upper or lower bound of a value.
+            samples (int, optional): Max number of samples kept per metric.
+            event_window (int, optional): Max number of values per sample.
+            time_window_ms (int, optional): Max age of an individual sample.
+            tags (dict of {str: str}, optional): Tags for each metric.
+        """
+        self.quota = quota
+        self._samples = samples
+        self.event_window = event_window
+        self.time_window_ms = time_window_ms
+        # tags should be OrderedDict (not supported in py26)
+        self.tags = tags if tags else {}
+
+    @property
+    def samples(self):
+        return self._samples
+
+    @samples.setter
+    def samples(self, value):
+        if value < 1:
+            raise ValueError('The number of samples must be at least 1.')
+        self._samples = value

kafka/metrics/metric_name.py

@@ -0,0 +1,105 @@
+import copy
+
+
+class MetricName:
+    """
+    This class encapsulates a metric's name, logical group and its
+    related attributes (tags).
+
+    group, tags parameters can be used to create unique metric names.
+    e.g. domainName:type=group,key1=val1,key2=val2
+
+    Usage looks something like this:
+
+        # set up metrics:
+        metric_tags = {'client-id': 'producer-1', 'topic': 'topic'}
+        metric_config = MetricConfig(tags=metric_tags)
+
+        # metrics is the global repository of metrics and sensors
+        metrics = Metrics(metric_config)
+
+        sensor = metrics.sensor('message-sizes')
+        metric_name = metrics.metric_name('message-size-avg',
+                                          'producer-metrics',
+                                          'average message size')
+        sensor.add(metric_name, Avg())
+
+        metric_name = metrics.metric_name('message-size-max',
+        sensor.add(metric_name, Max())
+
+        tags = {'client-id': 'my-client', 'topic': 'my-topic'}
+        metric_name = metrics.metric_name('message-size-min',
+                                          'producer-metrics',
+                                          'message minimum size', tags)
+        sensor.add(metric_name, Min())
+
+        # as messages are sent we record the sizes
+        sensor.record(message_size)
+    """
+    __slots__ = ('_name', '_group', '_description', '_tags', '_hash')
+
+    def __init__(self, name, group, description=None, tags=None):
+        """
+        Arguments:
+            name (str): The name of the metric.
+            group (str): The logical group name of the metrics to which this
+                metric belongs.
+            description (str, optional): A human-readable description to
+                include in the metric.
+            tags (dict, optional): Additional key/val attributes of the metric.
+        """
+        if not (name and group):
+            raise ValueError('name and group must be non-empty.')
+        if tags is not None and not isinstance(tags, dict):
+            raise ValueError('tags must be a dict if present.')
+
+        self._name = name
+        self._group = group
+        self._description = description
+        self._tags = copy.copy(tags)
+        self._hash = 0
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def group(self):
+        return self._group
+
+    @property
+    def description(self):
+        return self._description
+
+    @property
+    def tags(self):
+        return copy.copy(self._tags)
+
+    def __hash__(self):
+        if self._hash != 0:
+            return self._hash
+        prime = 31
+        result = 1
+        result = prime * result + hash(self.group)
+        result = prime * result + hash(self.name)
+        tags_hash = hash(frozenset(self.tags.items())) if self.tags else 0
+        result = prime * result + tags_hash
+        self._hash = result
+        return result
+
+    def __eq__(self, other):
+        if self is other:
+            return True
+        if other is None:
+            return False
+        return (isinstance(self, type(other)) and
+                self.group == other.group and
+                self.name == other.name and
+                self.tags == other.tags)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __str__(self):
+        return 'MetricName(name=%s, group=%s, description=%s, tags=%s)' % (
+            self.name, self.group, self.description, self.tags)