kinto 18.1.1__py3-none-any.whl → 19.3.0__py3-none-any.whl

kinto/__init__.py

@@ -38,6 +38,7 @@ DEFAULT_SETTINGS = {
     "record_id_generator": "kinto.views.RelaxedUUID",
     "project_name": "kinto",
     "admin_assets_path": None,
+    "metrics_matchdict_fields": ["bucket_id", "collection_id", "group_id", "record_id"],
 }
 
 

kinto/core/__init__.py

@@ -66,8 +66,8 @@ DEFAULT_SETTINGS = {
         "kinto.core.initialization.setup_authentication",
         "kinto.core.initialization.setup_backoff",
         "kinto.core.initialization.setup_sentry",
-        "kinto.core.initialization.setup_statsd",
         "kinto.core.initialization.setup_listeners",
+        "kinto.core.initialization.setup_metrics",
         "kinto.core.events.setup_transaction_hook",
     ),
     "event_listeners": "",
@@ -96,6 +96,7 @@ DEFAULT_SETTINGS = {
     "statsd_backend": "kinto.core.statsd",
     "statsd_prefix": "kinto.core",
     "statsd_url": None,
+    "metrics_matchdict_fields": [],
     "storage_backend": "",
     "storage_url": "",
     "storage_max_fetch_size": 10000,

kinto/core/initialization.py

@@ -21,7 +21,7 @@ from pyramid.security import NO_PERMISSION_REQUIRED
 from pyramid.settings import asbool, aslist
 from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
 
-from kinto.core import cache, errors, permission, storage, utils
+from kinto.core import cache, errors, metrics, permission, storage, utils
 from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
 
 
@@ -334,51 +334,13 @@ def setup_sentry(config):
 
 
 def setup_statsd(config):
-    settings = config.get_settings()
-    config.registry.statsd = None
-
-    if settings["statsd_url"]:
-        statsd_mod = settings["statsd_backend"]
-        statsd_mod = config.maybe_dotted(statsd_mod)
-        client = statsd_mod.load_from_config(config)
-
-        config.registry.statsd = client
-
-        client.watch_execution_time(config.registry.cache, prefix="backend")
-        client.watch_execution_time(config.registry.storage, prefix="backend")
-        client.watch_execution_time(config.registry.permission, prefix="backend")
-
-        # Commit so that configured policy can be queried.
-        config.commit()
-        policy = config.registry.queryUtility(IAuthenticationPolicy)
-        if isinstance(policy, MultiAuthenticationPolicy):
-            for name, subpolicy in policy.get_policies():
-                client.watch_execution_time(subpolicy, prefix="authentication", classname=name)
-        else:
-            client.watch_execution_time(policy, prefix="authentication")
-
-        def on_new_response(event):
-            request = event.request
-
-            # Count unique users.
-            user_id = request.prefixed_userid
-            if user_id:
-                # Get rid of colons in metric packet (see #1282).
-                user_id = user_id.replace(":", ".")
-                client.count("users", unique=user_id)
-
-            # Count authentication verifications.
-            if hasattr(request, "authn_type"):
-                client.count(f"authn_type.{request.authn_type}")
-
-            # Count view calls.
-            service = request.current_service
-            if service:
-                client.count(f"view.{service.name}.{request.method}")
-
-        config.add_subscriber(on_new_response, NewResponse)
-
-        return client
+    # It would be pretty rare to find users that have a custom ``kinto.initialization_sequence`` setting.
+    # But just in case, warn that it will be removed in next major.
+    warnings.warn(
+        "``setup_statsd()`` is now deprecated. Use ``kinto.core.initialization.setup_metrics()`` instead.",
+        DeprecationWarning,
+    )
+    setup_metrics(config)
 
 
 def install_middlewares(app, settings):
@@ -466,6 +428,125 @@ def setup_logging(config):
     config.add_subscriber(on_new_response, NewResponse)
 
 
+def setup_metrics(config):
+    settings = config.get_settings()
+
+    # Register a no-op metrics service by default.
+    config.registry.registerUtility(metrics.NoOpMetricsService(), metrics.IMetricsService)
+
+    # This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
+    # `registry.storage`, `registry.cache`, etc. Consistency seems more important.
+    config.registry.__class__.metrics = property(
+        lambda reg: reg.queryUtility(metrics.IMetricsService)
+    )
+
+    def deprecated_registry(self):
+        warnings.warn(
+            "``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
+            DeprecationWarning,
+        )
+        return self.metrics
+
+    config.registry.__class__.statsd = property(deprecated_registry)
+
+    def on_app_created(event):
+        config = event.app
+        metrics_service = config.registry.metrics
+
+        metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
+        metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
+        metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
+
+        policy = config.registry.queryUtility(IAuthenticationPolicy)
+        if isinstance(policy, MultiAuthenticationPolicy):
+            for name, subpolicy in policy.get_policies():
+                metrics.watch_execution_time(
+                    metrics_service, subpolicy, prefix="authentication", classname=name
+                )
+        else:
+            metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
+
+    config.add_subscriber(on_app_created, ApplicationCreated)
+
+    def on_new_response(event):
+        request = event.request
+        metrics_service = config.registry.metrics
+
+        try:
+            endpoint = utils.strip_uri_prefix(request.path)
+        except UnicodeDecodeError as e:
+            # This `on_new_response` callback is also called when a HTTP 400
+            # is returned because of an invalid UTF-8 path. We still want metrics.
+            endpoint = str(e)
+
+        # Count unique users.
+        user_id = request.prefixed_userid
+        if user_id:
+            # Get rid of colons in metric packet (see #1282).
+            auth, user_id = user_id.split(":")
+            metrics_service.count("users", unique=[("auth", auth), ("userid", user_id)])
+
+        # Add extra labels to metrics, based on fields extracted from the request matchdict.
+        metrics_matchdict_fields = aslist(settings["metrics_matchdict_fields"])
+        # Turn the `id` field of object endpoints into `{resource}_id` (eg. `mushroom_id`, `bucket_id`)
+        enhanced_matchdict = dict(request.matchdict or {})
+        try:
+            enhanced_matchdict[request.current_resource_name + "_id"] = enhanced_matchdict.get(
+                "id", ""
+            )
+        except AttributeError:
+            # Not on a resource.
+            pass
+        metrics_matchdict_labels = [
+            (field, enhanced_matchdict.get(field, "")) for field in metrics_matchdict_fields
+        ]
+
+        # Count served requests.
+        metrics_service.count(
+            "request_summary",
+            unique=[
+                ("method", request.method.lower()),
+                ("endpoint", endpoint),
+                ("status", str(request.response.status_code)),
+            ]
+            + metrics_matchdict_labels,
+        )
+
+        try:
+            current = utils.msec_time()
+            duration = current - request._received_at
+            metrics_service.observe(
+                "request_duration",
+                duration,
+                labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
+            )
+        except AttributeError:  # pragma: no cover
+            # Logging was not setup in this Kinto app (unlikely but possible)
+            pass
+
+        # Observe response size.
+        metrics_service.observe(
+            "request_size",
+            len(request.response.body or b""),
+            labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
+        )
+
+        # Count authentication verifications.
+        if hasattr(request, "authn_type"):
+            metrics_service.count(f"authn_type.{request.authn_type}")
+
+        # Count view calls.
+        service = request.current_service
+        if service:
+            metrics_service.count(f"view.{service.name}.{request.method}")
+
+    config.add_subscriber(on_new_response, NewResponse)
+
+    # While statsd is deprecated, we include its plugin by default for retro-compability.
+    if settings["statsd_url"]:
+        config.include("kinto.plugins.statsd")
+
+
 class EventActionFilter:
     def __init__(self, actions, config):
         actions = ACTIONS.from_string_list(actions)
@@ -518,11 +599,9 @@ def setup_listeners(config):
             listener_mod = config.maybe_dotted(module_value)
             listener = listener_mod.load_from_config(config, prefix)
 
-        # If StatsD is enabled, monitor execution time of listeners.
-        if getattr(config.registry, "statsd", None):
-            statsd_client = config.registry.statsd
-            key = f"listeners.{name}"
-            listener = statsd_client.timer(key)(listener.__call__)
+        wrapped_listener = metrics.listener_with_timer(
+            config, f"listeners.{name}", listener.__call__
+        )
 
         # Optional filter by event action.
         actions_setting = prefix + "actions"
@@ -548,11 +627,11 @@ def setup_listeners(config):
         options = dict(for_actions=actions, for_resources=resource_names)
 
         if ACTIONS.READ in actions:
-            config.add_subscriber(listener, ResourceRead, **options)
+            config.add_subscriber(wrapped_listener, ResourceRead, **options)
             actions = [a for a in actions if a != ACTIONS.READ]
 
         if len(actions) > 0:
-            config.add_subscriber(listener, ResourceChanged, **options)
+            config.add_subscriber(wrapped_listener, ResourceChanged, **options)
 
 
 def load_default_settings(config, default_settings):

kinto/core/metrics.py

@@ -0,0 +1,93 @@
+import types
+
+from zope.interface import Interface, implementer
+
+from kinto.core import utils
+
+
+class IMetricsService(Interface):
+    """
+    An interface that defines the metrics service contract.
+    Any class implementing this must provide all its methods.
+    """
+
+    def timer(key):
+        """
+        Watch execution time.
+        """
+
+    def observe(self, key, value, labels=[]):
+        """
+        Observe a give `value` for the specified `key`.
+        """
+
+    def count(key, count=1, unique=None):
+        """
+        Count occurrences. If `unique` is set, overwrites the counter value
+        on each call.
+
+        `unique` should be of type ``list[tuple[str,str]]``.
+        """
+
+
+class NoOpTimer:
+    def __call__(self, f):
+        @utils.safe_wraps(f)
+        def _wrapped(*args, **kwargs):
+            return f(*args, **kwargs)
+
+        return _wrapped
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *args, **kwargs):
+        pass
+
+
+@implementer(IMetricsService)
+class NoOpMetricsService:
+    def timer(self, key):
+        return NoOpTimer()
+
+    def observe(self, key, value, labels=[]):
+        pass
+
+    def count(self, key, count=1, unique=None):
+        pass
+
+
+def watch_execution_time(metrics_service, obj, prefix="", classname=None):
+    """
+    Decorate all methods of an object in order to watch their execution time.
+    Metrics will be named `{prefix}.{classname}.{method}`.
+    """
+    classname = classname or utils.classname(obj)
+    members = dir(obj)
+    for name in members:
+        value = getattr(obj, name)
+        is_method = isinstance(value, types.MethodType)
+        if not name.startswith("_") and is_method:
+            statsd_key = f"{prefix}.{classname}.{name}"
+            decorated_method = metrics_service.timer(statsd_key)(value)
+            setattr(obj, name, decorated_method)
+
+
+def listener_with_timer(config, key, func):
+    """
+    Add a timer with the specified `key` on the specified `func`.
+    This is used to avoid evaluating `config.registry.metrics` during setup time
+    to avoid having to deal with initialization order and configuration committing.
+    """
+
+    def wrapped(*args, **kwargs):
+        metrics_service = config.registry.metrics
+        if not metrics_service:
+            # This only happens if `kinto.core.initialization.setup_metrics` is
+            # not listed in the `initialization_sequence` setting.
+            return func(*args, **kwargs)
+        # If metrics are enabled, monitor execution time of listeners.
+        with metrics_service.timer(key):
+            return func(*args, **kwargs)
+
+    return wrapped

kinto/core/statsd.py

@@ -1,63 +1 @@
-import types
-from urllib.parse import urlparse
-
-from pyramid.exceptions import ConfigurationError
-
-from kinto.core import utils
-
-
-try:
-    import statsd as statsd_module
-except ImportError:  # pragma: no cover
-    statsd_module = None
-
-
-class Client:
-    def __init__(self, host, port, prefix):
-        self._client = statsd_module.StatsClient(host, port, prefix=prefix)
-
-    def watch_execution_time(self, obj, prefix="", classname=None):
-        classname = classname or utils.classname(obj)
-        members = dir(obj)
-        for name in members:
-            value = getattr(obj, name)
-            is_method = isinstance(value, types.MethodType)
-            if not name.startswith("_") and is_method:
-                statsd_key = f"{prefix}.{classname}.{name}"
-                decorated_method = self.timer(statsd_key)(value)
-                setattr(obj, name, decorated_method)
-
-    def timer(self, key):
-        return self._client.timer(key)
-
-    def count(self, key, count=1, unique=None):
-        if unique is None:
-            return self._client.incr(key, count=count)
-        else:
-            return self._client.set(key, unique)
-
-
-def statsd_count(request, count_key):
-    statsd = request.registry.statsd
-    if statsd:
-        statsd.count(count_key)
-
-
-def load_from_config(config):
-    # If this is called, it means that a ``statsd_url`` was specified in settings.
-    # (see ``kinto.core.initialization``)
-    # Raise a proper error if the ``statsd`` module is not installed.
-    if statsd_module is None:
-        error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
-        raise ConfigurationError(error_msg)
-
-    settings = config.get_settings()
-    uri = settings["statsd_url"]
-    uri = urlparse(uri)
-
-    if settings["project_name"] != "":
-        prefix = settings["project_name"]
-    else:
-        prefix = settings["statsd_prefix"]
-
-    return Client(uri.hostname, uri.port, prefix)
+from kinto.plugins.statsd import load_from_config  # noqa: F401

kinto/core/testing.py

@@ -8,15 +8,19 @@ import webtest
 from cornice import errors as cornice_errors
 from pyramid.url import parse_url_overrides
 
-from kinto.core import DEFAULT_SETTINGS, statsd
+from kinto.core import DEFAULT_SETTINGS
 from kinto.core.storage import generators
 from kinto.core.utils import encode64, follow_subrequest, memcache, sqlalchemy
+from kinto.plugins import prometheus, statsd
 
 
 skip_if_ci = unittest.skipIf("CI" in os.environ, "ci")
 skip_if_no_postgresql = unittest.skipIf(sqlalchemy is None, "postgresql is not installed.")
 skip_if_no_memcached = unittest.skipIf(memcache is None, "memcached is not installed.")
 skip_if_no_statsd = unittest.skipIf(not statsd.statsd_module, "statsd is not installed.")
+skip_if_no_prometheus = unittest.skipIf(
+    not prometheus.prometheus_module, "prometheus is not installed."
+)
 
 
 class DummyRequest(mock.MagicMock):

kinto/core/utils.py

@@ -1,4 +1,5 @@
 import collections.abc as collections_abc
+import functools
 import hashlib
 import hmac
 import os
@@ -541,3 +542,10 @@ def apply_json_patch(obj, ops):
         raise ValueError(e)
 
     return result
+
+
+def safe_wraps(wrapper, *args, **kwargs):
+    """Safely wraps partial functions."""
+    while isinstance(wrapper, functools.partial):
+        wrapper = wrapper.func
+    return functools.wraps(wrapper, *args, **kwargs)

kinto/plugins/admin/VERSION

@@ -1 +1 @@
-3.0.3
+3.4.1

kinto/plugins/admin/build/VERSION

@@ -1 +1 @@
-3.0.3
+3.4.1