kinto 20.6.0__tar.gz → 21.0.0__tar.gz

{kinto-20.6.0 → kinto-21.0.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.6.0
+Version: 21.0.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.6.0 → kinto-21.0.0}/constraints.txt

@@ -73,7 +73,7 @@ markupsafe==2.1.4
     # via werkzeug
 msgpack==1.0.7
     # via bravado-core
-newrelic==10.8.1
+newrelic==10.9.0
     # via -r constraints.in
 packaging==23.2
     # via
@@ -149,7 +149,7 @@ rpds-py==0.17.1
     # via
     #   jsonschema
     #   referencing
-ruff==0.11.4
+ruff==0.11.6
     # via -r constraints.in
 sentry-sdk==2.8.0
     # via -r constraints.in

{kinto-20.6.0 → kinto-21.0.0}/kinto/core/initialization.py

@@ -1,7 +1,6 @@
 import logging
 import random
 import re
-import urllib.parse
 import warnings
 from datetime import datetime
 
@@ -466,14 +465,6 @@ def setup_metrics(config):
         request = event.request
         metrics_service = config.registry.metrics
 
-        try:
-            endpoint = utils.strip_uri_prefix(request.path)
-            endpoint = urllib.parse.quote_plus(endpoint, safe="/?&=-_")
-        except UnicodeDecodeError as e:
-            # This `on_new_response` callback is also called when a HTTP 400
-            # is returned because of an invalid UTF-8 path. We still want metrics.
-            endpoint = str(e)
-
         # Count unique users.
         user_id = request.prefixed_userid
         if user_id:
@@ -496,13 +487,27 @@ def setup_metrics(config):
             (field, enhanced_matchdict.get(field, "")) for field in metrics_matchdict_fields
         ]
 
+        status = event.response.status_code
+
+        service = request.current_service
+        if service:
+            # Use the service name as endpoint if available.
+            endpoint = service.name
+        elif route := request.matched_route:
+            # Use the route name as endpoint if we're not on a Cornice service.
+            endpoint = route.name
+        else:
+            endpoint = (
+                "unnamed" if status != 404 else "unknown"
+            )  # Do not multiply cardinality for unknown endpoints.
+
         # Count served requests.
         metrics_service.count(
             "request_summary",
             unique=[
                 ("method", request.method.lower()),
                 ("endpoint", endpoint),
-                ("status", str(event.response.status_code)),
+                ("status", str(status)),
             ]
             + metrics_matchdict_labels,
         )
@@ -510,10 +515,11 @@ def setup_metrics(config):
         try:
             current = utils.msec_time()
             duration = current - request._received_at
-            metrics_service.observe(
+            metrics_service.timer(
                 "request_duration",
-                duration,
-                labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
+                value=duration,
+                labels=[("endpoint", endpoint), ("method", request.method.lower())]
+                + metrics_matchdict_labels,
             )
         except AttributeError:  # pragma: no cover
             # Logging was not setup in this Kinto app (unlikely but possible)
@@ -527,13 +533,11 @@ def setup_metrics(config):
         )
 
         # Count authentication verifications.
-        if hasattr(request, "authn_type"):
-            metrics_service.count(f"authn_type.{request.authn_type}")
-
-        # Count view calls.
-        service = request.current_service
-        if service:
-            metrics_service.count(f"view.{service.name}.{request.method}")
+        try:
+            metrics_service.count("authentication", unique=[("type", request.authn_type)])
+        except AttributeError:
+            # Not authenticated
+            pass
 
     config.add_subscriber(on_new_response, NewResponse)
 

{kinto-20.6.0 → kinto-21.0.0}/kinto/core/metrics.py

@@ -47,7 +47,7 @@ class NoOpTimer:
 
 @implementer(IMetricsService)
 class NoOpMetricsService:
-    def timer(self, key):
+    def timer(self, key, value=None, labels=[]):
         return NoOpTimer()
 
     def observe(self, key, value, labels=[]):
@@ -65,11 +65,12 @@ def watch_execution_time(metrics_service, obj, prefix="", classname=None):
     classname = classname or utils.classname(obj)
     members = dir(obj)
     for name in members:
-        value = getattr(obj, name)
-        is_method = isinstance(value, types.MethodType)
+        method = getattr(obj, name)
+        is_method = isinstance(method, types.MethodType)
         if not name.startswith("_") and is_method:
-            statsd_key = f"{prefix}.{classname}.{name}"
-            decorated_method = metrics_service.timer(statsd_key)(value)
+            statsd_key = f"{prefix}.{classname}"
+            labels = [("method", name)]
+            decorated_method = metrics_service.timer(statsd_key, labels=labels)(method)
             setattr(obj, name, decorated_method)
 
 

{kinto-20.6.0 → kinto-21.0.0}/kinto/plugins/prometheus.py

@@ -49,10 +49,27 @@ def _fix_metric_name(s):
 
 
 class Timer:
-    def __init__(self, summary):
-        self.summary = summary
+    """
+    A decorator to time the execution of a function. It will use the
+    `prometheus_client.Histogram` to record the time taken by the function
+    in milliseconds. The histogram is passed as an argument to the
+    constructor.
+
+    Main limitation: it does not support `labels` on the decorator.
+    """
+
+    def __init__(self, histogram):
+        self.histogram = histogram
         self._start_time = None
 
+    def set_labels(self, labels):
+        if not labels:
+            return
+        self.histogram = self.histogram.labels(*(label_value for _, label_value in labels))
+
+    def observe(self, value):
+        return self.histogram.observe(value)
+
     def __call__(self, f):
         @safe_wraps(f)
         def _wrapped(*args, **kwargs):
@@ -61,7 +78,7 @@ class Timer:
                 return f(*args, **kwargs)
             finally:
                 dt_ms = 1000.0 * (time_now() - start_time)
-                self.summary.observe(dt_ms)
+                self.histogram.observe(dt_ms)
 
         return _wrapped
 
@@ -79,7 +96,7 @@ class Timer:
         if self._start_time is None:  # pragma: nocover
             raise RuntimeError("Timer has not started.")
         dt_ms = 1000.0 * (time_now() - self._start_time)
-        self.summary.observe(dt_ms)
+        self.histogram.observe(dt_ms)
         return self
 
 
@@ -95,21 +112,34 @@ class PrometheusService:
             prefix_clean = _fix_metric_name(prefix).replace("_", "") + "_"
         self.prefix = prefix_clean.lower()
 
-    def timer(self, key):
+    def timer(self, key, value=None, labels=[]):
         global _METRICS
         key = self.prefix + key
 
         if key not in _METRICS:
-            _METRICS[key] = prometheus_module.Summary(
-                _fix_metric_name(key), f"Summary of {key}", registry=get_registry()
+            _METRICS[key] = prometheus_module.Histogram(
+                _fix_metric_name(key),
+                f"Histogram of {key}",
+                registry=get_registry(),
+                labelnames=[label_name for label_name, _ in labels],
             )
 
-        if not isinstance(_METRICS[key], prometheus_module.Summary):
+        if not isinstance(_METRICS[key], prometheus_module.Histogram):
             raise RuntimeError(
                 f"Metric {key} already exists with different type ({_METRICS[key]})"
             )
 
-        return Timer(_METRICS[key])
+        timer = Timer(_METRICS[key])
+        timer.set_labels(labels)
+
+        if value is not None:
+            # We are timing something.
+            return timer.observe(value)
+
+        # We are not timing anything, just returning the timer object
+        # (eg. to be used as decorator or context manager).
+        # Note that in this case, the labels values will be the same for all calls.
+        return timer
 
     def observe(self, key, value, labels=[]):
         global _METRICS
@@ -188,9 +218,8 @@ def metrics_view(request):
 
 
 def _reset_multiproc_folder_content():  # pragma: no cover
-    if os.path.exists(PROMETHEUS_MULTIPROC_DIR):
-        shutil.rmtree(PROMETHEUS_MULTIPROC_DIR)
-    os.mkdir(PROMETHEUS_MULTIPROC_DIR)
+    shutil.rmtree(PROMETHEUS_MULTIPROC_DIR, ignore_errors=True)
+    os.makedirs(PROMETHEUS_MULTIPROC_DIR, exist_ok=True)
 
 
 def includeme(config):

{kinto-20.6.0 → kinto-21.0.0}/kinto/plugins/statsd.py

@@ -1,4 +1,5 @@
 import warnings
+from datetime import timedelta
 from urllib.parse import urlparse
 
 from pyramid.exceptions import ConfigurationError
@@ -26,7 +27,13 @@ class StatsDService:
     def __init__(self, host, port, prefix):
         self._client = statsd_module.StatsClient(host, port, prefix=prefix)
 
-    def timer(self, key):
+    def timer(self, key, value=None, labels=[]):
+        if labels:
+            # [("method", "get")] -> "method.get"
+            key = f"{key}." + ".".join(f"{label[0]}.{sanitize(label[1])}" for label in labels)
+        if value:
+            value = timedelta(seconds=value)
+            return self._client.timing(key, value)
         return self._client.timer(key)
 
     def observe(self, key, value, labels=[]):

{kinto-20.6.0 → kinto-21.0.0}/kinto.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kinto
-Version: 20.6.0
+Version: 21.0.0
 Summary: Kinto Web Service - Store, Sync, Share, and Self-Host.
 Author-email: Mozilla Services <developers@kinto-storage.org>
 License: Copyright 2012 - Mozilla Foundation

{kinto-20.6.0 → kinto-21.0.0}/tests/core/test_initialization.py

@@ -386,7 +386,7 @@ class MetricsConfigurationTest(unittest.TestCase):
         kinto.core.initialize(self.config, "0.0.1", "settings_prefix")
         app = webtest.TestApp(self.config.make_wsgi_app())
         app.get("/v0/", headers=get_user_headers("bob"))
-        self.mocked().count.assert_any_call("authn_type.basicauth")
+        self.mocked().count.assert_any_call("authentication", unique=[("type", "basicauth")])
 
     #
     # Endpoints.
@@ -404,7 +404,7 @@ class MetricsConfigurationTest(unittest.TestCase):
         app.get("/v0/__heartbeat__")
         self.mocked().count.assert_any_call(
             "request_summary",
-            unique=[("method", "get"), ("endpoint", "/__heartbeat__"), ("status", "200")],
+            unique=[("method", "get"), ("endpoint", "heartbeat"), ("status", "200")],
         )
 
     def test_statsd_sanitizes_url_in_metrics(self):
@@ -420,7 +420,7 @@ class MetricsConfigurationTest(unittest.TestCase):
                 ("method", "get"),
                 (
                     "endpoint",
-                    "/changeset%27%257C%2522%253F%253E%253C%21DOCTYPE%2522http%3A//xh3E%27%29%2C%27/l%27%29%2520from%2520dual%29%257C%27",
+                    "unknown",
                 ),
                 ("status", "404"),
             ],
@@ -433,25 +433,19 @@ class MetricsConfigurationTest(unittest.TestCase):
         self.mocked().observe.assert_any_call(
             "request_size",
             len("{}"),
-            labels=[("endpoint", "/__heartbeat__")],
+            labels=[("endpoint", "heartbeat")],
         )
 
     def test_statsd_observe_request_duration(self):
         kinto.core.initialize(self.config, "0.0.1", "settings_prefix")
         app = webtest.TestApp(self.config.make_wsgi_app())
         app.get("/v0/__heartbeat__")
-        self.mocked().observe.assert_any_call(
+        self.mocked().timer.assert_any_call(
             "request_duration",
-            mock.ANY,
-            labels=[("endpoint", "/__heartbeat__")],
+            value=mock.ANY,
+            labels=[("endpoint", "heartbeat"), ("method", "get")],
         )
 
-    def test_statsd_counts_views_and_methods(self):
-        kinto.core.initialize(self.config, "0.0.1", "settings_prefix")
-        app = webtest.TestApp(self.config.make_wsgi_app())
-        app.get("/v0/__heartbeat__")
-        self.mocked().count.assert_any_call("view.heartbeat.GET")
-
     def test_statsd_counts_unknown_urls(self):
         kinto.core.initialize(self.config, "0.0.1", "settings_prefix")
         app = webtest.TestApp(self.config.make_wsgi_app())

{kinto-20.6.0 → kinto-21.0.0}/tests/core/test_metrics.py

@@ -24,7 +24,9 @@ class WatchExecutionTimeTest(unittest.TestCase):
 
     def test_public_methods_generates_statsd_calls(self):
         self.test_object.test_method()
-        self.mocked.timer.assert_called_with("test.testedclass.test_method")
+        self.mocked.timer.assert_called_with(
+            "test.testedclass", labels=[("method", "test_method")]
+        )
 
     def test_private_methods_does_not_generates_statsd_calls(self):
         self.test_object._private_method()

{kinto-20.6.0 → kinto-21.0.0}/tests/plugins/test_prometheus.py

@@ -60,7 +60,7 @@ class ServiceTest(PrometheusWebTest):
             self.assertEqual(my_func(1, 1), 2)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("TYPE kintoprod_func_latency_context summary", resp.text)
+        self.assertIn("TYPE kintoprod_func_latency_context histogram", resp.text)
 
     def test_timer_can_be_used_as_decorator(self):
         decorated = self.app.app.registry.metrics.timer("func.latency.decorator")(my_func)
@@ -68,7 +68,7 @@ class ServiceTest(PrometheusWebTest):
         self.assertEqual(decorated(1, 1), 2)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("TYPE kintoprod_func_latency_decorator summary", resp.text)
+        self.assertIn("TYPE kintoprod_func_latency_decorator histogram", resp.text)
 
     def test_timer_can_be_used_as_decorator_on_partial_function(self):
         partial = functools.partial(my_func, 3)
@@ -77,7 +77,7 @@ class ServiceTest(PrometheusWebTest):
         self.assertEqual(decorated(3), 6)
 
         resp = self.app.get("/__metrics__")
-        self.assertIn("TYPE kintoprod_func_latency_partial summary", resp.text)
+        self.assertIn("TYPE kintoprod_func_latency_partial histogram", resp.text)
 
     def test_observe_a_single_value(self):
         self.app.app.registry.metrics.observe("price", 111)

{kinto-20.6.0 → kinto-21.0.0}/tests/test_views_metrics.py

@@ -26,20 +26,38 @@ class ViewsMetricsTest(BaseWebTest, unittest.TestCase):
         self.app.put("/buckets/beers/collections/barley", headers=self.headers)
         self.app.put("/buckets/beers/collections/barley/records/abc", headers=self.headers)
 
+        self.app.get("/buckets", headers=self.headers)
+        self.app.get("/buckets/beers/collections", headers=self.headers)
+        self.app.get("/buckets/beers/collections/barley/records", headers=self.headers)
+
         resp = self.app.get("/__metrics__")
+        print(resp.text)
+        self.assertIn(
+            'request_size_sum{bucket_id="beers",collection_id="",endpoint="bucket-object",group_id="",record_id=""}',
+            resp.text,
+        )
         self.assertIn(
-            'request_size_sum{bucket_id="beers",collection_id="",endpoint="/buckets/beers",group_id="",record_id=""}',
+            'request_size_sum{bucket_id="beers",collection_id="",endpoint="group-object",group_id="amateurs",record_id=""}',
             resp.text,
         )
         self.assertIn(
-            'request_size_sum{bucket_id="beers",collection_id="",endpoint="/buckets/beers/groups/amateurs",group_id="amateurs",record_id=""}',
+            'request_summary_total{bucket_id="beers",collection_id="barley",endpoint="collection-object",group_id="",method="put",record_id="",status="201"}',
+            resp.text,
+        )
+        self.assertIn(
+            'request_duration_sum{bucket_id="beers",collection_id="barley",endpoint="record-object",group_id="",method="put",record_id="abc"}',
+            resp.text,
+        )
+
+        self.assertIn(
+            'request_summary_total{bucket_id="",collection_id="",endpoint="bucket-plural",group_id="",method="get",record_id="",status="200"}',
             resp.text,
         )
         self.assertIn(
-            'request_summary_total{bucket_id="beers",collection_id="barley",endpoint="/buckets/beers/collections/barley",group_id="",method="put",record_id="",status="201"}',
+            'request_summary_total{bucket_id="beers",collection_id="",endpoint="collection-plural",group_id="",method="get",record_id="",status="200"}',
             resp.text,
         )
         self.assertIn(
-            'request_duration_sum{bucket_id="beers",collection_id="barley",endpoint="/buckets/beers/collections/barley/records/abc",group_id="",record_id="abc"}',
+            'request_summary_total{bucket_id="beers",collection_id="barley",endpoint="record-plural",group_id="",method="get",record_id="",status="200"}',
             resp.text,
         )