kinto 18.0.0__py3-none-any.whl → 19.3.2__py3-none-any.whl

kinto/__init__.py

@@ -38,6 +38,7 @@ DEFAULT_SETTINGS = {
     "record_id_generator": "kinto.views.RelaxedUUID",
     "project_name": "kinto",
     "admin_assets_path": None,
+    "metrics_matchdict_fields": ["bucket_id", "collection_id", "group_id", "record_id"],
 }
 
 

kinto/__main__.py

@@ -161,8 +161,7 @@ def main(args=None):
         if not backend:
             while True:
                 prompt = (
-                    "Select the backend you would like to use: "
-                    "(1 - postgresql, default - memory) "
+                    "Select the backend you would like to use: (1 - postgresql, default - memory) "
                 )
                 answer = input(prompt).strip()
                 try:

kinto/core/__init__.py

@@ -1,5 +1,5 @@
-"""Main entry point
-"""
+"""Main entry point"""
+
 import logging
 import tempfile
 
@@ -54,6 +54,7 @@ DEFAULT_SETTINGS = {
     "initialization_sequence": (
         "kinto.core.initialization.setup_request_bound_data",
         "kinto.core.initialization.setup_json_serializer",
+        "kinto.core.initialization.restrict_http_methods_if_readonly",
         "kinto.core.initialization.setup_csp_headers",
         "kinto.core.initialization.setup_logging",
         "kinto.core.initialization.setup_storage",
@@ -65,8 +66,8 @@ DEFAULT_SETTINGS = {
         "kinto.core.initialization.setup_authentication",
         "kinto.core.initialization.setup_backoff",
         "kinto.core.initialization.setup_sentry",
-        "kinto.core.initialization.setup_statsd",
         "kinto.core.initialization.setup_listeners",
+        "kinto.core.initialization.setup_metrics",
         "kinto.core.events.setup_transaction_hook",
     ),
     "event_listeners": "",
@@ -95,6 +96,7 @@ DEFAULT_SETTINGS = {
     "statsd_backend": "kinto.core.statsd",
     "statsd_prefix": "kinto.core",
     "statsd_url": None,
+    "metrics_matchdict_fields": [],
     "storage_backend": "",
     "storage_url": "",
     "storage_max_fetch_size": 10000,
@@ -108,10 +110,8 @@ DEFAULT_SETTINGS = {
     "trailing_slash_redirect_ttl_seconds": 3600,
     "multiauth.groupfinder": "kinto.core.authorization.groupfinder",
     "multiauth.policies": "",
-    "multiauth.policy.basicauth.use": (
-        "kinto.core.authentication." "BasicAuthAuthenticationPolicy"
-    ),
-    "multiauth.authorization_policy": ("kinto.core.authorization." "AuthorizationPolicy"),
+    "multiauth.policy.basicauth.use": "kinto.core.authentication.BasicAuthAuthenticationPolicy",
+    "multiauth.authorization_policy": "kinto.core.authorization.AuthorizationPolicy",
 }
 
 

kinto/core/initialization.py

@@ -1,6 +1,7 @@
 import logging
 import random
 import re
+import urllib.parse
 import warnings
 from datetime import datetime
 from secrets import token_hex
@@ -8,7 +9,12 @@ from secrets import token_hex
 from dateutil import parser as dateparser
 from pyramid.events import ApplicationCreated, NewRequest, NewResponse
 from pyramid.exceptions import ConfigurationError
-from pyramid.httpexceptions import HTTPBadRequest, HTTPGone, HTTPTemporaryRedirect
+from pyramid.httpexceptions import (
+    HTTPBadRequest,
+    HTTPGone,
+    HTTPMethodNotAllowed,
+    HTTPTemporaryRedirect,
+)
 from pyramid.interfaces import IAuthenticationPolicy
 from pyramid.renderers import JSON as JSONRenderer
 from pyramid.response import Response
@@ -16,7 +22,7 @@ from pyramid.security import NO_PERMISSION_REQUIRED
 from pyramid.settings import asbool, aslist
 from pyramid_multiauth import MultiAuthenticationPolicy, MultiAuthPolicySelected
 
-from kinto.core import cache, errors, permission, storage, utils
+from kinto.core import cache, errors, metrics, permission, storage, utils
 from kinto.core.events import ACTIONS, ResourceChanged, ResourceRead
 
 
@@ -78,6 +84,24 @@ def setup_csp_headers(config):
     config.add_subscriber(on_new_response, NewResponse)
 
 
+def restrict_http_methods_if_readonly(config):
+    """Prevent write operations if server is configured as read-only.
+
+    This is an additional layer of security on top of the verifications done
+    in :method:`kinto.core.Resource.register_service`, in case an installed
+    plugin does not take this setting into account in the definition of its
+    ad-hoc Pyramid views.
+    """
+    settings = config.get_settings()
+
+    def on_new_request(event):
+        if event.request.method.lower() not in ("get", "head", "options"):
+            raise HTTPMethodNotAllowed()
+
+    if asbool(settings["readonly"]):
+        config.add_subscriber(on_new_request, NewRequest)
+
+
 def setup_version_redirection(config):
     """Add a view which redirects to the current version of the API."""
     settings = config.get_settings()
@@ -189,7 +213,7 @@ def setup_deprecation(config):
 
 def _end_of_life_tween_factory(handler, registry):
     """Pyramid tween to handle service end of life."""
-    deprecation_msg = "The service you are trying to connect no longer exists" " at this location."
+    deprecation_msg = "The service you are trying to connect no longer exists at this location."
 
     def eos_tween(request):
         eos_date = registry.settings["eos"]
@@ -311,51 +335,13 @@ def setup_sentry(config):
 
 
 def setup_statsd(config):
-    settings = config.get_settings()
-    config.registry.statsd = None
-
-    if settings["statsd_url"]:
-        statsd_mod = settings["statsd_backend"]
-        statsd_mod = config.maybe_dotted(statsd_mod)
-        client = statsd_mod.load_from_config(config)
-
-        config.registry.statsd = client
-
-        client.watch_execution_time(config.registry.cache, prefix="backend")
-        client.watch_execution_time(config.registry.storage, prefix="backend")
-        client.watch_execution_time(config.registry.permission, prefix="backend")
-
-        # Commit so that configured policy can be queried.
-        config.commit()
-        policy = config.registry.queryUtility(IAuthenticationPolicy)
-        if isinstance(policy, MultiAuthenticationPolicy):
-            for name, subpolicy in policy.get_policies():
-                client.watch_execution_time(subpolicy, prefix="authentication", classname=name)
-        else:
-            client.watch_execution_time(policy, prefix="authentication")
-
-        def on_new_response(event):
-            request = event.request
-
-            # Count unique users.
-            user_id = request.prefixed_userid
-            if user_id:
-                # Get rid of colons in metric packet (see #1282).
-                user_id = user_id.replace(":", ".")
-                client.count("users", unique=user_id)
-
-            # Count authentication verifications.
-            if hasattr(request, "authn_type"):
-                client.count(f"authn_type.{request.authn_type}")
-
-            # Count view calls.
-            service = request.current_service
-            if service:
-                client.count(f"view.{service.name}.{request.method}")
-
-        config.add_subscriber(on_new_response, NewResponse)
-
-        return client
+    # It would be pretty rare to find users that have a custom ``kinto.initialization_sequence`` setting.
+    # But just in case, warn that it will be removed in next major.
+    warnings.warn(
+        "``setup_statsd()`` is now deprecated. Use ``kinto.core.initialization.setup_metrics()`` instead.",
+        DeprecationWarning,
+    )
+    setup_metrics(config)
 
 
 def install_middlewares(app, settings):
@@ -443,6 +429,126 @@ def setup_logging(config):
     config.add_subscriber(on_new_response, NewResponse)
 
 
+def setup_metrics(config):
+    settings = config.get_settings()
+
+    # Register a no-op metrics service by default.
+    config.registry.registerUtility(metrics.NoOpMetricsService(), metrics.IMetricsService)
+
+    # This does not fully respect the Pyramid/ZCA patterns, but the rest of Kinto uses
+    # `registry.storage`, `registry.cache`, etc. Consistency seems more important.
+    config.registry.__class__.metrics = property(
+        lambda reg: reg.queryUtility(metrics.IMetricsService)
+    )
+
+    def deprecated_registry(self):
+        warnings.warn(
+            "``config.registry.statsd`` is now deprecated. Use ``config.registry.metrics`` instead.",
+            DeprecationWarning,
+        )
+        return self.metrics
+
+    config.registry.__class__.statsd = property(deprecated_registry)
+
+    def on_app_created(event):
+        config = event.app
+        metrics_service = config.registry.metrics
+
+        metrics.watch_execution_time(metrics_service, config.registry.cache, prefix="backend")
+        metrics.watch_execution_time(metrics_service, config.registry.storage, prefix="backend")
+        metrics.watch_execution_time(metrics_service, config.registry.permission, prefix="backend")
+
+        policy = config.registry.queryUtility(IAuthenticationPolicy)
+        if isinstance(policy, MultiAuthenticationPolicy):
+            for name, subpolicy in policy.get_policies():
+                metrics.watch_execution_time(
+                    metrics_service, subpolicy, prefix="authentication", classname=name
+                )
+        else:
+            metrics.watch_execution_time(metrics_service, policy, prefix="authentication")
+
+    config.add_subscriber(on_app_created, ApplicationCreated)
+
+    def on_new_response(event):
+        request = event.request
+        metrics_service = config.registry.metrics
+
+        try:
+            endpoint = utils.strip_uri_prefix(request.path)
+            endpoint = urllib.parse.quote_plus(endpoint, safe="/?&=-_")
+        except UnicodeDecodeError as e:
+            # This `on_new_response` callback is also called when a HTTP 400
+            # is returned because of an invalid UTF-8 path. We still want metrics.
+            endpoint = str(e)
+
+        # Count unique users.
+        user_id = request.prefixed_userid
+        if user_id:
+            # Get rid of colons in metric packet (see #1282).
+            auth, user_id = user_id.split(":")
+            metrics_service.count("users", unique=[("auth", auth), ("userid", user_id)])
+
+        # Add extra labels to metrics, based on fields extracted from the request matchdict.
+        metrics_matchdict_fields = aslist(settings["metrics_matchdict_fields"])
+        # Turn the `id` field of object endpoints into `{resource}_id` (eg. `mushroom_id`, `bucket_id`)
+        enhanced_matchdict = dict(request.matchdict or {})
+        try:
+            enhanced_matchdict[request.current_resource_name + "_id"] = enhanced_matchdict.get(
+                "id", ""
+            )
+        except AttributeError:
+            # Not on a resource.
+            pass
+        metrics_matchdict_labels = [
+            (field, enhanced_matchdict.get(field, "")) for field in metrics_matchdict_fields
+        ]
+
+        # Count served requests.
+        metrics_service.count(
+            "request_summary",
+            unique=[
+                ("method", request.method.lower()),
+                ("endpoint", endpoint),
+                ("status", str(event.response.status_code)),
+            ]
+            + metrics_matchdict_labels,
+        )
+
+        try:
+            current = utils.msec_time()
+            duration = current - request._received_at
+            metrics_service.observe(
+                "request_duration",
+                duration,
+                labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
+            )
+        except AttributeError:  # pragma: no cover
+            # Logging was not setup in this Kinto app (unlikely but possible)
+            pass
+
+        # Observe response size.
+        metrics_service.observe(
+            "request_size",
+            len(event.response.body or b""),
+            labels=[("endpoint", endpoint)] + metrics_matchdict_labels,
+        )
+
+        # Count authentication verifications.
+        if hasattr(request, "authn_type"):
+            metrics_service.count(f"authn_type.{request.authn_type}")
+
+        # Count view calls.
+        service = request.current_service
+        if service:
+            metrics_service.count(f"view.{service.name}.{request.method}")
+
+    config.add_subscriber(on_new_response, NewResponse)
+
+    # While statsd is deprecated, we include its plugin by default for retro-compability.
+    if settings["statsd_url"]:
+        config.include("kinto.plugins.statsd")
+
+
 class EventActionFilter:
     def __init__(self, actions, config):
         actions = ACTIONS.from_string_list(actions)
@@ -495,11 +601,9 @@ def setup_listeners(config):
             listener_mod = config.maybe_dotted(module_value)
             listener = listener_mod.load_from_config(config, prefix)
 
-        # If StatsD is enabled, monitor execution time of listeners.
-        if getattr(config.registry, "statsd", None):
-            statsd_client = config.registry.statsd
-            key = f"listeners.{name}"
-            listener = statsd_client.timer(key)(listener.__call__)
+        wrapped_listener = metrics.listener_with_timer(
+            config, f"listeners.{name}", listener.__call__
+        )
 
         # Optional filter by event action.
         actions_setting = prefix + "actions"
@@ -525,11 +629,11 @@ def setup_listeners(config):
         options = dict(for_actions=actions, for_resources=resource_names)
 
         if ACTIONS.READ in actions:
-            config.add_subscriber(listener, ResourceRead, **options)
+            config.add_subscriber(wrapped_listener, ResourceRead, **options)
             actions = [a for a in actions if a != ACTIONS.READ]
 
         if len(actions) > 0:
-            config.add_subscriber(listener, ResourceChanged, **options)
+            config.add_subscriber(wrapped_listener, ResourceChanged, **options)
 
 
 def load_default_settings(config, default_settings):

kinto/core/metrics.py

@@ -0,0 +1,93 @@
+import types
+
+from zope.interface import Interface, implementer
+
+from kinto.core import utils
+
+
+class IMetricsService(Interface):
+    """
+    An interface that defines the metrics service contract.
+    Any class implementing this must provide all its methods.
+    """
+
+    def timer(key):
+        """
+        Watch execution time.
+        """
+
+    def observe(self, key, value, labels=[]):
+        """
+        Observe a give `value` for the specified `key`.
+        """
+
+    def count(key, count=1, unique=None):
+        """
+        Count occurrences. If `unique` is set, overwrites the counter value
+        on each call.
+
+        `unique` should be of type ``list[tuple[str,str]]``.
+        """
+
+
+class NoOpTimer:
+    def __call__(self, f):
+        @utils.safe_wraps(f)
+        def _wrapped(*args, **kwargs):
+            return f(*args, **kwargs)
+
+        return _wrapped
+
+    def __enter__(self):
+        pass
+
+    def __exit__(self, *args, **kwargs):
+        pass
+
+
+@implementer(IMetricsService)
+class NoOpMetricsService:
+    def timer(self, key):
+        return NoOpTimer()
+
+    def observe(self, key, value, labels=[]):
+        pass
+
+    def count(self, key, count=1, unique=None):
+        pass
+
+
+def watch_execution_time(metrics_service, obj, prefix="", classname=None):
+    """
+    Decorate all methods of an object in order to watch their execution time.
+    Metrics will be named `{prefix}.{classname}.{method}`.
+    """
+    classname = classname or utils.classname(obj)
+    members = dir(obj)
+    for name in members:
+        value = getattr(obj, name)
+        is_method = isinstance(value, types.MethodType)
+        if not name.startswith("_") and is_method:
+            statsd_key = f"{prefix}.{classname}.{name}"
+            decorated_method = metrics_service.timer(statsd_key)(value)
+            setattr(obj, name, decorated_method)
+
+
+def listener_with_timer(config, key, func):
+    """
+    Add a timer with the specified `key` on the specified `func`.
+    This is used to avoid evaluating `config.registry.metrics` during setup time
+    to avoid having to deal with initialization order and configuration committing.
+    """
+
+    def wrapped(*args, **kwargs):
+        metrics_service = config.registry.metrics
+        if not metrics_service:
+            # This only happens if `kinto.core.initialization.setup_metrics` is
+            # not listed in the `initialization_sequence` setting.
+            return func(*args, **kwargs)
+        # If metrics are enabled, monitor execution time of listeners.
+        with metrics_service.timer(key):
+            return func(*args, **kwargs)
+
+    return wrapped

kinto/core/permission/postgresql/__init__.py

@@ -246,7 +246,7 @@ class Permission(PermissionBase, MigratorMixin):
 
         query = f"""
         WITH required_perms AS (
-          VALUES {','.join(perm_values)}
+          VALUES {",".join(perm_values)}
         )
         SELECT principal
           FROM required_perms JOIN access_control_entries
@@ -292,14 +292,14 @@ class Permission(PermissionBase, MigratorMixin):
                 object_id_condition = "object_id LIKE pattern"
             else:
                 object_id_condition = (
-                    "object_id LIKE pattern " "AND object_id NOT LIKE pattern || '/%'"
+                    "object_id LIKE pattern AND object_id NOT LIKE pattern || '/%'"
                 )
             query = f"""
             WITH required_perms AS (
-              VALUES {','.join(perm_values)}
+              VALUES {",".join(perm_values)}
             ),
             user_principals AS (
-              VALUES {','.join(principals_values)}
+              VALUES {",".join(principals_values)}
             ),
             potential_objects AS (
               SELECT object_id, permission, required_perms.column1 AS pattern
@@ -341,7 +341,7 @@ class Permission(PermissionBase, MigratorMixin):
 
         query = f"""
         WITH required_perms AS (
-          VALUES {','.join(perms_values)}
+          VALUES {",".join(perms_values)}
         ),
         allowed_principals AS (
           SELECT principal
@@ -349,7 +349,7 @@ class Permission(PermissionBase, MigratorMixin):
               ON (object_id = column1 AND permission = column2)
         ),
         required_principals AS (
-          VALUES {','.join(principals_values)}
+          VALUES {",".join(principals_values)}
         )
         SELECT COUNT(*) AS matched
           FROM required_principals JOIN allowed_principals
@@ -412,7 +412,7 @@ class Permission(PermissionBase, MigratorMixin):
         if not new_aces:
             query = f"""
             WITH specified_perms AS (
-              VALUES {','.join(specified_perms)}
+              VALUES {",".join(specified_perms)}
             )
             DELETE FROM access_control_entries
              USING specified_perms
@@ -422,7 +422,7 @@ class Permission(PermissionBase, MigratorMixin):
         else:
             query = f"""
             WITH specified_perms AS (
-              VALUES {','.join(specified_perms)}
+              VALUES {",".join(specified_perms)}
             ),
             delete_specified AS (
               DELETE FROM access_control_entries
@@ -435,7 +435,7 @@ class Permission(PermissionBase, MigratorMixin):
               UNION SELECT :object_id
             ),
             new_aces AS (
-              VALUES {','.join(new_aces)}
+              VALUES {",".join(new_aces)}
             )
             INSERT INTO access_control_entries(object_id, permission, principal)
               SELECT DISTINCT d.object_id, n.column1, n.column2

kinto/core/resource/__init__.py

@@ -665,7 +665,7 @@ class Resource:
         obj = self._get_object_or_404(self.object_id)
         self._raise_412_if_modified(obj)
 
-        # Retreive the last_modified information from a querystring if present.
+        # Retrieve the last_modified information from a querystring if present.
         last_modified = self.request.validated["querystring"].get("last_modified")
 
         # If less or equal than current object. Ignore it.
@@ -1058,6 +1058,11 @@ class Resource:
 
     def _extract_filters(self):
         """Extracts filters from QueryString parameters."""
+
+        def is_valid_timestamp(value):
+            # Is either integer, or integer as string, or integer between 2 quotes.
+            return isinstance(value, int) or re.match(r'^(\d+)$|^("\d+")$', str(value))
+
         queryparams = self.request.validated["querystring"]
 
         filters = []
@@ -1081,7 +1086,7 @@ class Resource:
                     operator = COMPARISON.GT
                 else:
                     if param == "_to":
-                        message = "_to is now deprecated, " "you should use _before instead"
+                        message = "_to is now deprecated, you should use _before instead"
                         url = (
                             "https://kinto.readthedocs.io/en/2.4.0/api/"
                             "resource.html#list-of-available-url-"
@@ -1090,7 +1095,7 @@ class Resource:
                         send_alert(self.request, message, url)
                     operator = COMPARISON.LT
 
-                if value == "" or not isinstance(value, (int, str, type(None))):
+                if value is not None and not is_valid_timestamp(value):
                     raise_invalid(self.request, **error_details)
 
                 filters.append(Filter(self.model.modified_field, value, operator))
@@ -1127,7 +1132,7 @@ class Resource:
                 error_details["description"] = "Invalid character 0x00"
                 raise_invalid(self.request, **error_details)
 
-            if field == self.model.modified_field and value == "":
+            if field == self.model.modified_field and not is_valid_timestamp(value):
                 raise_invalid(self.request, **error_details)
 
             filters.append(Filter(field, value, operator))

kinto/core/resource/schema.py

@@ -37,8 +37,7 @@ class URL(URL):
 
     def __init__(self, *args, **kwargs):
         message = (
-            "`kinto.core.resource.schema.URL` is deprecated, "
-            "use `kinto.core.schema.URL` instead."
+            "`kinto.core.resource.schema.URL` is deprecated, use `kinto.core.schema.URL` instead."
         )
         warnings.warn(message, DeprecationWarning)
         super().__init__(*args, **kwargs)

kinto/core/scripts.py

@@ -1,6 +1,7 @@
 """
 kinto.core.scripts: utilities to build admin scripts for kinto-based services
 """
+
 import logging
 
 from pyramid.settings import asbool

kinto/core/statsd.py

@@ -1,63 +1 @@
-import types
-from urllib.parse import urlparse
-
-from pyramid.exceptions import ConfigurationError
-
-from kinto.core import utils
-
-
-try:
-    import statsd as statsd_module
-except ImportError:  # pragma: no cover
-    statsd_module = None
-
-
-class Client:
-    def __init__(self, host, port, prefix):
-        self._client = statsd_module.StatsClient(host, port, prefix=prefix)
-
-    def watch_execution_time(self, obj, prefix="", classname=None):
-        classname = classname or utils.classname(obj)
-        members = dir(obj)
-        for name in members:
-            value = getattr(obj, name)
-            is_method = isinstance(value, types.MethodType)
-            if not name.startswith("_") and is_method:
-                statsd_key = f"{prefix}.{classname}.{name}"
-                decorated_method = self.timer(statsd_key)(value)
-                setattr(obj, name, decorated_method)
-
-    def timer(self, key):
-        return self._client.timer(key)
-
-    def count(self, key, count=1, unique=None):
-        if unique is None:
-            return self._client.incr(key, count=count)
-        else:
-            return self._client.set(key, unique)
-
-
-def statsd_count(request, count_key):
-    statsd = request.registry.statsd
-    if statsd:
-        statsd.count(count_key)
-
-
-def load_from_config(config):
-    # If this is called, it means that a ``statsd_url`` was specified in settings.
-    # (see ``kinto.core.initialization``)
-    # Raise a proper error if the ``statsd`` module is not installed.
-    if statsd_module is None:
-        error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
-        raise ConfigurationError(error_msg)
-
-    settings = config.get_settings()
-    uri = settings["statsd_url"]
-    uri = urlparse(uri)
-
-    if settings["project_name"] != "":
-        prefix = settings["project_name"]
-    else:
-        prefix = settings["statsd_prefix"]
-
-    return Client(uri.hostname, uri.port, prefix)
+from kinto.plugins.statsd import load_from_config  # noqa: F401

kinto/core/storage/exceptions.py

@@ -1,5 +1,4 @@
-"""Exceptions raised by storage backend.
-"""
+"""Exceptions raised by storage backend."""
 
 
 class BackendError(Exception):

kinto/core/storage/postgresql/client.py

@@ -95,14 +95,14 @@ def create_from_config(config, prefix="", with_transaction=True):
     url = filtered_settings[prefix + "url"]
     existing_client = _CLIENTS[transaction_per_request].get(url)
     if existing_client:
-        msg = "Reuse existing PostgreSQL connection. " f"Parameters {prefix}* will be ignored."
+        msg = f"Reuse existing PostgreSQL connection. Parameters {prefix}* will be ignored."
         warnings.warn(msg)
         return existing_client
 
     # Initialize SQLAlchemy engine from filtered_settings.
     poolclass_key = prefix + "poolclass"
     filtered_settings.setdefault(
-        poolclass_key, ("kinto.core.storage.postgresql." "pool.QueuePoolWithMaxBacklog")
+        poolclass_key, ("kinto.core.storage.postgresql.pool.QueuePoolWithMaxBacklog")
     )
     filtered_settings[poolclass_key] = config.maybe_dotted(filtered_settings[poolclass_key])
     engine = sqlalchemy.engine_from_config(filtered_settings, prefix=prefix, url=url)

kinto/core/storage/utils.py

@@ -1,6 +1,7 @@
 """
 kinto.core.storage.utils: methods for making it easier to work with kinto storage objects
 """
+
 from kinto.core.storage import Filter
 from kinto.core.utils import COMPARISON