kinto 23.2.1__py3-none-any.whl

kinto/plugins/openid/__init__.py

@@ -0,0 +1,131 @@
+import requests
+from pyramid import authentication as base_auth
+from pyramid.interfaces import IAuthenticationPolicy
+from pyramid.settings import aslist
+from zope.interface import implementer
+
+from kinto.core import logger
+from kinto.core import utils as core_utils
+from kinto.core.openapi import OpenAPI
+
+from .utils import fetch_openid_config
+
+
+@implementer(IAuthenticationPolicy)
+class OpenIDConnectPolicy(base_auth.CallbackAuthenticationPolicy):
+    def __init__(self, issuer, client_id, realm="Realm", **kwargs):
+        self.realm = realm
+        self.issuer = issuer
+        self.client_id = client_id
+        self.client_secret = kwargs.get("client_secret", "")
+        self.header_type = kwargs.get("header_type", "Bearer")
+        self.userid_field = kwargs.get("userid_field", "sub")
+        self.verification_ttl = int(kwargs.get("verification_ttl_seconds", 86400))
+
+        # Fetch OpenID config (at instantiation, ie. startup)
+        self.oid_config = fetch_openid_config(issuer)
+
+        self._jwt_keys = None
+
+    def unauthenticated_userid(self, request):
+        """Return the userid or ``None`` if token could not be verified."""
+        settings = request.registry.settings
+        hmac_secret = settings["userid_hmac_secret"]
+
+        authorization = request.headers.get("Authorization", "")
+        try:
+            authmeth, access_token = authorization.split(" ", 1)
+        except ValueError:
+            return None
+
+        if authmeth.lower() != self.header_type.lower():
+            return None
+
+        # XXX JWT Access token
+        # https://auth0.com/docs/tokens/access-token#access-token-format
+
+        # Check cache if these tokens were already verified.
+        hmac_tokens = core_utils.hmac_digest(hmac_secret, access_token)
+        cache_key = f"openid:verify:{hmac_tokens}"
+        payload = request.registry.cache.get(cache_key)
+        if payload is None:
+            # This can take some time.
+            payload = self._verify_token(access_token)
+            if payload is None:
+                return None
+        # Save for next time / refresh ttl.
+        request.registry.cache.set(cache_key, payload, ttl=self.verification_ttl)
+        request.bound_data["user_profile"] = payload
+        # Extract meaningful field from userinfo (eg. email or sub)
+        return payload.get(self.userid_field)
+
+    def forget(self, request):
+        """A no-op. Credentials are sent on every request.
+        Return WWW-Authenticate Realm header for Bearer token.
+        """
+        return [("WWW-Authenticate", '%s realm="%s"' % (self.header_type, self.realm))]
+
+    def _verify_token(self, access_token):
+        uri = self.oid_config["userinfo_endpoint"]
+        # Opaque access token string. Fetch user info from profile.
+        try:
+            resp = requests.get(uri, headers={"Authorization": "Bearer " + access_token})
+            resp.raise_for_status()
+            userprofile = resp.json()
+            return userprofile
+
+        except (requests.exceptions.HTTPError, ValueError, KeyError) as e:
+            logger.debug("Unable to fetch user profile from %s (%s)" % (uri, e))
+            return None
+
+
+def get_user_profile(request):
+    return request.bound_data.get("user_profile", {})
+
+
+def includeme(config):
+    # Activate end-points.
+    config.scan("kinto.plugins.openid.views")
+
+    settings = config.get_settings()
+
+    openid_policies = []
+    for policy in aslist(settings["multiauth.policies"]):
+        v = settings.get("multiauth.policy.%s.use" % policy, "")
+        if v.endswith("OpenIDConnectPolicy"):
+            openid_policies.append(policy)
+
+    if len(openid_policies) == 0:
+        # Do not add the capability if no policy is configured.
+        return
+
+    providers_infos = []
+    for name in openid_policies:
+        issuer = settings["multiauth.policy.%s.issuer" % name]
+        openid_config = fetch_openid_config(issuer)
+
+        client_id = settings["multiauth.policy.%s.client_id" % name]
+        header_type = settings.get("multiauth.policy.%s.header_type", "Bearer")
+
+        providers_infos.append(
+            {
+                "name": name,
+                "issuer": openid_config["issuer"],
+                "auth_path": "/openid/%s/login" % name,
+                "client_id": client_id,
+                "header_type": header_type,
+                "userinfo_endpoint": openid_config["userinfo_endpoint"],
+            }
+        )
+
+        OpenAPI.expose_authentication_method(
+            name, {"type": "oauth2", "authorizationUrl": openid_config["authorization_endpoint"]}
+        )
+
+    config.add_api_capability(
+        "openid",
+        description="OpenID connect support.",
+        url="http://kinto.readthedocs.io/en/stable/api/1.x/authentication.html",
+        providers=providers_infos,
+    )
+    config.add_request_method(get_user_profile, name="get_user_profile")

kinto/plugins/openid/utils.py

@@ -0,0 +1,14 @@
+import requests
+
+
+_configs = {}
+
+
+def fetch_openid_config(issuer):
+    global _configs
+
+    if issuer not in _configs:
+        resp = requests.get(issuer.rstrip("/") + "/.well-known/openid-configuration")
+        _configs[issuer] = resp.json()
+
+    return _configs[issuer]

kinto/plugins/openid/views.py

@@ -0,0 +1,193 @@
+import base64
+import urllib.parse
+
+import colander
+import requests
+from pyramid import httpexceptions
+
+from kinto.core import Service
+from kinto.core.cornice.validators import colander_validator
+from kinto.core.errors import ERRORS, raise_invalid
+from kinto.core.resource.schema import ErrorResponseSchema
+from kinto.core.schema import URL
+from kinto.core.utils import random_bytes_hex
+
+from .utils import fetch_openid_config
+
+
+DEFAULT_STATE_TTL_SECONDS = 3600
+DEFAULT_STATE_LENGTH = 32
+
+
+class RedirectHeadersSchema(colander.MappingSchema):
+    """Redirect response headers."""
+
+    location = colander.SchemaNode(colander.String(), name="Location")
+
+
+class RedirectResponseSchema(colander.MappingSchema):
+    """Redirect response schema."""
+
+    headers = RedirectHeadersSchema()
+
+
+response_schemas = {
+    "307": RedirectResponseSchema(description="Successful redirection."),
+    "400": ErrorResponseSchema(description="The request is invalid."),
+}
+
+
+def provider_validator(request, **kwargs):
+    """
+    This validator verifies that the validator in URL (eg. /openid/auth0/login)
+    is a configured OpenIDConnect policy.
+    """
+    provider = request.matchdict["provider"]
+    used = request.registry.settings.get("multiauth.policy.%s.use" % provider, "")
+    if not used.endswith("OpenIDConnectPolicy"):
+        request.errors.add("path", "provider", "Unknow provider %r" % provider)
+
+
+class LoginQuerystringSchema(colander.MappingSchema):
+    """
+    Querystring schema for the login endpoint.
+    """
+
+    callback = URL()
+    scope = colander.SchemaNode(colander.String())
+    prompt = colander.SchemaNode(
+        colander.String(), validator=colander.Regex("none"), missing=colander.drop
+    )
+
+
+class LoginSchema(colander.MappingSchema):
+    querystring = LoginQuerystringSchema()
+
+
+login = Service(
+    name="openid_login", path="/openid/{provider}/login", description="Initiate the OAuth2 login"
+)
+
+
+@login.get(
+    schema=LoginSchema(),
+    validators=(colander_validator, provider_validator),
+    response_schemas=response_schemas,
+)
+def get_login(request):
+    """Initiates to login dance for the specified scopes and callback URI
+    using appropriate redirections."""
+
+    # Settings.
+    provider = request.matchdict["provider"]
+    settings_prefix = "multiauth.policy.%s." % provider
+    issuer = request.registry.settings[settings_prefix + "issuer"]
+    client_id = request.registry.settings[settings_prefix + "client_id"]
+    userid_field = request.registry.settings.get(settings_prefix + "userid_field")
+    state_ttl = int(
+        request.registry.settings.get(
+            settings_prefix + "state_ttl_seconds", DEFAULT_STATE_TTL_SECONDS
+        )
+    )
+    state_length = int(
+        request.registry.settings.get(settings_prefix + "state_length", DEFAULT_STATE_LENGTH)
+    )
+
+    # Read OpenID configuration (cached by issuer)
+    oid_config = fetch_openid_config(issuer)
+    auth_endpoint = oid_config["authorization_endpoint"]
+
+    scope = request.GET["scope"]
+    callback = request.GET["callback"]
+    prompt = request.GET.get("prompt")
+
+    # Check that email scope is requested if userid field is configured as email.
+    if userid_field == "email" and "email" not in scope:
+        error_details = {
+            "name": "scope",
+            "description": "Provider %s requires 'email' scope" % provider,
+        }
+        raise_invalid(request, **error_details)
+
+    # Generate a random string as state.
+    # And save it until code is traded.
+    state = random_bytes_hex(state_length)
+    request.registry.cache.set("openid:state:" + state, callback, ttl=state_ttl)
+
+    # Redirect the client to the Identity Provider that will eventually redirect
+    # to the OpenID token endpoint.
+    token_uri = request.route_url("openid_token", provider=provider)
+    params = dict(
+        client_id=client_id, response_type="code", scope=scope, redirect_uri=token_uri, state=state
+    )
+    if prompt:
+        # The 'prompt' parameter is optional.
+        params["prompt"] = prompt
+    redirect = f"{auth_endpoint}?{urllib.parse.urlencode(params)}"
+    raise httpexceptions.HTTPTemporaryRedirect(redirect)
+
+
+class TokenQuerystringSchema(colander.MappingSchema):
+    """
+    Querystring schema for the token endpoint.
+    """
+
+    code = colander.SchemaNode(colander.String())
+    state = colander.SchemaNode(colander.String())
+
+
+class TokenSchema(colander.MappingSchema):
+    querystring = TokenQuerystringSchema()
+
+
+token = Service(name="openid_token", path="/openid/{provider}/token", description="")
+
+
+@token.get(schema=TokenSchema(), validators=(colander_validator, provider_validator))
+def get_token(request):
+    """Trades the specified code and state against access and ID tokens.
+    The client is redirected to the original ``callback`` URI with the
+    result in querystring."""
+
+    # Settings.
+    provider = request.matchdict["provider"]
+    settings_prefix = "multiauth.policy.%s." % provider
+    issuer = request.registry.settings[settings_prefix + "issuer"]
+    client_id = request.registry.settings[settings_prefix + "client_id"]
+    client_secret = request.registry.settings[settings_prefix + "client_secret"]
+
+    # Read OpenID configuration (cached by issuer)
+    oid_config = fetch_openid_config(issuer)
+    token_endpoint = oid_config["token_endpoint"]
+
+    code = request.GET["code"]
+    state = request.GET["state"]
+
+    # State can be used only once.
+    callback = request.registry.cache.delete("openid:state:" + state)
+    if callback is None:
+        error_details = {
+            "name": "state",
+            "description": "Invalid state",
+            "errno": ERRORS.INVALID_AUTH_TOKEN.value,
+        }
+        raise_invalid(request, **error_details)
+
+    # Trade the code for tokens on the Identity Provider.
+    # Google Identity requires to specify again redirect_uri.
+    redirect_uri = request.route_url("openid_token", provider=provider)
+    data = {
+        "code": code,
+        "client_id": client_id,
+        "client_secret": client_secret,
+        "redirect_uri": redirect_uri,
+        "grant_type": "authorization_code",
+    }
+    resp = requests.post(token_endpoint, data=data)
+
+    # The IdP response is forwarded to the client in the querystring/location hash.
+    # (eg. callback=`http://localhost:3000/#tokens=`)
+    token_info = resp.text.encode("utf-8")
+    encoded_token = base64.b64encode(token_info)
+    redirect = callback + urllib.parse.quote(encoded_token.decode("utf-8"))
+    raise httpexceptions.HTTPTemporaryRedirect(redirect)

kinto/plugins/prometheus.py

@@ -0,0 +1,300 @@
+import logging
+import os
+import shutil
+import warnings
+from time import perf_counter as time_now
+
+from pyramid.exceptions import ConfigurationError
+from pyramid.response import Response
+from pyramid.settings import asbool, aslist
+from zope.interface import implementer
+
+from kinto.core import metrics
+from kinto.core.utils import safe_wraps
+
+
+try:
+    import prometheus_client as prometheus_module
+except ImportError:  # pragma: no cover
+    prometheus_module = None
+
+
+logger = logging.getLogger(__name__)
+
+_METRICS = {}
+_REGISTRY = None
+
+
+PROMETHEUS_MULTIPROC_DIR = os.getenv("PROMETHEUS_MULTIPROC_DIR")
+
+
+def get_registry():
+    global _REGISTRY
+
+    if _REGISTRY is None:
+        if PROMETHEUS_MULTIPROC_DIR:  # pragma: no cover
+            from prometheus_client import multiprocess
+
+            _reset_multiproc_folder_content()
+            # Ref: https://prometheus.github.io/client_python/multiprocess/
+            _REGISTRY = prometheus_module.CollectorRegistry()
+            multiprocess.MultiProcessCollector(_REGISTRY)
+        else:
+            _REGISTRY = prometheus_module.REGISTRY
+            logger.warning("Prometheus metrics will run in single-process mode only.")
+    return _REGISTRY
+
+
+def _fix_metric_name(s):
+    return s.replace("-", "_").replace(".", "_").replace(" ", "_")
+
+
+class Timer:
+    """
+    A decorator to time the execution of a function. It will use the
+    `prometheus_client.Histogram` to record the time taken by the function
+    in seconds. The histogram is passed as an argument to the
+    constructor.
+
+    Main limitation: it does not support `labels` on the decorator.
+    """
+
+    def __init__(self, histogram):
+        self.histogram = histogram
+        self._start_time = None
+
+    def set_labels(self, labels):
+        if not labels:
+            return
+        self.histogram = self.histogram.labels(*(label_value for _, label_value in labels))
+
+    def observe(self, value):
+        return self.histogram.observe(value)
+
+    def __call__(self, f):
+        @safe_wraps(f)
+        def _wrapped(*args, **kwargs):
+            start_time = time_now()
+            try:
+                return f(*args, **kwargs)
+            finally:
+                dt_sec = time_now() - start_time
+                self.histogram.observe(dt_sec)
+
+        return _wrapped
+
+    def __enter__(self):
+        return self.start()
+
+    def __exit__(self, typ, value, tb):
+        self.stop()
+
+    def start(self):
+        self._start_time = time_now()
+        return self
+
+    def stop(self):
+        if self._start_time is None:  # pragma: nocover
+            raise RuntimeError("Timer has not started.")
+        dt_sec = time_now() - self._start_time
+        self.histogram.observe(dt_sec)
+        return self
+
+
+class NoOpHistogram:  # pragma: no cover
+    def observe(self, value):
+        pass
+
+    def labels(self, *args):
+        return self
+
+
+@implementer(metrics.IMetricsService)
+class PrometheusService:
+    def __init__(self, prefix="", disabled_metrics=[], histogram_buckets=None):
+        prefix_clean = ""
+        if prefix:
+            # In GCP Console, the metrics are grouped by the first
+            # word before the first underscore. Here we make sure the specified
+            # prefix is not mixed up with metrics names.
+            # (eg. `remote-settings` -> `remotesettings_`, `kinto_` -> `kinto_`)
+            prefix_clean = _fix_metric_name(prefix).replace("_", "") + "_"
+        self.prefix = prefix_clean.lower()
+        self.disabled_metrics = [m.replace(self.prefix, "") for m in disabled_metrics]
+        self.histogram_buckets = histogram_buckets
+
+    def timer(self, key, value=None, labels=[]):
+        global _METRICS
+
+        key = _fix_metric_name(key)
+        if key in self.disabled_metrics:
+            return Timer(histogram=NoOpHistogram())
+
+        key = self.prefix + key
+        if key not in _METRICS:
+            _METRICS[key] = prometheus_module.Histogram(
+                key,
+                f"Histogram of {key}",
+                labelnames=[label_name for label_name, _ in labels],
+                buckets=self.histogram_buckets,
+            )
+
+        if not isinstance(_METRICS[key], prometheus_module.Histogram):
+            raise RuntimeError(
+                f"Metric {key} already exists with different type ({_METRICS[key]})"
+            )
+
+        timer = Timer(histogram=_METRICS[key])
+        timer.set_labels(labels)
+
+        if value is not None:
+            # We are timing something.
+            return timer.observe(value)
+
+        # We are not timing anything, just returning the timer object
+        # (eg. to be used as decorator or context manager).
+        # Note that in this case, the labels values will be the same for all calls.
+        return timer
+
+    def observe(self, key, value, labels=[]):
+        global _METRICS
+
+        key = _fix_metric_name(key)
+        if key in self.disabled_metrics:
+            return
+
+        key = self.prefix + key
+        if key not in _METRICS:
+            _METRICS[key] = prometheus_module.Summary(
+                key,
+                f"Summary of {key}",
+                labelnames=[label_name for label_name, _ in labels],
+            )
+
+        if not isinstance(_METRICS[key], prometheus_module.Summary):
+            raise RuntimeError(
+                f"Metric {key} already exists with different type ({_METRICS[key]})"
+            )
+
+        m = _METRICS[key]
+        if labels:
+            m = m.labels(*(label_value for _, label_value in labels))
+
+        m.observe(value)
+
+    def count(self, key, count=1, unique=None):
+        global _METRICS
+
+        key = _fix_metric_name(key)
+        if key in self.disabled_metrics:
+            return
+
+        labels = []
+        if unique:
+            if isinstance(unique, str):
+                warnings.warn(
+                    "`unique` parameter should be of type ``list[tuple[str, str]]``",
+                    DeprecationWarning,
+                )
+                # Turn `unique` into a group and a value:
+                # "bob" -> "group.bob"
+                # "method.basicauth.mat" -> [("method_basicauth", "mat")]`
+                if "." not in unique:
+                    unique = f"group.{unique}"
+                label_name, label_value = unique.rsplit(".", 1)
+                unique = [(label_name, label_value)]
+
+            labels = [
+                (_fix_metric_name(label_name), label_value) for label_name, label_value in unique
+            ]
+
+        key = self.prefix + key
+        if key not in _METRICS:
+            _METRICS[key] = prometheus_module.Counter(
+                key,
+                f"Counter of {key}",
+                labelnames=[label_name for label_name, _ in labels],
+            )
+
+        if not isinstance(_METRICS[key], prometheus_module.Counter):
+            raise RuntimeError(
+                f"Metric {key} already exists with different type ({_METRICS[key]})"
+            )
+
+        m = _METRICS[key]
+        if labels:
+            m = m.labels(*(label_value for _, label_value in labels))
+
+        m.inc(count)
+
+
+def metrics_view(request):
+    registry = get_registry()
+    data = prometheus_module.generate_latest(registry)
+    resp = Response(body=data)
+    resp.headers["Content-Type"] = prometheus_module.CONTENT_TYPE_LATEST
+    resp.headers["Content-Length"] = str(len(data))
+    return resp
+
+
+def _reset_multiproc_folder_content():  # pragma: no cover
+    shutil.rmtree(PROMETHEUS_MULTIPROC_DIR, ignore_errors=True)
+    os.makedirs(PROMETHEUS_MULTIPROC_DIR, exist_ok=True)
+
+
+def reset_registry():
+    # This is mainly useful in tests, where the plugin is included
+    # several times with different settings.
+    registry = get_registry()
+
+    for collector in _METRICS.values():
+        try:
+            registry.unregister(collector)
+        except KeyError:  # pragma: no cover
+            pass
+    _METRICS.clear()
+
+
+def includeme(config):
+    if prometheus_module is None:
+        error_msg = (
+            "Please install Kinto with monitoring dependencies (e.g. prometheus-client package)"
+        )
+        raise ConfigurationError(error_msg)
+
+    settings = config.get_settings()
+
+    if not asbool(settings.get("prometheus_created_metrics_enabled", True)):
+        prometheus_module.disable_created_metrics()
+
+    prefix = settings.get("prometheus_prefix", settings["project_name"])
+    disabled_metrics = aslist(settings.get("prometheus_disabled_metrics", ""))
+
+    # Default buckets for histogram metrics are (.005, .01, .025, .05, .075, .1, .25, .5, .75, 1.0, 2.5, 5.0, 7.5, 10.0, INF)
+    # we reduce it from 15 to 8 values by default here, and let the user override it if needed.
+    histogram_buckets_values = aslist(
+        settings.get(
+            "prometheus_histogram_buckets", "0.01 0.05 0.1 0.5 1.0 3.0 6.0 Inf"
+        )  # Note: Inf is added by default.
+    )
+    histogram_buckets = [float(x) for x in histogram_buckets_values]
+    # Note: we don't need to check for INF or list size, it's done in the prometheus_client library.
+
+    get_registry()  # Initialize the registry.
+
+    metrics_impl = PrometheusService(
+        prefix=prefix, disabled_metrics=disabled_metrics, histogram_buckets=histogram_buckets
+    )
+
+    config.add_api_capability(
+        "prometheus",
+        description="Prometheus metrics.",
+        url="https://github.com/Kinto/kinto/",
+        prefix=metrics_impl.prefix,
+        disabled_metrics=disabled_metrics,
+    )
+
+    config.add_route("prometheus_metrics", "/__metrics__")
+    config.add_view(metrics_view, route_name="prometheus_metrics")
+
+    config.registry.registerUtility(metrics_impl, metrics.IMetricsService)