kinto 23.2.1__py3-none-any.whl

kinto/core/authorization.py

@@ -0,0 +1,311 @@
+import functools
+import logging
+
+from pyramid.authorization import Authenticated
+from pyramid.interfaces import IAuthorizationPolicy
+from pyramid.settings import aslist
+from zope.interface import implementer
+
+from kinto.core import utils
+from kinto.core.storage import exceptions as storage_exceptions
+
+
+logger = logging.getLogger(__name__)
+
+# When permission is set to "private", only the current user is allowed.
+PRIVATE = "private"
+
+# A permission is called "dynamic" when it's computed at request time.
+DYNAMIC = "dynamic"
+
+
+def groupfinder(userid, request):
+    """Fetch principals from permission backend for the specified `userid`.
+
+    This is plugged by default using the ``multiauth.groupfinder`` setting.
+    """
+    backend = getattr(request.registry, "permission", None)
+    # Permission backend not configured. Ignore.
+    if not backend:
+        return []
+
+    # Safety check when Kinto-Core is used without pyramid_multiauth.
+    if request.prefixed_userid:
+        userid = request.prefixed_userid
+
+    # Query the permission backend only once per request (e.g. batch).
+    reify_key = userid + "_principals"
+    if reify_key not in request.bound_data:
+        principals = backend.get_user_principals(userid)
+        request.bound_data[reify_key] = principals
+
+    return request.bound_data[reify_key]
+
+
+@implementer(IAuthorizationPolicy)
+class AuthorizationPolicy:
+    """Default authorization class, that leverages the permission backend
+    for shareable resources.
+    """
+
+    get_bound_permissions = None
+    """Callable that takes an object id and a permission and returns
+    a list of tuples (<object id>, <permission>). Useful when objects
+    permission depend on others."""
+
+    def permits(self, context, principals, permission):
+        if permission == PRIVATE:
+            # When using the private permission, we bypass the permissions
+            # backend, and simply authorize if authenticated.
+            return Authenticated in principals
+
+        principals = context.get_prefixed_principals()
+
+        create_permission = f"{context.resource_name}:create"
+
+        permission = context.required_permission
+        if permission == "create":
+            permission = create_permission
+
+        object_id = context.permission_object_id
+        bound_perms = self._get_bound_permissions(object_id, permission)
+
+        allowed = context.check_permission(principals, bound_perms)
+
+        # Here we consider that parent URI is one path level above.
+        parent_uri = "/".join(object_id.split("/")[:-1]) if object_id else None
+
+        # If not allowed to delete/patch, and target object is missing, and
+        # allowed to read the parent, then view is permitted (will raise 404
+        # later anyway). See Kinto/kinto#918
+        is_object_unknown = not context.on_plural_endpoint and context.current_object is None
+        if context.required_permission == "write" and is_object_unknown:
+            bound_perms = self._get_bound_permissions(parent_uri, "read")
+            allowed = context.check_permission(principals, bound_perms)
+
+        # If not allowed on this plural endpoint, but some objects are shared with
+        # the current user, then authorize.
+        # The :class:`kinto.core.resource.Resource` class will take care of the filtering.
+        is_list_operation = (
+            context.on_plural_endpoint
+            and not permission.endswith("create")
+            and context.current_object is None
+        )
+        if not allowed and is_list_operation:
+            allowed = bool(
+                context.fetch_shared_objects(permission, principals, self.get_bound_permissions)
+            )
+            if not allowed:
+                # If allowed to create this kind of object on parent,
+                # then allow to obtain the list.
+                if len(bound_perms) > 0:
+                    bound_perms = [(parent_uri, create_permission)]
+                else:
+                    bound_perms = [("", "create")]  # Root object.
+                allowed = context.check_permission(principals, bound_perms)
+
+        if not allowed:
+            logger.info(
+                "Permission %r on %r not granted to %r.",
+                permission,
+                object_id,
+                principals[0],
+                extra=dict(userid=principals[0], uri=object_id, perm=permission),
+            )
+
+        return allowed
+
+    def _get_bound_permissions(self, object_id, permission):
+        if self.get_bound_permissions is None:
+            # Permission to 'write' gives permission to 'read'.
+            bound = [(object_id, permission)]
+            if permission == "read":
+                bound += [(object_id, "write")]
+            return bound
+        return self.get_bound_permissions(object_id, permission)
+
+    def principals_allowed_by_permission(self, context, permission):
+        raise NotImplementedError()  # PRAGMA NOCOVER
+
+
+class RouteFactory:
+    resource_name = None
+    on_plural_endpoint = False
+    required_permission = None
+    permission_object_id = None
+    current_object = None
+    shared_ids = None
+
+    method_permissions = {
+        "head": "read",
+        "get": "read",
+        "post": "create",
+        "delete": "write",
+        "patch": "write",
+    }
+
+    def __init__(self, request):
+        # Store some shortcuts.
+        permission = request.registry.permission
+        self._check_permission = permission.check_permission
+        self._get_accessible_objects = permission.get_accessible_objects
+
+        self.get_prefixed_principals = functools.partial(utils.prefixed_principals, request)
+
+        # Store current resource and required permission.
+        service = utils.current_service(request)
+        is_on_resource = (
+            service is not None and hasattr(service, "viewset") and hasattr(service, "resource")
+        )
+        self._resource = None
+        if is_on_resource:
+            self.resource_name = request.current_resource_name
+            self.on_plural_endpoint = getattr(service, "type", None) == "plural"
+
+            # Check if this request targets an individual object.
+            # Its existence will affect permissions checking (cf `_find_required_permission()`).
+            # There are cases where the permission is not directly related to the HTTP method,
+            # For example:
+            # - with POST on plural endpoint, with an id supplied
+            # - with PUT on an object, which can either be creation or update
+            is_write_on_object = not self.on_plural_endpoint and request.method.lower() in (
+                "put",
+                "delete",
+                "patch",
+            )
+            is_post_on_plural = self.on_plural_endpoint and request.method.lower() == "post"
+            if is_write_on_object or is_post_on_plural:
+                # We instantiate the resource to determine the object targeted by the request.
+                self._resource = resource = service.resource(request=request, context=self)
+                if resource.object_id is not None:  # Skip POST on plural without id.
+                    try:
+                        # Save a reference, to avoid refetching from storage in resource.
+                        self.current_object = resource.model.get_object(resource.object_id)
+                    except storage_exceptions.ObjectNotFoundError:
+                        pass
+
+            self.permission_object_id, self.required_permission = self._find_required_permission(
+                request, service
+            )
+
+            # To obtain shared objects on a plural endpoint, use a match:
+            self._object_id_match = self.get_permission_object_id(request, "*")
+
+        self._settings = request.registry.settings
+
+    def check_permission(self, principals, bound_perms):
+        """Read allowed principals from settings, if not any, query the permission
+        backend to check if view is allowed.
+        """
+        if not bound_perms:
+            bound_perms = [(self.resource_name, self.required_permission)]
+        for _, permission in bound_perms:
+            # With Kinto inheritance tree, we can have: `permission = "record:create"`
+            if self.resource_name and permission.startswith(self.resource_name):
+                setting = f"{permission.replace(':', '_')}_principals"
+            else:
+                setting = f"{self.resource_name}_{permission}_principals"
+            allowed_principals = aslist(self._settings.get(setting, ""))
+            if allowed_principals:
+                if bool(set(allowed_principals) & set(principals)):
+                    return True
+        return self._check_permission(principals, bound_perms)
+
+    def fetch_shared_objects(self, perm, principals, get_bound_permissions):
+        """Fetch objects that are readable or writable for the current
+        principals.
+
+        See :meth:`kinto.core.authorization.AuthorizationPolicy.permits`
+
+        If no object is shared, it returns None.
+
+        .. warning::
+            This sets the ``shared_ids`` attribute to the context with the
+            return value. The attribute is then read by
+            :class:`kinto.core.resource.Resource`
+        """
+        if get_bound_permissions:
+            bound_perms = get_bound_permissions(self._object_id_match, perm)
+        else:
+            bound_perms = [(self._object_id_match, perm)]
+        by_obj_id = self._get_accessible_objects(principals, bound_perms, with_children=False)
+        ids = by_obj_id.keys()
+        # Store for later use in ``Resource``.
+        self.shared_ids = [self._extract_object_id(id_) for id_ in ids]
+        return self.shared_ids
+
+    def get_permission_object_id(self, request, object_id=None):
+        """Returns the permission object id for the current request.
+        In the nominal case, it is just the current URI without version prefix.
+        For plural endpoint, it is the related object URI using the specified
+        `object_id`.
+
+        See :meth:`kinto.core.resource.model.SharableModel` and
+        :meth:`kinto.core.authorization.RouteFactory.__init__`
+        """
+        object_uri = utils.strip_uri_prefix(request.path)
+
+        if self.on_plural_endpoint and object_id is not None:
+            # With the current request on a plural endpoint, the object URI must
+            # be found out by inspecting the "plural" service and its sibling
+            # "object" service. (see `register_resource()`)
+            matchdict = {**request.matchdict, "id": object_id}
+            try:
+                object_uri = utils.instance_uri(request, self.resource_name, **matchdict)
+                object_uri = object_uri.replace("%2A", "*")
+            except KeyError:
+                # Maybe the resource has no single object endpoint.
+                # We consider that object URIs in permissions backend will
+                # be stored naively:
+                object_uri = f"{object_uri}/{object_id}"
+
+        return object_uri
+
+    def _extract_object_id(self, object_uri):
+        # XXX: Rewrite using kinto.core.utils.view_lookup() and matchdict['id']
+        return object_uri.split("/")[-1]
+
+    def _find_required_permission(self, request, service):
+        """Find out what is the permission object id and the required
+        permission.
+
+        .. note::
+            This method saves an attribute ``self.current_object`` used
+            in :class:`kinto.core.resource.Resource`.
+        """
+        # By default, it's a URI a and permission associated to the method.
+        permission_object_id = self.get_permission_object_id(request)
+        method = request.method.lower()
+        required_permission = self.method_permissions.get(method)
+
+        # For create permission, the object id is the plural endpoint.
+        plural_path = str(service.plural_path)
+        plural_path = plural_path.format_map(request.matchdict)
+
+        # In the case of a "PUT", check if the targetted object already
+        # exists, return "write" if it does, "create" otherwise.
+        if request.method.lower() == "put":
+            if self.current_object is None:
+                # The object does not exist, the permission to create on
+                # the related plural endpoint is required.
+                permission_object_id = plural_path
+                required_permission = "create"
+            else:
+                # For safe creations, the user needs a create permission.
+                # See Kinto/kinto#792
+                if request.headers.get("If-None-Match") == "*":
+                    permission_object_id = plural_path
+                    required_permission = "create"
+                else:
+                    required_permission = "write"
+
+        # In the case of a "POST" on a plural endpoint, if an "id" was
+        # specified, then the object is returned. The required permission
+        # is thus "read" on this object.
+        if request.method.lower() == "post" and self.current_object is not None:
+            permission_object_id = self.get_permission_object_id(
+                request, object_id=self._resource.object_id
+            )
+            required_permission = "read"
+
+        return (permission_object_id, required_permission)

kinto/core/cache/__init__.py

@@ -0,0 +1,131 @@
+import logging
+import random
+
+
+logger = logging.getLogger(__name__)
+
+
+_HEARTBEAT_DELETE_RATE = 0.5
+_HEARTBEAT_KEY = "__heartbeat__"
+_HEARTBEAT_TTL_SECONDS = 3600
+
+
+_CACHE_HIT_METRIC_KEY = "cache_hits"
+_CACHE_MISS_METRIC_KEY = "cache_misses"
+
+
+class CacheBase:
+    def __init__(self, *args, **kwargs):
+        self.prefix = kwargs["cache_prefix"]
+        self.max_size_bytes = kwargs.get("cache_max_size_bytes")
+        self.set_metrics_backend(kwargs.get("metrics_backend"))
+
+    def initialize_schema(self, dry_run=False):
+        """Create every necessary objects (like tables or indices) in the
+        backend.
+
+        This is executed when the ``kinto migrate`` command is run.
+
+        :param bool dry_run: simulate instead of executing the operations.
+        """
+        raise NotImplementedError
+
+    def flush(self):
+        """Delete every values."""
+        raise NotImplementedError
+
+    def ttl(self, key):
+        """Obtain the expiration value of the specified `key`.
+
+        :param str key: key
+        :returns: number of seconds or negative if no TTL.
+        :rtype: float
+        """
+        raise NotImplementedError
+
+    def expire(self, key, ttl):
+        """Set the expiration value `ttl` for the specified `key`.
+
+        :param str key: key
+        :param float ttl: number of seconds
+        """
+        raise NotImplementedError
+
+    def set(self, key, value, ttl):
+        """Store a value with the specified `key`.
+
+        :param str key: key
+        :param str value: value to store
+        :param float ttl: expire after number of seconds
+        """
+        raise NotImplementedError
+
+    def get(self, key):
+        """Obtain the value of the specified `key`.
+
+        :param str key: key
+        :returns: the stored value or None if missing.
+        :rtype: str
+        """
+        raise NotImplementedError
+
+    def delete(self, key):
+        """Delete the value of the specified `key`.
+
+        :param str key: key
+        """
+        raise NotImplementedError
+
+    def set_metrics_backend(self, metrics_backend):
+        """Set a metrics backend via the `CacheMetricsBackend` adapter.
+
+        :param metrics_backend: A metrics backend implementing the IMetricsService interface.
+        """
+        self.metrics_backend = CacheMetricsBackend(metrics_backend)
+
+
+class CacheMetricsBackend:
+    """
+    A simple adapter for tracking cache-related metrics.
+    """
+
+    def __init__(self, metrics_backend, *args, **kwargs):
+        """Initialize with a given metrics backend.
+
+        :param metrics_backend: A metrics backend implementing the IMetricsService interface.
+        """
+        self._backend = metrics_backend
+
+    def count_hit(self):
+        """Increment the cache hit counter."""
+        if self._backend:
+            self._backend.count(key=_CACHE_HIT_METRIC_KEY)
+
+    def count_miss(self):
+        """Increment the cache miss counter."""
+        if self._backend:
+            self._backend.count(key=_CACHE_MISS_METRIC_KEY)
+
+
+def heartbeat(backend):
+    def ping(request):
+        """Test that cache backend is operational.
+
+        :param request: current request object
+        :type request: :class:`~pyramid:pyramid.request.Request`
+        :returns: ``True`` is everything is ok, ``False`` otherwise.
+        :rtype: bool
+        """
+        # No specific case for readonly mode because the cache should
+        # continue to work in that mode.
+        try:
+            if random.SystemRandom().random() < _HEARTBEAT_DELETE_RATE:
+                backend.delete(_HEARTBEAT_KEY)
+                return backend.get(_HEARTBEAT_KEY) is None
+            backend.set(_HEARTBEAT_KEY, "alive", _HEARTBEAT_TTL_SECONDS)
+            return backend.get(_HEARTBEAT_KEY) == "alive"
+        except Exception:
+            logger.exception("Heartbeat Failure")
+            return False
+
+    return ping

kinto/core/cache/memcached.py

@@ -0,0 +1,112 @@
+import logging
+from functools import wraps
+from math import ceil, floor
+from time import time
+
+from pyramid.settings import aslist
+
+from kinto.core.cache import CacheBase
+from kinto.core.storage import exceptions
+from kinto.core.utils import json, memcache
+
+
+logger = logging.getLogger(__name__)
+
+
+def wrap_memcached_error(func):
+    @wraps(func)
+    def wrapped(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except TypeError:
+            raise
+        except (
+            memcache.Client.MemcachedKeyError,
+            memcache.Client.MemcachedStringEncodingError,
+        ) as e:
+            logger.exception(e)
+            raise exceptions.BackendError(original=e)
+
+    return wrapped
+
+
+def create_from_config(config, prefix=""):
+    """Memcached client instantiation from settings."""
+    settings = config.get_settings()
+    hosts = aslist(settings[prefix + "hosts"])
+    return memcache.Client(hosts)
+
+
+class Cache(CacheBase):
+    """Cache backend implementation using Memcached.
+
+    Enable in configuration::
+
+        kinto.cache_backend = kinto.core.cache.memcached
+
+    *(Optional)* Instance location URI can be customized::
+
+        kinto.cache_hosts = 127.0.0.1:11211 127.0.0.1:11212
+
+    :noindex:
+
+    """
+
+    def __init__(self, client, *args, **kwargs):
+        super(Cache, self).__init__(*args, **kwargs)
+        self._client = client
+
+    def initialize_schema(self, dry_run=False):
+        # Nothing to do.
+        pass
+
+    @wrap_memcached_error
+    def flush(self):
+        self._client.flush_all()
+
+    @wrap_memcached_error
+    def _get(self, key):
+        value = self._client.get(self.prefix + key)
+        if not value:
+            self.metrics_backend.count_miss()
+            return None, 0
+        self.metrics_backend.count_hit()
+        data = json.loads(value)
+        return data["value"], data["ttl"]
+
+    def ttl(self, key):
+        _, ttl = self._get(key)
+        val = ttl - time()
+        return floor(val)
+
+    def get(self, key):
+        value, _ = self._get(key)
+        return value
+
+    @wrap_memcached_error
+    def expire(self, key, ttl):
+        if ttl == 0:
+            self.delete(key)
+        else:
+            # We can't use touch here because we need to update the TTL value in the record.
+            value = self.get(key)
+            self.set(key, value, ttl)
+
+    @wrap_memcached_error
+    def set(self, key, value, ttl):
+        if isinstance(value, bytes):
+            raise TypeError("a string-like object is required, not 'bytes'")
+        value = json.dumps({"value": value, "ttl": ceil(time() + ttl)})
+        self._client.set(self.prefix + key, value, int(ttl))
+
+    @wrap_memcached_error
+    def delete(self, key):
+        value = self.get(key)
+        self._client.delete(self.prefix + key)
+        return value
+
+
+def load_from_config(config):
+    settings = config.get_settings()
+    client = create_from_config(config, prefix="cache_")
+    return Cache(client, cache_prefix=settings["cache_prefix"])

kinto/core/cache/memory.py

@@ -0,0 +1,104 @@
+import logging
+
+from kinto.core.cache import CacheBase
+from kinto.core.decorators import synchronized
+from kinto.core.utils import msec_time
+
+
+logger = logging.getLogger(__name__)
+
+
+class Cache(CacheBase):
+    """Cache backend implementation in local process memory.
+
+    Enable in configuration::
+
+        kinto.cache_backend = kinto.core.cache.memory
+
+    :noindex:
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.flush()
+
+    def initialize_schema(self, dry_run=False):
+        # Nothing to do.
+        pass
+
+    def flush(self):
+        self._created_at = {}
+        self._ttl = {}
+        self._store = {}
+        self._quota = 0
+
+    def _clean_expired(self):
+        current = msec_time()
+        expired = [k for k, v in self._ttl.items() if current >= v]
+        for expired_item_key in expired:
+            self.delete(expired_item_key[len(self.prefix) :])
+
+    def _clean_oversized(self):
+        if self._quota < self.max_size_bytes:
+            return
+
+        for key, value in sorted(self._created_at.items(), key=lambda k: k[1]):
+            if self._quota < (self.max_size_bytes * 0.8):
+                break
+            self.delete(key[len(self.prefix) :])
+
+    @synchronized
+    def ttl(self, key):
+        ttl = self._ttl.get(self.prefix + key)
+        if ttl is not None:
+            return (ttl - msec_time()) / 1000.0
+        return -1
+
+    @synchronized
+    def expire(self, key, ttl):
+        self._ttl[self.prefix + key] = msec_time() + int(ttl * 1000.0)
+
+    @synchronized
+    def set(self, key, value, ttl):
+        if isinstance(value, bytes):
+            raise TypeError("a string-like object is required, not 'bytes'")
+        self._clean_expired()
+        self._clean_oversized()
+        self.expire(key, ttl)
+        item_key = self.prefix + key
+        self._store[item_key] = value
+        self._created_at[item_key] = msec_time()
+        self._quota += size_of(item_key, value)
+
+    @synchronized
+    def get(self, key):
+        self._clean_expired()
+        value = self._store.get(self.prefix + key)
+        if value is None:
+            self.metrics_backend.count_miss()
+            return None
+        self.metrics_backend.count_hit()
+        return value
+
+    @synchronized
+    def delete(self, key):
+        key = self.prefix + key
+        self._ttl.pop(key, None)
+        self._created_at.pop(key, None)
+        value = self._store.pop(key, None)
+        self._quota -= size_of(key, value)
+        return value
+
+
+def load_from_config(config):
+    settings = config.get_settings()
+    return Cache(
+        cache_prefix=settings["cache_prefix"],
+        cache_max_size_bytes=settings["cache_max_size_bytes"],
+    )
+
+
+def size_of(key, value):
+    # Key used for ttl, created_at and store.
+    # Int size is 24 bytes one for ttl and one for created_at values
+    return len(key) * 3 + len(str(value)) + 24 * 2