kinto 23.2.1__py3-none-any.whl

kinto/plugins/statsd.py

@@ -0,0 +1,85 @@
+import warnings
+from datetime import timedelta
+from urllib.parse import urlparse
+
+from pyramid.exceptions import ConfigurationError
+from zope.interface import implementer
+
+from kinto.core import metrics
+
+
+try:
+    import statsd as statsd_module
+except ImportError:  # pragma: no cover
+    statsd_module = None
+
+
+def sanitize(value):
+    """
+    Telegraf does not support ':' in values.
+    See https://github.com/influxdata/telegraf/issues/4495
+    """
+    return value.replace(":", "") if isinstance(value, str) else value
+
+
+@implementer(metrics.IMetricsService)
+class StatsDService:
+    def __init__(self, host, port, prefix):
+        self._client = statsd_module.StatsClient(host, port, prefix=prefix)
+
+    def timer(self, key, value=None, labels=[]):
+        if labels:
+            # [("method", "get")] -> "method.get"
+            key = f"{key}." + ".".join(f"{label[0]}.{sanitize(label[1])}" for label in labels)
+        if value:
+            value = timedelta(seconds=value)
+            return self._client.timing(key, value)
+        return self._client.timer(key)
+
+    def observe(self, key, value, labels=[]):
+        return self._client.gauge(key, sanitize(value))
+
+    def count(self, key, count=1, unique=None):
+        if unique is None:
+            return self._client.incr(key, count=count)
+        if isinstance(unique, list):
+            # [("method", "get")] -> "method.get"
+            # [("endpoint", "/"), ("method", "get")] -> "endpoint./.method.get"
+            unique = ".".join(f"{label[0]}.{sanitize(label[1])}" for label in unique)
+        else:
+            warnings.warn(
+                "`unique` parameter should be of type ``list[tuple[str, str]]``",
+                DeprecationWarning,
+            )
+        return self._client.set(key, unique)
+
+
+def load_from_config(config):
+    # If this is called, it means that a ``statsd_url`` was specified in settings.
+    # (see ``kinto.core.initialization``)
+    # Raise a proper error if the ``statsd`` module is not installed.
+    if statsd_module is None:
+        error_msg = "Please install Kinto with monitoring dependencies (e.g. statsd package)"
+        raise ConfigurationError(error_msg)
+
+    settings = config.get_settings()
+    uri = settings["statsd_url"]
+    uri = urlparse(uri)
+
+    if settings["project_name"] != "":
+        prefix = settings["project_name"]
+    else:
+        prefix = settings["statsd_prefix"]
+
+    return StatsDService(uri.hostname, uri.port, prefix)
+
+
+def includeme(config):
+    settings = config.get_settings()
+
+    # TODO: this backend abstraction may not be required anymore.
+    statsd_mod = settings["statsd_backend"]
+    statsd_mod = config.maybe_dotted(statsd_mod)
+    client = statsd_mod.load_from_config(config)
+
+    config.registry.registerUtility(client, metrics.IMetricsService)

kinto/schema_validation.py

@@ -0,0 +1,135 @@
+import colander
+from jsonschema import Draft7Validator as DraftValidator
+from jsonschema import RefResolutionError, SchemaError, ValidationError
+from jsonschema.validators import validator_for
+from pyramid.settings import asbool
+
+from kinto.core import utils
+from kinto.core.errors import raise_invalid
+from kinto.views import object_exists_or_404
+
+
+class JSONSchemaMapping(colander.SchemaNode):
+    def schema_type(self, **kw):
+        return colander.Mapping(unknown="preserve")
+
+    def deserialize(self, cstruct=colander.null):
+        # Start by deserializing a simple mapping.
+        validated = super().deserialize(cstruct)
+        try:
+            check_schema(validated)
+        except ValidationError as e:
+            self.raise_invalid(e.message)
+        return validated
+
+
+def check_schema(data):
+    try:
+        DraftValidator.check_schema(data)
+    except SchemaError as e:
+        message = e.path.pop() + e.message
+        raise ValidationError(message)
+
+
+# Module level global that stores a version of every possible schema (as a <class 'dict'>)
+# turned into a jsonschema instance (as <class 'jsonschema.validators.Validator'>).
+_schema_cache = {}
+
+
+def validate(data, schema):
+    """Raise a ValidationError or a RefResolutionError if the data doesn't validate
+    with the given schema.
+
+    Note that this function is just a "wrapper" on `jsonschema.validate()` but with
+    some memoization based on the schema for better repeat performance.
+    """
+    # Because the schema is a dict, it can't be used as a hash key so it needs to be
+    # "transformed" to something that is hashable. The quickest solution is to convert
+    # it to a string.
+    # Note that the order of the dict will determine the string it becomes. The solution
+    # to that would a canonical serializer like `json.dumps(..., sort_keys=True)` but it's
+    # overkill since the assumption is that the schema is very unlikely to be exactly
+    # the same but different order.
+    cache_key = str(schema)
+    if cache_key not in _schema_cache:
+        # This is essentially what the `jsonschema.validate()` shortcut function does.
+        cls = validator_for(schema)
+        cls.check_schema(schema)
+        _schema_cache[cache_key] = cls(schema)
+    return _schema_cache[cache_key].validate(data)
+
+
+def validate_schema(data, schema, id_field, ignore_fields=None):
+    if ignore_fields is None:
+        ignore_fields = []
+    # Only ignore the `id` field if the schema does not explicitly mention it.
+    if id_field not in schema.get("properties", {}):
+        ignore_fields += (id_field,)
+
+    required_fields = [f for f in schema.get("required", []) if f not in ignore_fields]
+    # jsonschema doesn't accept 'required': [] yet.
+    # See https://github.com/Julian/jsonschema/issues/337.
+    # In the meantime, strip out 'required' if no other fields are required.
+    if required_fields:
+        schema = {**schema, "required": required_fields}
+    else:
+        schema = {f: v for f, v in schema.items() if f != "required"}
+
+    data = {f: v for f, v in data.items() if f not in ignore_fields}
+
+    try:
+        validate(data, schema)
+    except ValidationError as e:
+        if e.path:
+            field = e.path[-1]
+        elif e.validator_value:
+            field = e.validator_value[-1]
+        else:
+            field = e.schema_path[-1]
+        e.field = field
+        raise e
+    # Raise an error here if a reference in the schema doesn't resolve.
+    # jsonschema doesn't provide schema validation checking upon creation yet,
+    # it must be validated against data.
+    # See https://github.com/Julian/jsonschema/issues/399
+    # For future support https://github.com/Julian/jsonschema/issues/346.
+    except RefResolutionError as e:
+        raise e
+
+
+def validate_from_bucket_schema_or_400(data, resource_name, request, id_field, ignore_fields):
+    """Lookup in the parent objects if a schema was defined for this resource.
+
+    If the schema validation feature is enabled, if a schema is/are defined, and if the
+    data does not validate it/them, then it raises a 400 exception.
+    """
+    settings = request.registry.settings
+    schema_validation = "experimental_collection_schema_validation"
+    # If disabled from settings, do nothing.
+    if not asbool(settings.get(schema_validation)):
+        return
+
+    bucket_id = request.matchdict["bucket_id"]
+    bucket_uri = utils.instance_uri(request, "bucket", id=bucket_id)
+    buckets = request.bound_data.setdefault("buckets", {})
+    if bucket_uri not in buckets:
+        # Unknown yet, fetch from storage.
+        bucket = object_exists_or_404(
+            request, resource_name="bucket", parent_id="", object_id=bucket_id
+        )
+        buckets[bucket_uri] = bucket
+
+    # Let's see if the bucket defines a schema for this resource.
+    metadata_field = f"{resource_name}:schema"
+    bucket = buckets[bucket_uri]
+    if metadata_field not in bucket:
+        return
+
+    # Validate or fail with 400.
+    schema = bucket[metadata_field]
+    try:
+        validate_schema(data, schema, ignore_fields=ignore_fields, id_field=id_field)
+    except ValidationError as e:
+        raise_invalid(request, name=e.field, description=e.message)
+    except RefResolutionError as e:
+        raise_invalid(request, name="schema", description=str(e))

kinto/views/__init__.py

@@ -0,0 +1,34 @@
+import random
+import string
+
+from pyramid.httpexceptions import HTTPNotFound
+
+from kinto.core.errors import ERRORS, http_error
+from kinto.core.storage import exceptions, generators
+
+
+class NameGenerator(generators.Generator):
+    def __call__(self):
+        alpha_num = string.ascii_letters + string.digits
+        alphabet = alpha_num + "-_"
+        letters = [random.SystemRandom().choice(alpha_num)]
+        letters += [random.SystemRandom().choice(alphabet) for x in range(7)]
+
+        return "".join(letters)
+
+
+class RelaxedUUID(generators.UUID4):
+    """A generator that generates UUIDs but accepts any string."""
+
+    regexp = generators.Generator.regexp
+
+
+def object_exists_or_404(request, resource_name, object_id, parent_id=""):
+    storage = request.registry.storage
+    try:
+        return storage.get(resource_name=resource_name, parent_id=parent_id, object_id=object_id)
+    except exceptions.ObjectNotFoundError:
+        # XXX: We gave up putting details about parent id here (See #53).
+        details = {"id": object_id, "resource_name": resource_name}
+        response = http_error(HTTPNotFound(), errno=ERRORS.MISSING_RESOURCE, details=details)
+        raise response

kinto/views/admin.py

@@ -0,0 +1,195 @@
+"""
+Special views for administration.
+"""
+
+import collections
+import itertools
+
+import colander
+
+from kinto.authorization import RouteFactory
+from kinto.core import resource
+from kinto.core import utils as core_utils
+from kinto.core.events import ACTIONS, notify_resource_event
+from kinto.core.resource import viewset
+from kinto.core.storage import Filter
+
+
+def slice_into_batches(iterable, batch_size):
+    # Taken from https://code.activestate.com/recipes/303279-getting-items-in-batches/
+    i = iter(iterable)
+    while True:
+        batchiter = itertools.islice(i, batch_size)
+        try:
+            yield itertools.chain([next(batchiter)], batchiter)
+        except StopIteration:
+            return
+
+
+class Deleted(resource.ResourceSchema):
+    principal = colander.SchemaNode(colander.String())
+
+    class Options:
+        preserve_unknown = False
+
+
+class UserDataFactory(RouteFactory):
+    method_permissions = {"delete": "delete"}
+
+
+class UserDataViewSet(viewset.ViewSet):
+    factory = UserDataFactory
+
+
+def get_parent_uri(object_uri):
+    """Get the parent URI for an object_uri.
+
+    In order to be generic over any kind of resource hierarchy, we do
+    this by string manipulation on the URI instead of trying to parse
+    the URI, identify the parent resource, and generate a new URI.
+
+    """
+    path = object_uri.rsplit("/", 2)
+    # len(path) == 1: no '/', probably a broken URL?
+    # len(path) == 2: one '/', doesn't conform to our URL scheme
+    if len(path) < 3:
+        return ""
+
+    return path[0]
+
+
+def condense_under_parents(request, object_uris):
+    """Simplify object_uris by removing "duplicates".
+
+    Deleting a resource usually cascades to all its descendant
+    resources. Use this out-of-band knowledge to remove any objects
+    which will already be deleted by virtue of deleting their
+    ancestors.
+
+    """
+    # Sort object_uris so we see ancestors before descendants.
+    object_uris = list(object_uris)
+    object_uris.sort()
+
+    ancestor_object_uris = set()
+    for object_uri in object_uris:
+        include = True
+        parent_uri = get_parent_uri(object_uri)
+        while parent_uri:
+            if parent_uri in ancestor_object_uris:
+                # It's being deleted already.
+                include = False
+                break
+            parent_uri = get_parent_uri(parent_uri)
+
+        if include:
+            ancestor_object_uris.add(object_uri)
+
+    return list(ancestor_object_uris)
+
+
+@resource.register(
+    name="user-data",
+    description="Delete the data owned by a user",
+    plural_path="/__user_data__",
+    object_path="/__user_data__/{{principal}}",
+    viewset=UserDataViewSet(),
+    plural_methods=tuple(),
+    object_methods=("DELETE",),
+)
+class UserData(resource.Resource):
+    schema = Deleted
+
+    def delete(self):
+        principal = self.request.matchdict["principal"]
+        storage = self.request.registry.storage
+        permission = self.request.registry.permission
+        object_uris_and_permissions = permission.get_accessible_objects([principal])
+        object_uris = list(object_uris_and_permissions.keys())
+        write_perm_principals = permission.get_objects_permissions(object_uris, ["write"])
+        to_delete = set()
+        for object_uri, principals in zip(object_uris, write_perm_principals):
+            principals = principals["write"]
+            # "Ownership" isn't a real concept in Kinto, so instead we
+            # define ownership as meaning "this user is the only one
+            # who can write to this object".
+            if principals == set([principal]):
+                to_delete.add(object_uri)
+
+        # Any accessible objects that won't be deleted, need to have
+        # the user's permission removed.
+        for object_uri, permissions in object_uris_and_permissions.items():
+            if object_uri in to_delete:
+                continue
+
+            for perm in permissions:
+                permission.remove_principal_from_ace(object_uri, perm, principal)
+
+        to_delete = condense_under_parents(self.request, to_delete)
+
+        # Group by (parent_uri, resource of child) to make fewer
+        # requests to storage backend.
+        # Store the parsed object IDs, since those are what we
+        # actually give to the storage backend.
+        object_ids_by_parent_uri = collections.defaultdict(list)
+        # Store also the object URIs, which we give to the permission backend.
+        objects_by_parent_uri = collections.defaultdict(list)
+        # We have to get the matchdict of the child here anyhow, so
+        # keep that to generate events later.
+        matchdicts_by_parent_uri = {}
+        for object_uri in to_delete:
+            parent_uri = get_parent_uri(object_uri)
+            resource_name, matchdict = core_utils.view_lookup(self.request, object_uri)
+            objects_by_parent_uri[(parent_uri, resource_name)].append(object_uri)
+            object_ids_by_parent_uri[(parent_uri, resource_name)].append(matchdict["id"])
+            # This overwrites previous matchdicts for the parent, but
+            # we'll only use the fields that are relevant to the
+            # parent, which will be the same for each child.
+            matchdicts_by_parent_uri[parent_uri] = matchdict
+
+        for (parent_uri, resource_name), object_ids in object_ids_by_parent_uri.items():
+            # Generate the parent matchdict from an arbitrary child's matchdict.
+            matchdict = {**matchdicts_by_parent_uri[parent_uri]}
+            matchdict.pop("id", None)
+
+            # Deletes are paginated too, so take the page size from settings.
+            batch_size = self.request.registry.settings["storage_max_fetch_size"]
+            for batch in slice_into_batches(object_ids, batch_size):
+                batch = list(batch)
+                filters = [Filter("id", batch, core_utils.COMPARISON.IN)]
+                timestamp = storage.resource_timestamp(resource_name, parent_uri)
+                records = storage.list_all(
+                    resource_name=resource_name, parent_id=parent_uri, filters=filters
+                )
+                tombstones = storage.delete_all(
+                    resource_name=resource_name, parent_id=parent_uri, filters=filters
+                )
+                notify_resource_event(
+                    self.request,
+                    parent_uri,
+                    timestamp,
+                    tombstones,
+                    ACTIONS.DELETE,
+                    old=records,
+                    resource_name=resource_name,
+                    resource_data=matchdict,
+                )
+                # FIXME: need to purge the above tombstones, but no
+                # way to purge just some tombstones for just this
+                # principal
+
+                # Clear permissions from the deleted objects, for
+                # example those of other users.
+                permission.delete_object_permissions(
+                    *objects_by_parent_uri[(parent_uri, resource_name)]
+                )
+
+        # Remove this principal from existing users.
+        permission.remove_principal(principal)
+
+        # Remove this principal from all groups that contain it.
+        associated_principals = permission.get_user_principals(principal)
+        for associated_principal in associated_principals:
+            permission.remove_user_principal(principal, associated_principal)
+
+        return {"data": {"principal": principal}}

kinto/views/buckets.py

@@ -0,0 +1,45 @@
+import colander
+from pyramid.events import subscriber
+
+from kinto.core import resource
+from kinto.core.events import ACTIONS, ResourceChanged
+from kinto.core.utils import instance_uri
+from kinto.schema_validation import JSONSchemaMapping
+
+
+class BucketSchema(resource.ResourceSchema):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self["collection:schema"] = JSONSchemaMapping(missing=colander.drop)
+        self["group:schema"] = JSONSchemaMapping(missing=colander.drop)
+        self["record:schema"] = JSONSchemaMapping(missing=colander.drop)
+
+
+@resource.register(name="bucket", plural_path="/buckets", object_path="/buckets/{{id}}")
+class Bucket(resource.Resource):
+    schema = BucketSchema
+    permissions = ("read", "write", "collection:create", "group:create")
+
+    def get_parent_id(self, request):
+        # Buckets are not isolated by user, unlike Kinto-Core resources.
+        return ""
+
+
+@subscriber(ResourceChanged, for_resources=("bucket",), for_actions=(ACTIONS.DELETE,))
+def on_buckets_deleted(event):
+    """Some buckets were deleted, delete sub-resources."""
+    storage = event.request.registry.storage
+    permission = event.request.registry.permission
+
+    for change in event.impacted_objects:
+        bucket = change["old"]
+        bucket_uri = instance_uri(event.request, "bucket", id=bucket["id"])
+
+        # Delete everything with current parent id (eg. collections, groups)
+        # and descending children objects (eg. records).
+        for pattern in (bucket_uri, bucket_uri + "/*"):
+            storage.delete_all(parent_id=pattern, resource_name=None, with_deleted=False)
+            # Remove remaining tombstones too.
+            storage.purge_deleted(parent_id=pattern, resource_name=None)
+            # Remove related permissions
+            permission.delete_object_permissions(pattern)

kinto/views/collections.py

@@ -0,0 +1,58 @@
+import colander
+from pyramid.events import subscriber
+
+from kinto.core import resource, utils
+from kinto.core.events import ACTIONS, ResourceChanged
+from kinto.schema_validation import JSONSchemaMapping, validate_from_bucket_schema_or_400
+
+
+class CollectionSchema(resource.ResourceSchema):
+    schema = JSONSchemaMapping(missing=colander.drop)
+    cache_expires = colander.SchemaNode(colander.Int(), missing=colander.drop)
+
+
+@resource.register(
+    name="collection",
+    plural_path="/buckets/{{bucket_id}}/collections",
+    object_path="/buckets/{{bucket_id}}/collections/{{id}}",
+)
+class Collection(resource.Resource):
+    schema = CollectionSchema
+    permissions = ("read", "write", "record:create")
+
+    def get_parent_id(self, request):
+        bucket_id = request.matchdict["bucket_id"]
+        parent_id = utils.instance_uri(request, "bucket", id=bucket_id)
+        return parent_id
+
+    def process_object(self, new, old=None):
+        """Additional collection schema validation from bucket, if any."""
+        new = super().process_object(new, old)
+
+        # Remove internal and auto-assigned fields.
+        internal_fields = (self.model.modified_field, self.model.permissions_field)
+        validate_from_bucket_schema_or_400(
+            new,
+            resource_name="collection",
+            request=self.request,
+            ignore_fields=internal_fields,
+            id_field=self.model.id_field,
+        )
+        return new
+
+
+@subscriber(ResourceChanged, for_resources=("collection",), for_actions=(ACTIONS.DELETE,))
+def on_collections_deleted(event):
+    """Some collections were deleted, delete records."""
+    storage = event.request.registry.storage
+    permission = event.request.registry.permission
+
+    for change in event.impacted_objects:
+        collection = change["old"]
+        bucket_id = event.payload["bucket_id"]
+        parent_id = utils.instance_uri(
+            event.request, "collection", bucket_id=bucket_id, id=collection["id"]
+        )
+        storage.delete_all(resource_name=None, parent_id=parent_id, with_deleted=False)
+        storage.purge_deleted(resource_name=None, parent_id=parent_id)
+        permission.delete_object_permissions(parent_id + "/*")

kinto/views/contribute.py

@@ -0,0 +1,39 @@
+import json
+import os
+
+import colander
+from pyramid.security import NO_PERMISSION_REQUIRED
+
+from kinto.core import Service
+
+
+HERE = os.path.dirname(__file__)
+ORIGIN = os.path.dirname(HERE)  # package root.
+_CONTRIBUTE_INFO = None
+
+contribute = Service(
+    name="contribute.json", description="Open-source information", path="/contribute.json"
+)
+
+
+class ContributeResponseSchema(colander.MappingSchema):
+    body = colander.SchemaNode(colander.Mapping(unknown="preserve"))
+
+
+contribute_responses = {
+    "200": ContributeResponseSchema(description="Return open source contributing information.")
+}
+
+
+@contribute.get(
+    permission=NO_PERMISSION_REQUIRED,
+    tags=["Utilities"],
+    operation_id="contribute",
+    response_schemas=contribute_responses,
+)
+def contribute_get(request):
+    global _CONTRIBUTE_INFO
+    if _CONTRIBUTE_INFO is None:
+        with open(os.path.join(ORIGIN, "contribute.json")) as f:
+            _CONTRIBUTE_INFO = json.load(f)
+    return _CONTRIBUTE_INFO

kinto/views/groups.py

@@ -0,0 +1,90 @@
+import colander
+from pyramid.events import subscriber
+
+from kinto.core import resource, utils
+from kinto.core.events import ACTIONS, ResourceChanged
+from kinto.schema_validation import validate_from_bucket_schema_or_400
+
+
+def validate_member(node, member):
+    if member.startswith("/buckets/") or member == "system.Everyone":
+        raise colander.Invalid(node, f"'{member}' is not a valid user ID.")
+
+
+class GroupSchema(resource.ResourceSchema):
+    members = colander.SchemaNode(
+        colander.Sequence(),
+        colander.SchemaNode(colander.String(), validator=validate_member),
+        missing=[],
+    )
+
+
+@resource.register(
+    name="group",
+    plural_path="/buckets/{{bucket_id}}/groups",
+    object_path="/buckets/{{bucket_id}}/groups/{{id}}",
+)
+class Group(resource.Resource):
+    schema = GroupSchema
+
+    def get_parent_id(self, request):
+        bucket_id = request.matchdict["bucket_id"]
+        parent_id = utils.instance_uri(request, "bucket", id=bucket_id)
+        return parent_id
+
+    def process_object(self, new, old=None):
+        """Additional collection schema validation from bucket, if any."""
+        new = super().process_object(new, old)
+
+        # Remove internal and auto-assigned fields.
+        internal_fields = (self.model.modified_field, self.model.permissions_field)
+        validate_from_bucket_schema_or_400(
+            new,
+            resource_name="group",
+            request=self.request,
+            ignore_fields=internal_fields,
+            id_field=self.model.id_field,
+        )
+
+        return new
+
+
+@subscriber(ResourceChanged, for_resources=("group",), for_actions=(ACTIONS.DELETE,))
+def on_groups_deleted(event):
+    """Some groups were deleted, remove them from users principals."""
+    permission_backend = event.request.registry.permission
+
+    for change in event.impacted_objects:
+        group = change["old"]
+        bucket_id = event.payload["bucket_id"]
+        group_uri = utils.instance_uri(event.request, "group", bucket_id=bucket_id, id=group["id"])
+
+        permission_backend.remove_principal(group_uri)
+
+
+@subscriber(
+    ResourceChanged, for_resources=("group",), for_actions=(ACTIONS.CREATE, ACTIONS.UPDATE)
+)
+def on_groups_changed(event):
+    """Some groups were changed, update users principals."""
+    permission_backend = event.request.registry.permission
+
+    for change in event.impacted_objects:
+        if "old" in change:
+            existing_record_members = set(change["old"].get("members", []))
+        else:
+            existing_record_members = set()
+
+        group = change["new"]
+        group_uri = f"/buckets/{event.payload['bucket_id']}/groups/{group['id']}"
+        new_record_members = set(group.get("members", []))
+        new_members = new_record_members - existing_record_members
+        removed_members = existing_record_members - new_record_members
+
+        for member in new_members:
+            # Add the group to the member principal.
+            permission_backend.add_user_principal(member, group_uri)
+
+        for member in removed_members:
+            # Remove the group from the member principal.
+            permission_backend.remove_user_principal(member, group_uri)