kinto 23.2.1__py3-none-any.whl

kinto/core/views/batch.py

@@ -0,0 +1,163 @@
+import logging
+
+import colander
+from pyramid import httpexceptions
+from pyramid.security import NO_PERMISSION_REQUIRED
+
+from kinto.core import Service, errors
+from kinto.core.cornice.validators import colander_validator
+from kinto.core.errors import ErrorSchema
+from kinto.core.resource.viewset import CONTENT_TYPES
+from kinto.core.utils import build_request, build_response, merge_dicts
+
+
+subrequest_logger = logging.getLogger("subrequest.summary")
+
+valid_http_method = colander.OneOf(("GET", "HEAD", "DELETE", "TRACE", "POST", "PUT", "PATCH"))
+
+
+def string_values(node, cstruct):
+    """Validate that a ``colander.Mapping`` only has strings in its values.
+
+    .. warning::
+
+        Should be associated to a ``colander.Mapping`` schema node.
+    """
+    are_strings = [isinstance(v, str) for v in cstruct.values()]
+    if not all(are_strings):
+        error_msg = f"{cstruct} contains non string value"
+        raise colander.Invalid(node, error_msg)
+
+
+class BatchRequestSchema(colander.MappingSchema):
+    method = colander.SchemaNode(
+        colander.String(), validator=valid_http_method, missing=colander.drop
+    )
+    path = colander.SchemaNode(colander.String(), validator=colander.Regex("^/"))
+    headers = colander.SchemaNode(
+        colander.Mapping(unknown="preserve"), validator=string_values, missing=colander.drop
+    )
+    body = colander.SchemaNode(colander.Mapping(unknown="preserve"), missing=colander.drop)
+
+    @staticmethod
+    def schema_type():
+        return colander.Mapping(unknown="raise")
+
+
+class BatchPayloadSchema(colander.MappingSchema):
+    defaults = BatchRequestSchema(missing=colander.drop).clone()
+    requests = colander.SchemaNode(colander.Sequence(), BatchRequestSchema())
+
+    @staticmethod
+    def schema_type():
+        return colander.Mapping(unknown="raise")
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # On defaults, path is not mandatory.
+        self.get("defaults").get("path").missing = colander.drop
+
+    def deserialize(self, cstruct=colander.null):
+        """Preprocess received data to carefully merge defaults."""
+        if cstruct is not colander.null:
+            defaults = cstruct.get("defaults")
+            requests = cstruct.get("requests")
+            if isinstance(defaults, dict) and isinstance(requests, list):
+                for request in requests:
+                    if isinstance(request, dict):
+                        merge_dicts(request, defaults)
+        return super().deserialize(cstruct)
+
+
+class BatchRequest(colander.MappingSchema):
+    body = BatchPayloadSchema()
+
+
+class BatchResponseSchema(colander.MappingSchema):
+    status = colander.SchemaNode(colander.Integer())
+    path = colander.SchemaNode(colander.String())
+    headers = colander.SchemaNode(
+        colander.Mapping(unknown="preserve"), validator=string_values, missing=colander.drop
+    )
+    body = colander.SchemaNode(colander.Mapping(unknown="preserve"), missing=colander.drop)
+
+
+class BatchResponseBodySchema(colander.MappingSchema):
+    responses = colander.SequenceSchema(BatchResponseSchema(missing=colander.drop))
+
+
+class BatchResponse(colander.MappingSchema):
+    body = BatchResponseBodySchema()
+
+
+class ErrorResponseSchema(colander.MappingSchema):
+    body = ErrorSchema()
+
+
+batch_responses = {
+    "200": BatchResponse(description="Return a list of operation responses."),
+    "400": ErrorResponseSchema(description="The request was badly formatted."),
+    "default": ErrorResponseSchema(description="an unknown error occurred."),
+}
+
+batch = Service(name="batch", path="/batch", description="Batch operations")
+
+
+@batch.post(
+    schema=BatchRequest(),
+    validators=(colander_validator,),
+    content_type=CONTENT_TYPES,
+    permission=NO_PERMISSION_REQUIRED,
+    tags=["Batch"],
+    operation_id="batch",
+    response_schemas=batch_responses,
+)
+def post_batch(request):
+    requests = request.validated["body"]["requests"]
+
+    request.log_context(batch_size=len(requests))
+
+    limit = request.registry.settings["batch_max_requests"]
+    if limit and len(requests) > int(limit):
+        error_msg = f"Number of requests is limited to {limit}"
+        request.errors.add("body", "requests", error_msg)
+        return
+
+    if any([batch.path in req["path"] for req in requests]):
+        error_msg = f"Recursive call on {batch.path} endpoint is forbidden."
+        request.errors.add("body", "requests", error_msg)
+        return
+
+    responses = []
+
+    for subrequest_spec in requests:
+        subrequest = build_request(request, subrequest_spec)
+
+        log_context = {
+            **request.log_context(),
+            "path": subrequest.path,
+            "method": subrequest.method,
+        }
+        try:
+            # Invoke subrequest without individual transaction.
+            resp, subrequest = request.follow_subrequest(subrequest, use_tweens=False)
+        except httpexceptions.HTTPException as e:
+            # Since some request in the batch failed, we need to stop the parent request
+            # through Pyramid's transaction manager. 5XX errors are already caught by
+            # pyramid_tm's commit_veto
+            # https://github.com/Kinto/kinto/issues/624
+            if e.status_code == 409:
+                request.tm.abort()
+
+            if e.content_type == "application/json":
+                resp = e
+            else:
+                # JSONify raw Pyramid errors.
+                resp = errors.http_error(e)
+
+        subrequest_logger.info("subrequest.summary", extra=log_context)
+
+        dict_resp = build_response(resp, subrequest)
+        responses.append(dict_resp)
+
+    return {"responses": responses}

kinto/core/views/errors.py

@@ -0,0 +1,145 @@
+import logging
+
+from pyramid import httpexceptions
+from pyramid.authorization import Authenticated
+from pyramid.httpexceptions import HTTPTemporaryRedirect
+from pyramid.security import NO_PERMISSION_REQUIRED, forget
+from pyramid.settings import asbool
+from pyramid.view import view_config
+
+from kinto.core.errors import ERRORS, http_error, request_GET
+from kinto.core.storage import exceptions as storage_exceptions
+from kinto.core.utils import reapply_cors
+
+
+logger = logging.getLogger()
+
+
+@view_config(context=httpexceptions.HTTPForbidden, permission=NO_PERMISSION_REQUIRED)
+def authorization_required(response, request):
+    """Distinguish authentication required (``401 Unauthorized``) from
+    not allowed (``403 Forbidden``).
+    """
+    if Authenticated not in request.effective_principals:
+        if response.content_type != "application/json":
+            # This is always the case when `HTTPForbidden` is raised by Pyramid
+            # on protected views with unauthenticated requests.
+            error_msg = "Please authenticate yourself to use this endpoint."
+            response = http_error(
+                httpexceptions.HTTPUnauthorized(),
+                errno=ERRORS.MISSING_AUTH_TOKEN,
+                message=error_msg,
+            )
+        response.headers.extend(forget(request))
+        return response
+
+    if response.content_type != "application/json":
+        error_msg = "This user cannot access this resource."
+        response = http_error(
+            httpexceptions.HTTPForbidden(), errno=ERRORS.FORBIDDEN, message=error_msg
+        )
+    return reapply_cors(request, response)
+
+
+@view_config(context=httpexceptions.HTTPNotFound, permission=NO_PERMISSION_REQUIRED)
+def page_not_found(response, request):
+    """Return a JSON 404 error response."""
+    config_key = "trailing_slash_redirect_enabled"
+    redirect_enabled = request.registry.settings[config_key]
+    trailing_slash_redirection_enabled = asbool(redirect_enabled)
+
+    querystring = request.url[(request.url.rindex(request.path) + len(request.path)) :]
+
+    errno = ERRORS.MISSING_RESOURCE
+    error_msg = "The resource you are looking for could not be found."
+
+    if not request.path.startswith(f"/{request.registry.route_prefix}"):
+        errno = ERRORS.VERSION_NOT_AVAILABLE
+        error_msg = "The requested API version is not available on this server."
+    elif trailing_slash_redirection_enabled:
+        redirect = None
+
+        if request.path.endswith("/"):
+            path = request.path.rstrip("/")
+            redirect = f"{path}{querystring}"
+        elif request.path == f"/{request.registry.route_prefix}":
+            # Case for /v0 -> /v0/
+            redirect = f"/{request.registry.route_prefix}/{querystring}"
+
+        if redirect:
+            resp = HTTPTemporaryRedirect(redirect)
+            cache_seconds = int(request.registry.settings["trailing_slash_redirect_ttl_seconds"])
+            if cache_seconds >= 0:
+                resp.cache_expires(cache_seconds)
+            return reapply_cors(request, resp)
+
+    if response.content_type != "application/json":
+        response = http_error(httpexceptions.HTTPNotFound(), errno=errno, message=error_msg)
+    return reapply_cors(request, response)
+
+
+@view_config(context=httpexceptions.HTTPServiceUnavailable, permission=NO_PERMISSION_REQUIRED)
+def service_unavailable(response, request):
+    if response.content_type != "application/json":
+        error_msg = (
+            "Service temporary unavailable due to overloading or maintenance, please retry later."
+        )
+        response = http_error(response, errno=ERRORS.BACKEND, message=error_msg)
+
+    retry_after = request.registry.settings["retry_after_seconds"]
+    response.headers["Retry-After"] = str(retry_after)
+    return reapply_cors(request, response)
+
+
+@view_config(context=httpexceptions.HTTPMethodNotAllowed, permission=NO_PERMISSION_REQUIRED)
+def method_not_allowed(context, request):
+    if context.content_type == "application/json":
+        return context
+
+    response = http_error(
+        context, errno=ERRORS.METHOD_NOT_ALLOWED, message="Method not allowed on this endpoint."
+    )
+    return reapply_cors(request, response)
+
+
+@view_config(context=Exception, permission=NO_PERMISSION_REQUIRED)
+@view_config(context=httpexceptions.HTTPException, permission=NO_PERMISSION_REQUIRED)
+def error(context, request):
+    """Catch server errors and trace them."""
+    if isinstance(context, httpexceptions.Response):
+        return reapply_cors(request, context)
+
+    if isinstance(context, storage_exceptions.IntegrityError):
+        error_msg = "Integrity constraint violated, please retry."
+        response = http_error(
+            httpexceptions.HTTPConflict(), errno=ERRORS.CONSTRAINT_VIOLATED, message=error_msg
+        )
+        retry_after = request.registry.settings["retry_after_seconds"]
+        response.headers["Retry-After"] = str(retry_after)
+        return reapply_cors(request, response)
+
+    # Log some information about current request.
+    extra = {"path": request.path, "method": request.method}
+    qs = dict(request_GET(request))
+    if qs:
+        extra["querystring"] = qs
+    # Take errno from original exception, or undefined if unknown/unhandled.
+    try:
+        extra["errno"] = context.errno.value
+    except AttributeError:
+        extra["errno"] = ERRORS.UNDEFINED.value
+
+    if isinstance(context, storage_exceptions.BackendError):
+        logger.critical(context.original, extra=extra, exc_info=context)
+        response = httpexceptions.HTTPServiceUnavailable()
+        return service_unavailable(response, request)
+
+    # Within the exception view, sys.exc_info() will return null.
+    # see https://github.com/python/cpython/blob/ce9e62544/Lib/logging/__init__.py#L1460-L1462
+    logger.error(context, extra=extra, exc_info=context)
+
+    error_msg = "A programmatic error occured, developers have been informed."
+    info = request.registry.settings["error_info_link"]
+    response = http_error(httpexceptions.HTTPInternalServerError(), message=error_msg, info=info)
+
+    return reapply_cors(request, response)

kinto/core/views/heartbeat.py

@@ -0,0 +1,106 @@
+import logging
+from concurrent.futures import ThreadPoolExecutor, wait
+
+import colander
+import transaction
+from pyramid.security import NO_PERMISSION_REQUIRED
+
+from kinto.core import Service
+
+
+logger = logging.getLogger(__name__)
+
+
+heartbeat = Service(name="heartbeat", path="/__heartbeat__", description="Server health")
+
+
+class HeartbeatResponseSchema(colander.MappingSchema):
+    body = colander.SchemaNode(colander.Mapping(unknown="preserve"))
+
+
+heartbeat_responses = {
+    "200": HeartbeatResponseSchema(description="Server is working properly."),
+    "503": HeartbeatResponseSchema(description="One or more subsystems failing."),
+}
+
+
+@heartbeat.get(
+    permission=NO_PERMISSION_REQUIRED,
+    tags=["Utilities"],
+    operation_id="__heartbeat__",
+    response_schemas=heartbeat_responses,
+)
+def get_heartbeat(request):
+    """Return information about server health."""
+    status = {}
+
+    def heartbeat_check(name, func):
+        status[name] = False
+        status[name] = func(request)
+        # Since the heartbeat checks run concurrently, their transactions
+        # overlap and might end in shared lock errors. By aborting here
+        # we clean-up the state on each heartbeat call instead of once at the
+        # end of the request. See bug Kinto/kinto#804
+        transaction.abort()
+
+    # Start executing heartbeats concurrently.
+    heartbeats = request.registry.heartbeats
+    pool = ThreadPoolExecutor(max_workers=max(1, len(heartbeats.keys())))
+    futures = []
+    for name, func in heartbeats.items():
+        future = pool.submit(heartbeat_check, name, func)
+        future.__heartbeat_name = name  # For logging purposes.
+        futures.append(future)
+
+    # Wait for the results, with timeout.
+    seconds = float(request.registry.settings["heartbeat_timeout_seconds"])
+    done, not_done = wait(futures, timeout=seconds)
+
+    # A heartbeat is supposed to return True or False, and never raise.
+    # Just in case, go though results to spot any potential exception.
+    for future in done:
+        exc = future.exception()
+        if exc is not None:
+            logger.error(f"'{future.__heartbeat_name}' heartbeat failed.")
+            logger.error(exc)
+
+    # Log timed-out heartbeats.
+    for future in not_done:
+        name = future.__heartbeat_name
+        error_msg = f"'{name}' heartbeat has exceeded timeout of {seconds} seconds."
+        logger.error(error_msg)
+
+    # If any has failed, return a 503 error response.
+    has_error = not all([v or v is None for v in status.values()])
+    if has_error:
+        request.response.status = 503
+
+    return status
+
+
+class LbHeartbeatResponseSchema(colander.MappingSchema):
+    body = colander.SchemaNode(colander.Mapping())
+
+
+lbheartbeat_responses = {
+    "200": LbHeartbeatResponseSchema(description="Returned if server is reachable.")
+}
+
+
+lbheartbeat = Service(name="lbheartbeat", path="/__lbheartbeat__", description="Web head health")
+
+
+@lbheartbeat.get(
+    permission=NO_PERMISSION_REQUIRED,
+    tags=["Utilities"],
+    operation_id="__lbheartbeat__",
+    response_schemas=lbheartbeat_responses,
+)
+def get_lbheartbeat(request):
+    """Return successful healthy response.
+
+    If the load-balancer tries to access this URL and fails, this means the
+    Web head is not operational and should be dropped.
+    """
+    status = {}
+    return status

kinto/core/views/hello.py

@@ -0,0 +1,69 @@
+import colander
+from pyramid.authorization import Authenticated
+from pyramid.security import NO_PERMISSION_REQUIRED
+
+from kinto.config import config_attributes
+from kinto.core import Service
+
+
+hello = Service(name="hello", path="/", description="Welcome")
+
+
+class HelloResponseSchema(colander.MappingSchema):
+    body = colander.SchemaNode(colander.Mapping(unknown="preserve"))
+
+
+hello_response_schemas = {
+    "200": HelloResponseSchema(description="Return information about the running Instance.")
+}
+
+
+@hello.get(
+    permission=NO_PERMISSION_REQUIRED,
+    tags=["Utilities"],
+    operation_id="server_info",
+    response_schemas=hello_response_schemas,
+)
+def get_hello(request):
+    """Return information regarding the current instance."""
+    settings = request.registry.settings
+
+    project_name = settings["project_name"]
+    project_version = settings["project_version"]
+
+    data = dict(
+        project_name=project_name,
+        project_version=project_version,
+        http_api_version=settings["http_api_version"],
+        project_docs=settings["project_docs"],
+        url=request.route_url(hello.name),
+        config=config_attributes(),
+    )
+
+    eos = get_eos(request)
+    if eos:
+        data["eos"] = eos
+
+    data["settings"] = {}
+    public_settings = request.registry.public_settings
+    for setting in list(public_settings):
+        data["settings"][setting] = settings[setting]
+
+    # If current user is authenticated, add user info:
+    # (Note: this will call authenticated_userid() with multiauth+groupfinder)
+    if Authenticated in request.effective_principals:
+        data["user"] = request.get_user_info()
+
+    if settings["readonly"]:
+        # Information can be cached.
+        cache_seconds = int(settings["root_cache_expires_seconds"])
+        request.response.cache_expires(cache_seconds)
+
+    # Application can register and expose arbitrary capabilities.
+    data["capabilities"] = request.registry.api_capabilities
+
+    return data
+
+
+def get_eos(request):
+    return request.registry.settings["eos"]

kinto/core/views/openapi.py

@@ -0,0 +1,35 @@
+import colander
+from pyramid.security import NO_PERMISSION_REQUIRED
+
+from kinto.core import Service
+from kinto.core.cornice.service import get_services
+from kinto.core.openapi import OpenAPI
+
+
+openapi = Service(name="openapi", path="/__api__", description="OpenAPI description")
+
+
+class OpenAPIResponseSchema(colander.MappingSchema):
+    body = colander.SchemaNode(colander.Mapping(unknown="preserve"))
+
+
+openapi_response_schemas = {
+    "200": OpenAPIResponseSchema(
+        description="Return an OpenAPI description of the running instance."
+    )
+}
+
+
+@openapi.get(
+    permission=NO_PERMISSION_REQUIRED,
+    response_schemas=openapi_response_schemas,
+    tags=["Utilities"],
+    operation_id="get_openapi_spec",
+)
+def openapi_view(request):
+    # Only build json once
+    try:
+        return openapi_view.__json__
+    except AttributeError:
+        openapi_view.__json__ = OpenAPI(get_services(), request).generate()
+        return openapi_view.__json__

kinto/core/views/version.py

@@ -0,0 +1,50 @@
+import os
+
+import colander
+from pyramid.security import NO_PERMISSION_REQUIRED
+
+from kinto.core import Service
+from kinto.core.utils import json
+
+
+HERE = os.path.dirname(__file__)
+ORIGIN = os.path.dirname(HERE)
+
+
+class VersionResponseSchema(colander.MappingSchema):
+    body = colander.SchemaNode(colander.Mapping(unknown="preserve"))
+
+
+version_response_schemas = {
+    "200": VersionResponseSchema(description="Return the running Instance version information.")
+}
+
+
+version = Service(name="version", path="/__version__", description="Version")
+
+
+@version.get(
+    permission=NO_PERMISSION_REQUIRED,
+    tags=["Utilities"],
+    operation_id="__version__",
+    response_schemas=version_response_schemas,
+)
+def version_view(request):
+    try:
+        return version_view.__json__
+    except AttributeError:
+        pass
+
+    location = request.registry.settings["version_json_path"]
+    files = [
+        location,  # Default is current working dir.
+        os.path.join(ORIGIN, "version.json"),  # Relative to the package root.
+        os.path.join(HERE, "version.json"),  # Relative to this file.
+    ]
+    for version_file in files:
+        if os.path.exists(version_file):
+            with open(version_file) as f:
+                version_view.__json__ = json.load(f)
+                return version_view.__json__  # First one wins.
+
+    raise FileNotFoundError("Version file missing from {}".format(",".join(files)))

kinto/events.py

@@ -0,0 +1,3 @@
+class ServerFlushed:
+    def __init__(self, request):
+        self.request = request

kinto/plugins/accounts/__init__.py

@@ -0,0 +1,94 @@
+import re
+import sys
+
+from pyramid.exceptions import ConfigurationError
+
+from kinto.authorization import PERMISSIONS_INHERITANCE_TREE
+
+from .authentication import AccountsAuthenticationPolicy as AccountsPolicy
+from .utils import ACCOUNT_CACHE_KEY, ACCOUNT_POLICY_NAME
+
+
+__all__ = [
+    "ACCOUNT_CACHE_KEY",
+    "ACCOUNT_POLICY_NAME",
+    "AccountsPolicy",
+]
+
+DOCS_URL = "https://kinto.readthedocs.io/en/stable/api/1.x/accounts.html"
+
+
+def includeme(config):
+    settings = config.get_settings()
+    config.add_api_capability(
+        "accounts",
+        description="Manage user accounts.",
+        url="https://kinto.readthedocs.io/en/latest/api/1.x/accounts.html",
+        validation_enabled=False,
+    )
+    kwargs = {}
+    config.scan("kinto.plugins.accounts.views", **kwargs)
+
+    PERMISSIONS_INHERITANCE_TREE["root"].update({"account:create": {}})
+    PERMISSIONS_INHERITANCE_TREE["account"] = {
+        "write": {"account": ["write"]},
+        "read": {"account": ["write", "read"]},
+    }
+
+    # Check that the account policy is mentioned in config if included.
+    accountClass = "AccountsPolicy"
+    policy = None
+    for k, v in settings.items():
+        m = re.match("multiauth\\.policy\\.(.*)\\.use", k)
+        if m:
+            if v.endswith(accountClass) or v.endswith("AccountsAuthenticationPolicy"):
+                policy = m.group(1)
+
+    if settings["storage_backend"] == "kinto.core.storage.memory":  # pragma: no cover
+        error_msg = (
+            "\033[1;31;40m"
+            "The account plugin works really poorly with the memory backend because "
+            "accounts are flushed at each startup.\nThis is why you can't use the "
+            "kinto create-user command with it."
+            "\033[0;37;40m"
+        )
+        print(error_msg, file=sys.stderr)
+
+    if not policy:
+        error_msg = (
+            "Account policy missing the 'multiauth.policy.*.use' "
+            f"setting. See {accountClass} in docs {DOCS_URL}."
+        )
+        raise ConfigurationError(error_msg)
+
+    # Add some safety to avoid weird behaviour with basicauth default policy.
+    auth_policies = settings["multiauth.policies"]
+    if "basicauth" in auth_policies and policy in auth_policies:
+        if auth_policies.index("basicauth") < auth_policies.index(policy):
+            error_msg = (
+                "'basicauth' should not be mentioned before '%s' in 'multiauth.policies' setting."
+            ) % policy
+            raise ConfigurationError(error_msg)
+
+    # We assume anyone in account_create_principals is to create
+    # accounts for other people.
+    # No one can create accounts for other people unless they are an
+    # "admin", defined as someone matching account_write_principals.
+    # Therefore any account that is in account_create_principals
+    # should be in account_write_principals too.
+    creators = set(settings.get("account_create_principals", "").split())
+    admins = set(settings.get("account_write_principals", "").split())
+    cant_create_anything = creators.difference(admins)
+    # system.Everyone isn't an account.
+    cant_create_anything.discard("system.Everyone")
+    if cant_create_anything:
+        message = (
+            "Configuration has some principals in account_create_principals "
+            "but not in account_write_principals. These principals will only be "
+            "able to create their own accounts. This may not be what you want.\n"
+            "If you want these users to be able to create accounts for other users, "
+            "add them to account_write_principals.\n"
+            f"Affected users: {list(cant_create_anything)}"
+        )
+
+        raise ConfigurationError(message)