leblob 0.4.0.dev1__py3-none-any.whl

leblob/s3/routing.py

@@ -0,0 +1,443 @@
+"""Map an HTTP request to an S3 operation, and validate names once."""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from enum import Enum, auto
+from typing import TYPE_CHECKING
+from urllib.parse import parse_qsl, unquote_to_bytes
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from leblob.model import HttpRequest
+
+
+class Operation(Enum):
+    HEALTH = auto()
+    METRICS = auto()
+    LIST_BUCKETS = auto()
+
+    CREATE_BUCKET = auto()
+    DELETE_BUCKET = auto()
+    HEAD_BUCKET = auto()
+    GET_BUCKET_LOCATION = auto()
+    GET_BUCKET_VERSIONING = auto()
+    PUT_BUCKET_VERSIONING = auto()
+    GET_BUCKET_LIFECYCLE = auto()
+    PUT_BUCKET_LIFECYCLE = auto()
+    DELETE_BUCKET_LIFECYCLE = auto()
+    GET_BUCKET_CORS = auto()
+    PUT_BUCKET_CORS = auto()
+    DELETE_BUCKET_CORS = auto()
+    CORS_PREFLIGHT = auto()
+    POST_OBJECT = auto()
+    GET_BUCKET_ACL = auto()
+    PUT_BUCKET_ACL = auto()
+    LIST_OBJECTS = auto()
+    LIST_OBJECTS_V2 = auto()
+    LIST_MULTIPART_UPLOADS = auto()
+    LIST_OBJECT_VERSIONS = auto()
+    DELETE_OBJECTS = auto()
+
+    PUT_OBJECT = auto()
+    COPY_OBJECT = auto()
+    GET_OBJECT = auto()
+    HEAD_OBJECT = auto()
+    DELETE_OBJECT = auto()
+    DELETE_OBJECT_VERSION = auto()
+    GET_OBJECT_TAGGING = auto()
+    PUT_OBJECT_TAGGING = auto()
+    DELETE_OBJECT_TAGGING = auto()
+    GET_OBJECT_ACL = auto()
+    PUT_OBJECT_ACL = auto()
+    CREATE_MULTIPART_UPLOAD = auto()
+    UPLOAD_PART = auto()
+    UPLOAD_PART_COPY = auto()
+    COMPLETE_MULTIPART_UPLOAD = auto()
+    ABORT_MULTIPART_UPLOAD = auto()
+    LIST_PARTS = auto()
+
+
+@dataclass(frozen=True, slots=True)
+class ServiceRoute:
+    operation: Operation
+
+
+@dataclass(frozen=True, slots=True)
+class BucketRoute:
+    operation: Operation
+    bucket: str
+
+
+@dataclass(frozen=True, slots=True)
+class ObjectRoute:
+    operation: Operation
+    bucket: str
+    key: str
+
+
+type Route = ServiceRoute | BucketRoute | ObjectRoute
+type _Rule = tuple[tuple[tuple[str, Operation], ...], Operation | None]
+
+HEALTH_PATH = b"/-/health"
+METRICS_PATH = b"/-/metrics"
+MAX_KEY_BYTES = 1024
+
+_BUCKET_NAME = re.compile(r"[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]")
+_BUCKET_NAME_BAD_SEQUENCE = re.compile(r"\.\.|\.-|-\.")
+_IP_ADDRESS = re.compile(r"\d+\.\d+\.\d+\.\d+")
+
+# Each entry: (query parameter that selects an operation, ...), default operation.
+_BUCKET_RULES: dict[str, _Rule] = {
+    "GET": (
+        (
+            ("location", Operation.GET_BUCKET_LOCATION),
+            ("versioning", Operation.GET_BUCKET_VERSIONING),
+            ("lifecycle", Operation.GET_BUCKET_LIFECYCLE),
+            ("cors", Operation.GET_BUCKET_CORS),
+            ("acl", Operation.GET_BUCKET_ACL),
+            ("uploads", Operation.LIST_MULTIPART_UPLOADS),
+            ("versions", Operation.LIST_OBJECT_VERSIONS),
+            ("list-type", Operation.LIST_OBJECTS_V2),
+        ),
+        Operation.LIST_OBJECTS,
+    ),
+    "HEAD": ((), Operation.HEAD_BUCKET),
+    "PUT": (
+        (
+            ("versioning", Operation.PUT_BUCKET_VERSIONING),
+            ("lifecycle", Operation.PUT_BUCKET_LIFECYCLE),
+            ("cors", Operation.PUT_BUCKET_CORS),
+            ("acl", Operation.PUT_BUCKET_ACL),
+        ),
+        Operation.CREATE_BUCKET,
+    ),
+    "DELETE": (
+        (
+            ("lifecycle", Operation.DELETE_BUCKET_LIFECYCLE),
+            ("cors", Operation.DELETE_BUCKET_CORS),
+        ),
+        Operation.DELETE_BUCKET,
+    ),
+    "POST": ((("delete", Operation.DELETE_OBJECTS),), None),
+}
+_OBJECT_RULES: dict[str, _Rule] = {
+    "GET": (
+        (
+            ("tagging", Operation.GET_OBJECT_TAGGING),
+            ("acl", Operation.GET_OBJECT_ACL),
+            ("uploadId", Operation.LIST_PARTS),
+        ),
+        Operation.GET_OBJECT,
+    ),
+    "HEAD": ((), Operation.HEAD_OBJECT),
+    "PUT": (
+        (
+            ("tagging", Operation.PUT_OBJECT_TAGGING),
+            ("acl", Operation.PUT_OBJECT_ACL),
+            ("uploadId", Operation.UPLOAD_PART),
+        ),
+        Operation.PUT_OBJECT,
+    ),
+    "DELETE": (
+        (
+            # First: `?versionId=x&tagging` must not delete the version.
+            ("tagging", Operation.DELETE_OBJECT_TAGGING),
+            ("uploadId", Operation.ABORT_MULTIPART_UPLOAD),
+            ("versionId", Operation.DELETE_OBJECT_VERSION),
+        ),
+        Operation.DELETE_OBJECT,
+    ),
+    "POST": (
+        (
+            ("uploads", Operation.CREATE_MULTIPART_UPLOAD),
+            ("uploadId", Operation.COMPLETE_MULTIPART_UPLOAD),
+        ),
+        None,
+    ),
+}
+
+# Query parameters that change what a request means. Serving a plain object or
+# listing in answer to one of these would be a silent lie, so they are refused.
+_UNSUPPORTED_PARAMETERS = frozenset({
+    "accelerate",
+    "acl",
+    "analytics",
+    "attributes",
+    "cors",
+    "encryption",
+    "intelligent-tiering",
+    "inventory",
+    "legal-hold",
+    "lifecycle",
+    "logging",
+    "metrics",
+    "notification",
+    "object-lock",
+    "ownershipControls",
+    "partNumber",
+    "policy",
+    "policyStatus",
+    "publicAccessBlock",
+    "replication",
+    "requestPayment",
+    "restore",
+    "retention",
+    "select",
+    "tagging",
+    "torrent",
+    "versionId",
+    "versioning",
+    "website",
+})
+
+
+_VERSIONED_READS = frozenset({"versionId"})
+_COPY_SOURCE_HEADER = "x-amz-copy-source"
+_COPY_RANGE_HEADER = "x-amz-copy-source-range"
+_COPYING_FORM = {
+    Operation.PUT_OBJECT: Operation.COPY_OBJECT,
+    Operation.UPLOAD_PART: Operation.UPLOAD_PART_COPY,
+}
+_CUSTOMER_KEY_PREFIX = "x-amz-server-side-encryption-customer-"
+_COPY_SOURCE_KEY_PREFIX = "x-amz-copy-source-server-side-encryption-customer-"
+_REQUESTS_WITH_A_KEY = frozenset({
+    Operation.PUT_OBJECT,
+    Operation.GET_OBJECT,
+    Operation.HEAD_OBJECT,
+    Operation.COPY_OBJECT,
+    Operation.CREATE_MULTIPART_UPLOAD,
+    Operation.UPLOAD_PART,
+    Operation.UPLOAD_PART_COPY,
+})
+_REQUESTS_WITH_A_SOURCE_KEY = frozenset({
+    Operation.COPY_OBJECT,
+    Operation.UPLOAD_PART_COPY,
+})
+_ACL_HEADER = "x-amz-acl"
+_CANNED_ACLS = frozenset({"private", "public-read"})
+_WRITES_WITH_ACL = frozenset({
+    Operation.CREATE_BUCKET,
+    Operation.PUT_BUCKET_ACL,
+    Operation.PUT_OBJECT_ACL,
+    Operation.PUT_OBJECT,
+    Operation.COPY_OBJECT,
+    Operation.CREATE_MULTIPART_UPLOAD,
+})
+_TAGS_HEADER = "x-amz-tagging"
+_WRITES_WITH_TAGS = frozenset({
+    Operation.PUT_OBJECT,
+    Operation.COPY_OBJECT,
+    Operation.CREATE_MULTIPART_UPLOAD,
+})
+_PARTS_OF_AN_UPLOAD = frozenset({"partNumber"})
+
+_UNSUPPORTED_HEADER_PREFIXES = (
+    "x-amz-grant-",
+    "x-amz-if-match-",
+    "x-amz-object-lock-",
+    "x-amz-server-side-encryption",
+    "x-amz-website-redirect-location",
+)
+# Asking for what we do anyway is fine: rclone sends the object-lock header on
+# every CreateBucket, with the value "false".
+_ACCEPTED_HEADER_VALUES = {
+    "x-amz-bucket-object-lock-enabled": "false",
+    "x-amz-storage-class": "STANDARD",
+}
+
+
+def route(request: HttpRequest, base_domain: str | None) -> Route:
+    """Decide which S3 operation a request asks for."""
+    if request.raw_path == HEALTH_PATH:
+        return ServiceRoute(Operation.HEALTH)
+    if request.raw_path == METRICS_PATH:
+        return ServiceRoute(Operation.METRICS)
+    refuse_unsupported_headers(request.headers)
+
+    host = request.headers.get("host", "")
+    bucket, raw_key = _split_target(host, request.raw_path, base_domain)
+    if bucket is None:
+        if request.method != "GET":
+            raise S3Error(errors.METHOD_NOT_ALLOWED)
+        return ServiceRoute(Operation.LIST_BUCKETS)
+
+    if not raw_key:
+        return _route_bucket(request, bucket)
+    if request.method == "OPTIONS":
+        return ObjectRoute(Operation.CORS_PREFLIGHT, bucket, parse_key(raw_key))
+
+    operation = _select(
+        _OBJECT_RULES, request.method, request.query, _tolerated(request)
+    )
+    if _COPY_SOURCE_HEADER in request.headers:
+        operation = _COPYING_FORM.get(operation, operation)
+    _refuse_stray_headers(request.headers, operation)
+    return ObjectRoute(operation, bucket, parse_key(raw_key))
+
+
+def _route_bucket(request: HttpRequest, bucket: str) -> BucketRoute:
+    # Whatever the query asks for: it describes the request a browser means to send.
+    if request.method == "OPTIONS":
+        return BucketRoute(Operation.CORS_PREFLIGHT, bucket)
+    selected = (
+        Operation.POST_OBJECT
+        if _is_form_upload(request)
+        else _select(_BUCKET_RULES, request.method, request.query)
+    )
+    _refuse_stray_headers(request.headers, selected)
+    return BucketRoute(selected, bucket)
+
+
+def parse_query(raw_query: bytes) -> dict[str, str]:
+    return dict(parse_qsl(raw_query.decode("latin-1"), keep_blank_values=True))
+
+
+def parse_bucket_name(name: str) -> str:
+    """Return a valid bucket name, or raise InvalidBucketName."""
+    valid = (
+        _BUCKET_NAME.fullmatch(name)
+        and not _BUCKET_NAME_BAD_SEQUENCE.search(name)
+        and not _IP_ADDRESS.fullmatch(name)
+    )
+    if not valid:
+        raise S3Error(errors.INVALID_BUCKET_NAME, f"Invalid bucket name: {name!r}.")
+    return name
+
+
+def parse_raw_bucket(raw_bucket: bytes) -> str:
+    """Return a valid bucket name from its percent-encoded wire form."""
+    return parse_bucket_name(unquote_to_bytes(raw_bucket).decode(errors="replace"))
+
+
+def parse_key(raw_key: bytes) -> str:
+    """Return a valid object key from its percent-encoded wire form."""
+    return parse_decoded_key(unquote_to_bytes(raw_key))
+
+
+def parse_decoded_key(decoded: bytes) -> str:
+    """Return a valid object key from its raw bytes."""
+    if len(decoded) > MAX_KEY_BYTES:
+        raise S3Error(errors.KEY_TOO_LONG)
+    if b"\x00" in decoded:
+        raise S3Error(errors.INVALID_ARGUMENT, "Object keys cannot contain NUL.")
+    try:
+        return decoded.decode("utf-8")
+    except UnicodeDecodeError:
+        raise S3Error(
+            errors.INVALID_ARGUMENT, "Object keys must be valid UTF-8."
+        ) from None
+
+
+def refuse_unsupported_headers(headers: Mapping[str, str]) -> None:
+    """Refuse promises we can't keep, above all encryption.
+
+    Ignoring them would store plaintext while the client believes it is encrypted.
+    """
+    for name, value in headers.items():
+        if name == _ACL_HEADER and value not in _CANNED_ACLS:
+            raise S3Error(
+                errors.NOT_IMPLEMENTED, f"{name}: {value} is not implemented."
+            )
+        if name.startswith(_UNSUPPORTED_HEADER_PREFIXES) and not name.startswith(
+            _CUSTOMER_KEY_PREFIX
+        ):
+            raise S3Error(errors.NOT_IMPLEMENTED, f"Header {name} is not implemented.")
+        if _ACCEPTED_HEADER_VALUES.get(name, value) != value:
+            raise S3Error(
+                errors.NOT_IMPLEMENTED, f"{name}: {value} is not implemented."
+            )
+
+
+def _refuse_stray_headers(headers: Mapping[str, str], operation: Operation) -> None:
+    """Refuse what a request asks for and its operation wouldn't do, not to lose it."""
+    if _COPY_SOURCE_HEADER in headers and operation not in _REQUESTS_WITH_A_SOURCE_KEY:
+        raise S3Error(errors.NOT_IMPLEMENTED, "This request can't copy.")
+    # Copying a whole object instead would be more than was asked for.
+    if _COPY_RANGE_HEADER in headers and operation is not Operation.UPLOAD_PART_COPY:
+        raise S3Error(errors.NOT_IMPLEMENTED, "This request can't copy a range.")
+    if _TAGS_HEADER in headers and operation not in _WRITES_WITH_TAGS:
+        raise S3Error(errors.NOT_IMPLEMENTED, "This request can't set tags.")
+    if headers.get(_ACL_HEADER) == "public-read" and operation not in _WRITES_WITH_ACL:
+        raise S3Error(errors.NOT_IMPLEMENTED, "This request can't set an ACL.")
+    # A key ignored means a client believes it encrypted what it didn't.
+    for name in headers:
+        if (
+            name.startswith(_CUSTOMER_KEY_PREFIX)
+            and operation not in _REQUESTS_WITH_A_KEY
+        ) or (
+            name.startswith(_COPY_SOURCE_KEY_PREFIX)
+            and operation not in _REQUESTS_WITH_A_SOURCE_KEY
+        ):
+            raise S3Error(errors.NOT_IMPLEMENTED, f"This request can't take {name}.")
+
+
+def _is_form_upload(request: HttpRequest) -> bool:
+    return (
+        request.method == "POST"
+        and not request.query
+        and request.headers
+        .get("content-type", "")
+        .lower()
+        .startswith("multipart/form-data")
+    )
+
+
+def _split_target(
+    host: str, raw_path: bytes, base_domain: str | None
+) -> tuple[str | None, bytes]:
+    """Separate the bucket from the still-encoded key, for both addressing styles."""
+    path = raw_path.removeprefix(b"/")
+    hostname = host.rsplit(":", 1)[0].lower()
+    if base_domain and hostname.endswith(f".{base_domain}"):
+        return parse_bucket_name(hostname.removesuffix(f".{base_domain}")), path
+
+    raw_bucket, _, raw_key = path.partition(b"/")
+    if not raw_bucket:
+        return None, b""
+    return parse_raw_bucket(raw_bucket), raw_key
+
+
+def _tolerated(request: HttpRequest) -> frozenset[str]:
+    """Parameters that are part of an object operation, and don't change its meaning."""
+    if "uploadId" in request.query:
+        return _PARTS_OF_AN_UPLOAD
+    versioned = (
+        request.method in {"GET", "HEAD"}
+        or "tagging" in request.query
+        or (request.method == "PUT" and "acl" in request.query)
+    )
+    return _VERSIONED_READS if versioned else frozenset()
+
+
+def _select(
+    rules: dict[str, _Rule],
+    method: str,
+    query: Mapping[str, str],
+    tolerated: frozenset[str] = frozenset(),
+) -> Operation:
+    if method not in rules:
+        raise S3Error(errors.METHOD_NOT_ALLOWED)
+    selectors, default = rules[method]
+    # Before the selectors: `?versionId=x&tagging` must not delete a version.
+    selecting = {parameter for parameter, _ in selectors}
+    unsupported = sorted(
+        (query.keys() & _UNSUPPORTED_PARAMETERS) - selecting - tolerated
+    )
+    if unsupported:
+        raise S3Error(
+            errors.NOT_IMPLEMENTED,
+            f"{method} with ?{unsupported[0]} is not implemented.",
+        )
+    for parameter, operation in selectors:
+        if parameter in query:
+            return operation
+
+    if default is None:
+        raise S3Error(errors.NOT_IMPLEMENTED, f"This {method} form is not implemented.")
+    return default