leblob 0.4.0.dev1__py3-none-any.whl

leblob/s3/xml.py

@@ -0,0 +1,436 @@
+"""S3 XML documents: rendering responses and parsing request bodies."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from email.utils import formatdate
+from typing import TYPE_CHECKING
+from urllib.parse import quote
+
+from leblob.model import (
+    Checksum,
+    DeleteTarget,
+    RequestedPart,
+    Versioning,
+)
+from leblob.s3 import errors, xml_base as base
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Mapping
+    from xml.etree.ElementTree import Element
+
+    from leblob.model import (
+        BucketRecord,
+        DeleteOutcome,
+        ObjectSummary,
+        Part,
+        Upload,
+        UploadRef,
+        VersionSummary,
+    )
+    from leblob.s3.listing import ListPage
+
+    type Encoder = Callable[[str], str]
+
+MAX_DELETE_KEYS = 1000
+_CHECKSUM_ELEMENT_PREFIX = "Checksum"
+_CHECKSUM_TYPE_ELEMENT = "ChecksumType"
+
+
+def http_date(milliseconds: int) -> str:
+    return formatdate(milliseconds / 1000, usegmt=True)
+
+
+def iso_date(milliseconds: int) -> str:
+    moment = datetime.fromtimestamp(milliseconds / 1000, UTC)
+    return f"{moment:%Y-%m-%dT%H:%M:%S}.{milliseconds % 1000:03d}Z"
+
+
+def render_error(code: str, message: str, resource: str, request_id: str) -> bytes:
+    return base.document(
+        "Error",
+        base.element("Code", code),
+        base.element("Message", message),
+        base.element("Resource", resource),
+        base.element("RequestId", request_id),
+        namespace="",
+    )
+
+
+def render_buckets(buckets: Iterable[BucketRecord]) -> bytes:
+    entries = "".join(
+        "<Bucket>"
+        f"{base.element('Name', bucket.name)}"
+        f"{base.element('CreationDate', iso_date(bucket.created_at))}"
+        "</Bucket>"
+        for bucket in buckets
+    )
+    return base.document(
+        "ListAllMyBucketsResult", base.OWNER, f"<Buckets>{entries}</Buckets>"
+    )
+
+
+def render_location() -> bytes:
+    return base.document("LocationConstraint")
+
+
+def render_versioning(state: Versioning | None) -> bytes:
+    status = base.element("Status", state.value) if state else ""
+    return base.document("VersioningConfiguration", status)
+
+
+def parse_versioning(body: bytes) -> Versioning | None:
+    """Return the state asked for, or None if the request leaves it as it is."""
+    root = base.parse_document(body)
+    if base.optional_child_text(root, "MfaDelete") == "Enabled":
+        raise S3Error(errors.NOT_IMPLEMENTED, "MFA delete is not implemented.")
+    status = base.optional_child_text(root, "Status")
+    if status is None:
+        return None
+    try:
+        return Versioning(status)
+    except ValueError:
+        raise S3Error(errors.MALFORMED_XML, f"Bad status: {status!r}.") from None
+
+
+def render_post_result(bucket: str, key: str, etag: str, location: str) -> bytes:
+    return base.document(
+        "PostResponse",
+        base.element("Location", location),
+        base.element("Bucket", bucket),
+        base.element("Key", key),
+        base.element("ETag", f'"{etag}"'),
+    )
+
+
+def render_list_v1(
+    bucket: str,
+    query: Mapping[str, str],
+    page: ListPage,
+    max_keys: int,
+) -> bytes:
+    encode = _encoder(query)
+    marker = (
+        base.element("NextMarker", encode(page.next_marker))
+        if page.next_marker is not None and query.get("delimiter")
+        else ""
+    )
+    return base.document(
+        "ListBucketResult",
+        base.element("Name", bucket),
+        # Not encoded: botocore doesn't decode this one element in version 1.
+        base.element("Prefix", query.get("prefix", "")),
+        base.element("Marker", encode(query.get("marker", ""))),
+        marker,
+        base.element("MaxKeys", max_keys),
+        _optional_delimiter(query, encode),
+        _optional_encoding_type(query),
+        base.element("IsTruncated", _boolean(page.is_truncated)),
+        _render_page(page, encode, owner=True),
+    )
+
+
+def render_list_v2(
+    bucket: str,
+    query: Mapping[str, str],
+    page: ListPage,
+    max_keys: int,
+    next_token: str | None,
+) -> bytes:
+    encode = _encoder(query)
+    return base.document(
+        "ListBucketResult",
+        base.element("Name", bucket),
+        base.element("Prefix", encode(query.get("prefix", ""))),
+        base.element("KeyCount", len(page.objects) + len(page.common_prefixes)),
+        base.element("MaxKeys", max_keys),
+        _optional_delimiter(query, encode),
+        _optional_encoding_type(query),
+        base.element("IsTruncated", _boolean(page.is_truncated)),
+        base.optional("ContinuationToken", query.get("continuation-token")),
+        base.optional("NextContinuationToken", next_token),
+        base.optional(
+            "StartAfter",
+            encode(query["start-after"]) if "start-after" in query else None,
+        ),
+        _render_page(page, encode, owner=query.get("fetch-owner") == "true"),
+    )
+
+
+def render_versions(
+    bucket: str,
+    query: Mapping[str, str],
+    page: ListPage[VersionSummary],
+    max_keys: int,
+) -> bytes:
+    encode = _encoder(query)
+    next_marker = ""
+    if page.next_marker is not None:
+        last = page.objects[-1] if page.objects else None
+        next_version = (
+            last.version_id if last is not None and last.key == page.next_marker else ""
+        )
+        next_marker = base.element(
+            "NextKeyMarker", encode(page.next_marker)
+        ) + base.element("NextVersionIdMarker", next_version)
+    prefixes = "".join(
+        f"<CommonPrefixes>{base.element('Prefix', encode(common))}</CommonPrefixes>"
+        for common in page.common_prefixes
+    )
+    return base.document(
+        "ListVersionsResult",
+        base.element("Name", bucket),
+        base.element("Prefix", encode(query.get("prefix", ""))),
+        base.element("KeyMarker", encode(query.get("key-marker", ""))),
+        base.element("VersionIdMarker", query.get("version-id-marker", "")),
+        next_marker,
+        base.element("MaxKeys", max_keys),
+        _optional_delimiter(query, encode),
+        _optional_encoding_type(query),
+        base.element("IsTruncated", _boolean(page.is_truncated)),
+        "".join(_render_version(item, encode) for item in page.objects),
+        prefixes,
+    )
+
+
+def _render_version(item: VersionSummary, encode: Encoder) -> str:
+    common = (
+        f"{base.element('Key', encode(item.key))}"
+        f"{base.element('VersionId', item.version_id)}"
+        f"{base.element('IsLatest', _boolean(item.is_latest))}"
+        f"{base.element('LastModified', iso_date(item.modified_at))}"
+    )
+    if item.is_delete_marker:
+        return f"<DeleteMarker>{common}{base.OWNER}</DeleteMarker>"
+    return (
+        f"<Version>{common}"
+        f"{base.element('ETag', f'"{item.etag}"')}"
+        f"{base.element('Size', item.size)}"
+        f"{base.element('StorageClass', 'STANDARD')}"
+        f"{base.OWNER}</Version>"
+    )
+
+
+def render_copy_result(etag: str, modified_at: int) -> bytes:
+    return base.document(
+        "CopyObjectResult",
+        base.element("ETag", f'"{etag}"'),
+        base.element("LastModified", iso_date(modified_at)),
+    )
+
+
+def render_copy_part_result(
+    etag: str, modified_at: int, checksum: Checksum | None
+) -> bytes:
+    return base.document(
+        "CopyPartResult",
+        base.element("ETag", f'"{etag}"'),
+        base.element("LastModified", iso_date(modified_at)),
+        _render_checksum(checksum),
+    )
+
+
+def render_initiate(bucket: str, key: str, upload_id: str) -> bytes:
+    return base.document(
+        "InitiateMultipartUploadResult",
+        base.element("Bucket", bucket),
+        base.element("Key", key),
+        base.element("UploadId", upload_id),
+    )
+
+
+def render_complete(
+    bucket: str, key: str, etag: str, checksum: Checksum | None
+) -> bytes:
+    checksum_type = (
+        base.element("ChecksumType", checksum.checksum_type) if checksum else ""
+    )
+    return base.document(
+        "CompleteMultipartUploadResult",
+        base.element("Location", f"/{bucket}/{quote(key)}"),
+        base.element("Bucket", bucket),
+        base.element("Key", key),
+        base.element("ETag", f'"{etag}"'),
+        _render_checksum(checksum),
+        checksum_type,
+    )
+
+
+@dataclass(frozen=True, slots=True)
+class PartsPage:
+    parts: tuple[Part, ...]
+    max_parts: int
+    marker: int
+    next_marker: int | None
+
+
+def render_parts(upload: UploadRef, page: PartsPage) -> bytes:
+    entries = "".join(
+        "<Part>"
+        f"{base.element('PartNumber', part.number)}"
+        f"{base.element('LastModified', iso_date(part.modified_at))}"
+        f"{base.element('ETag', f'"{part.blob.md5}"')}"
+        f"{base.element('Size', part.blob.size)}"
+        f"{_render_checksum(part.checksum)}"
+        "</Part>"
+        for part in page.parts
+    )
+    return base.document(
+        "ListPartsResult",
+        base.element("Bucket", upload.bucket),
+        base.element("Key", upload.key),
+        base.element("UploadId", upload.upload_id),
+        base.element("PartNumberMarker", page.marker),
+        base.optional("NextPartNumberMarker", page.next_marker),
+        base.element("MaxParts", page.max_parts),
+        base.element("IsTruncated", _boolean(page.next_marker is not None)),
+        entries,
+    )
+
+
+def render_uploads(
+    bucket: str,
+    query: Mapping[str, str],
+    uploads: Iterable[Upload],
+    max_uploads: int,
+    next_position: tuple[str, str] | None,
+) -> bytes:
+    encode = _encoder(query)
+    entries = "".join(
+        "<Upload>"
+        f"{base.element('Key', encode(upload.key))}"
+        f"{base.element('UploadId', upload.upload_id)}"
+        f"{base.OWNER}"
+        f"{base.element('Initiated', iso_date(upload.created_at))}"
+        "</Upload>"
+        for upload in uploads
+    )
+    next_key, next_upload_id = next_position or (None, None)
+    return base.document(
+        "ListMultipartUploadsResult",
+        base.element("Bucket", bucket),
+        base.element("KeyMarker", encode(query.get("key-marker", ""))),
+        base.element("UploadIdMarker", query.get("upload-id-marker", "")),
+        base.optional("NextKeyMarker", encode(next_key) if next_key else None),
+        base.optional("NextUploadIdMarker", next_upload_id),
+        base.element("Prefix", encode(query.get("prefix", ""))),
+        base.element("MaxUploads", max_uploads),
+        _optional_encoding_type(query),
+        base.element("IsTruncated", _boolean(next_position is not None)),
+        entries,
+    )
+
+
+def render_deleted(outcomes: Iterable[DeleteOutcome], *, quiet: bool) -> bytes:
+    if quiet:
+        return base.document("DeleteResult")
+    return base.document("DeleteResult", "".join(map(_render_outcome, outcomes)))
+
+
+def _render_outcome(outcome: DeleteOutcome) -> str:
+    version = base.optional("VersionId", outcome.version_id)
+    marker = ""
+    if outcome.is_delete_marker:
+        marker = base.element("DeleteMarker", "true") + base.optional(
+            "DeleteMarkerVersionId", outcome.version_id
+        )
+        version = ""
+    return f"<Deleted>{base.element('Key', outcome.key)}{version}{marker}</Deleted>"
+
+
+def parse_delete(body: bytes) -> tuple[list[DeleteTarget], bool]:
+    """Return what to delete and whether the response should be quiet."""
+    root = base.parse_document(body)
+    keys = [
+        DeleteTarget(
+            base.child_text(item, "Key"), base.optional_child_text(item, "VersionId")
+        )
+        for item in root
+        if base.local_name(item.tag) == "Object"
+    ]
+    quiet = any(
+        base.local_name(item.tag) == "Quiet"
+        and (item.text or "").strip().lower() == "true"
+        for item in root
+    )
+    if len(keys) > MAX_DELETE_KEYS:
+        raise S3Error(errors.MALFORMED_XML, f"At most {MAX_DELETE_KEYS} keys at once.")
+    return keys, quiet
+
+
+def parse_complete(body: bytes) -> tuple[RequestedPart, ...]:
+    """Return the parts a client wants to assemble."""
+    parts = []
+    for item in base.parse_document(body):
+        if base.local_name(item.tag) != "Part":
+            continue
+        number = base.child_text(item, "PartNumber")
+        if not number.isdecimal():
+            raise S3Error(errors.MALFORMED_XML, f"Bad part number: {number!r}.")
+        etag = base.child_text(item, "ETag").strip().strip('"')
+        parts.append(RequestedPart(int(number), etag, _part_checksums(item)))
+    return tuple(parts)
+
+
+def _part_checksums(part: Element) -> tuple[Checksum, ...]:
+    return tuple(
+        Checksum(
+            base.local_name(child.tag).removeprefix(_CHECKSUM_ELEMENT_PREFIX).lower(),
+            child.text or "",
+        )
+        for child in part
+        if base.local_name(child.tag).startswith(_CHECKSUM_ELEMENT_PREFIX)
+        and base.local_name(child.tag) != _CHECKSUM_TYPE_ELEMENT
+    )
+
+
+def _encoder(query: Mapping[str, str]) -> Encoder:
+    """Honour encoding-type=url, which is what lets clients receive any key."""
+    if query.get("encoding-type") == "url":
+        return lambda value: quote(value, safe="/")
+    return lambda value: value
+
+
+def _render_page(page: ListPage, encode: Encoder, *, owner: bool) -> str:
+    return "".join((
+        *(_render_summary(item, encode, owner=owner) for item in page.objects),
+        *(
+            f"<CommonPrefixes>{base.element('Prefix', encode(common))}</CommonPrefixes>"
+            for common in page.common_prefixes
+        ),
+    ))
+
+
+def _render_summary(item: ObjectSummary, encode: Encoder, *, owner: bool) -> str:
+    return (
+        "<Contents>"
+        f"{base.element('Key', encode(item.key))}"
+        f"{base.element('LastModified', iso_date(item.modified_at))}"
+        f"{base.element('ETag', f'"{item.etag}"')}"
+        f"{base.element('Size', item.size)}"
+        f"{base.element('StorageClass', 'STANDARD')}"
+        f"{base.OWNER if owner else ''}"
+        "</Contents>"
+    )
+
+
+def _render_checksum(checksum: Checksum | None) -> str:
+    if checksum is None:
+        return ""
+    return base.element(f"Checksum{checksum.algorithm.upper()}", checksum.value)
+
+
+def _optional_delimiter(query: Mapping[str, str], encode: Encoder) -> str:
+    return base.optional(
+        "Delimiter", encode(query["delimiter"]) if query.get("delimiter") else None
+    )
+
+
+def _optional_encoding_type(query: Mapping[str, str]) -> str:
+    return base.optional("EncodingType", query.get("encoding-type"))
+
+
+def _boolean(value: bool) -> str:
+    return "true" if value else "false"

leblob/s3/xml_base.py

@@ -0,0 +1,64 @@
+"""What every S3 XML document here is built and read with. Pure."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+from xml.sax.saxutils import escape
+
+from defusedxml import DefusedXmlException, ElementTree
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from xml.etree.ElementTree import Element
+
+NAMESPACE = "http://s3.amazonaws.com/doc/2006-03-01/"
+
+
+OWNER_ID = "leblob"
+
+
+OWNER = f"<Owner><ID>{OWNER_ID}</ID><DisplayName>{OWNER_ID}</DisplayName></Owner>"
+
+
+def parse_document(body: bytes) -> Element:
+    """Parse untrusted XML, refusing DTDs and entities in any encoding."""
+    try:
+        return ElementTree.fromstring(body, forbid_dtd=True)
+    except (ElementTree.ParseError, DefusedXmlException):
+        raise S3Error(errors.MALFORMED_XML) from None
+
+
+def child_text(parent: Element, name: str) -> str:
+    text = optional_child_text(parent, name)
+    if text is None:
+        raise S3Error(errors.MALFORMED_XML, f"Missing element: {name}.")
+    return text
+
+
+def optional_child_text(parent: Element, name: str) -> str | None:
+    for child in parent:
+        if local_name(child.tag) == name:
+            return child.text
+    return None
+
+
+def local_name(tag: str) -> str:
+    return tag.rpartition("}")[2]
+
+
+def optional(tag: str, value: object | None) -> str:
+    return "" if value is None else element(tag, value)
+
+
+def element(tag: str, value: object) -> str:
+    return f"<{tag}>{escape(str(value))}</{tag}>"
+
+
+def document(root: str, *children: str, namespace: str = NAMESPACE) -> bytes:
+    attribute = f' xmlns="{namespace}"' if namespace else ""
+    body = "".join(children)
+    return (
+        f'<?xml version="1.0" encoding="UTF-8"?>\n<{root}{attribute}>{body}</{root}>'
+    ).encode()