leblob 0.4.0.dev1__py3-none-any.whl

leblob/s3/listing.py

@@ -0,0 +1,196 @@
+"""Listing logic: prefixes, delimiters and stateless pagination.
+
+The scan takes its rows from a `fetch` callable, so all of the logic here runs
+without a database. Python compares strings by code point, which is the same
+order as comparing their UTF-8 bytes, the order S3 specifies.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+from dataclasses import dataclass, replace
+from itertools import islice, takewhile
+from typing import TYPE_CHECKING, Protocol
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Iterator, Mapping
+
+    from leblob.model import VersionSummary
+
+
+class _Keyed(Protocol):
+    @property
+    def key(self) -> str: ...
+
+
+MAX_KEYS = 1000
+
+_MAX_CODE_POINT = 0x10FFFF
+_SURROGATES_START = 0xD800
+_AFTER_SURROGATES = 0xE000
+
+
+@dataclass(frozen=True, slots=True)
+class ListQuery:
+    prefix: str
+    delimiter: str
+    after: str
+    max_keys: int
+
+
+@dataclass(frozen=True, slots=True)
+class ListPage[T]:
+    objects: tuple[T, ...]
+    common_prefixes: tuple[str, ...]
+    next_marker: str | None
+
+    @property
+    def is_truncated(self) -> bool:
+        return self.next_marker is not None
+
+
+def scan[T: _Keyed](
+    fetch: Callable[[str, str | None, int], Iterator[T]],
+    query: ListQuery,
+    *,
+    resume_inside_key: bool = False,
+) -> ListPage[T]:
+    """Produce one page, skipping over each common prefix rather than walking it.
+
+    `fetch(lower, upper, limit)` yields up to `limit` rows in key order, with
+    `lower <= key`, and `key < upper` unless `upper` is None. It must be lazy: each
+    common prefix uses one row of a fetch and abandons the others.
+    """
+    prefix, delimiter, max_keys = query.prefix, query.delimiter, query.max_keys
+    if max_keys == 0:
+        return ListPage((), (), None)
+
+    lower = (
+        max(prefix, query.after)
+        if resume_inside_key
+        else _first_key_after(query.after, prefix, delimiter)
+    )
+    upper = key_after_prefix(prefix) if prefix else None
+    objects: list[T] = []
+    prefixes: list[str] = []
+    last_entry = ""
+
+    while lower is not None:
+        remaining = max_keys - len(objects) - len(prefixes)
+        rows = fetch(lower, upper, remaining + 1)
+        lower = None
+        for row in rows:
+            if len(objects) + len(prefixes) == max_keys:
+                return ListPage(tuple(objects), tuple(prefixes), last_entry)
+            common = common_prefix(row.key, prefix, delimiter)
+            if common is None:
+                objects.append(row)
+                last_entry = row.key
+                continue
+            prefixes.append(common)
+            last_entry = common
+            lower = key_after_prefix(common)
+            break
+    return ListPage(tuple(objects), tuple(prefixes), None)
+
+
+def scan_versions(
+    fetch: Callable[[str], Iterator[VersionSummary]],
+    query: ListQuery,
+    version_marker: str,
+) -> ListPage[VersionSummary]:
+    """Produce one page of versions, newest first within each key.
+
+    `fetch(lower)` yields every version from that key on, in order. A page can end
+    within a key, so `version_marker` resumes after that version of `query.after`.
+    """
+
+    def windowed(lower: str, upper: str | None, limit: int) -> Iterator[VersionSummary]:
+        rows: Iterable[VersionSummary] = fetch(lower)
+        if upper is not None:
+            rows = takewhile(lambda row: row.key < upper, rows)
+        return islice(
+            _after_version(_mark_latest(rows), query.after, version_marker), limit
+        )
+
+    return scan(windowed, query, resume_inside_key=bool(version_marker))
+
+
+def _mark_latest(rows: Iterable[VersionSummary]) -> Iterator[VersionSummary]:
+    """The first version of each key is its latest, as rows arrive newest first."""
+    previous = None
+    for row in rows:
+        yield replace(row, is_latest=row.key != previous)
+        previous = row.key
+
+
+def _after_version(
+    rows: Iterable[VersionSummary], key: str, version_id: str
+) -> Iterator[VersionSummary]:
+    passed = not version_id
+    for row in rows:
+        if row.key == key and not passed:
+            passed = row.version_id == version_id
+            continue
+        yield row
+
+
+def common_prefix(key: str, prefix: str, delimiter: str) -> str | None:
+    """Return the prefix this key rolls up into, if it does."""
+    if not delimiter:
+        return None
+    position = key.find(delimiter, len(prefix))
+    if position < 0:
+        return None
+    return key[: position + len(delimiter)]
+
+
+def key_after_prefix(prefix: str) -> str | None:
+    """Return the smallest string above all those starting with `prefix`."""
+    for index in reversed(range(len(prefix))):
+        code = ord(prefix[index])
+        if code == _MAX_CODE_POINT:
+            continue
+        code += 1
+        if _SURROGATES_START <= code < _AFTER_SURROGATES:
+            code = _AFTER_SURROGATES
+        return prefix[:index] + chr(code)
+    return None
+
+
+def parse_max_keys(query: Mapping[str, str], name: str = "max-keys") -> int:
+    value = query.get(name, str(MAX_KEYS))
+    if not value.isdecimal():
+        raise S3Error(
+            errors.INVALID_ARGUMENT, f"{name} must be a non-negative integer."
+        )
+    return min(int(value), MAX_KEYS)
+
+
+def encode_token(marker: str) -> str:
+    """Encode a position; the token holds the whole state, so any worker can resume."""
+    return base64.urlsafe_b64encode(marker.encode()).decode()
+
+
+def decode_token(token: str) -> str:
+    try:
+        # Strict: lenient decoding would turn a corrupt token into "start over".
+        return base64.b64decode(token.encode(), altchars=b"-_", validate=True).decode()
+    except (binascii.Error, UnicodeDecodeError):
+        raise S3Error(
+            errors.INVALID_ARGUMENT, "The continuation token is not valid."
+        ) from None
+
+
+def _first_key_after(after: str, prefix: str, delimiter: str) -> str | None:
+    """Return the scan's lower bound. Keys cannot contain NUL, so `x\\0` follows `x`."""
+    if not after:
+        return prefix
+    if common_prefix(after, prefix, delimiter) == after:
+        lower = key_after_prefix(after)
+        return None if lower is None else max(prefix, lower)
+    return max(prefix, f"{after}\0")

leblob/s3/multipart.py

@@ -0,0 +1,184 @@
+"""Rules for turning uploaded parts into a finished object."""
+
+from __future__ import annotations
+
+import hashlib
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from leblob.model import ChecksumPlan, ChecksumType, CompletionRequest
+from leblob.s3 import errors, payload
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Mapping, Sequence
+
+    from leblob.model import Blob, Checksum, Part, RequestedPart
+
+MIN_PART_NUMBER = 1
+MAX_PART_NUMBER = 10_000
+MIN_PART_BYTES = 5 * 1024 * 1024
+
+# Only a CRC can be combined into a whole object's. The first type is the default.
+_CHECKSUM_TYPES = {
+    "crc32": (ChecksumType.COMPOSITE, ChecksumType.FULL_OBJECT),
+    "crc32c": (ChecksumType.COMPOSITE, ChecksumType.FULL_OBJECT),
+    "crc64nvme": (ChecksumType.FULL_OBJECT,),
+    "sha1": (ChecksumType.COMPOSITE,),
+    "sha256": (ChecksumType.COMPOSITE,),
+}
+
+
+@dataclass(frozen=True, slots=True)
+class Assembly:
+    """The object a completed upload produces."""
+
+    parts: tuple[Part, ...]
+    size: int
+    etag: str
+    unused_blob_ids: tuple[str, ...]
+
+    @property
+    def blobs(self) -> tuple[Blob, ...]:
+        return tuple(part.blob for part in self.parts)
+
+
+def parse_part_number(value: str) -> int:
+    number = int(value) if value.isdecimal() else 0
+    if not MIN_PART_NUMBER <= number <= MAX_PART_NUMBER:
+        raise S3Error(
+            errors.INVALID_ARGUMENT,
+            f"Part numbers must be between {MIN_PART_NUMBER} and {MAX_PART_NUMBER}.",
+        )
+    return number
+
+
+def parse_checksum_plan(headers: Mapping[str, str]) -> ChecksumPlan | None:
+    """Read and check the checksum a new upload declares, if any."""
+    name = headers.get("x-amz-checksum-algorithm")
+    requested_type = headers.get("x-amz-checksum-type")
+    if name is None:
+        if requested_type is not None:
+            raise S3Error(
+                errors.INVALID_REQUEST, "A checksum type needs a checksum algorithm."
+            )
+        return None
+    algorithm = payload.check_algorithm_supported(name)
+    allowed = _CHECKSUM_TYPES[algorithm]
+    if requested_type is None:
+        return ChecksumPlan(algorithm, allowed[0])
+    if requested_type not in allowed:
+        raise S3Error(
+            errors.INVALID_REQUEST,
+            f"{name} does not support the checksum type {requested_type}.",
+        )
+    return ChecksumPlan(algorithm, ChecksumType(requested_type))
+
+
+def parse_completion(
+    parts: tuple[RequestedPart, ...], headers: Mapping[str, str]
+) -> CompletionRequest:
+    return CompletionRequest(
+        parts,
+        payload.parse_checksum_header(headers),
+        headers.get("x-amz-checksum-type"),
+        payload.parse_length(headers.get("x-amz-mp-object-size")),
+    )
+
+
+def assemble(
+    requested: Iterable[RequestedPart],
+    stored: Mapping[int, Part],
+    client_size: Callable[[int], int] | None = None,
+) -> Assembly:
+    """Check the client's list of parts against what was uploaded.
+
+    Sizes are as the client knows them: `client_size` turns a size on disk into that,
+    if storing the parts took more room than the bytes they hold.
+    """
+    size_of = client_size or int
+    listed = list(requested)
+    if not listed:
+        raise S3Error(errors.MALFORMED_XML, "The list of parts is empty.")
+
+    numbers = [item.number for item in listed]
+    if numbers != sorted(set(numbers)):
+        raise S3Error(errors.INVALID_PART_ORDER)
+    for item in listed:
+        part = stored.get(item.number)
+        if (
+            part is None
+            or part.blob.md5 != item.etag
+            or any(claim != part.checksum for claim in item.checksums)
+        ):
+            raise S3Error(errors.INVALID_PART, f"Part {item.number} does not match.")
+
+    parts = tuple(stored[number] for number in numbers)
+    for part in parts[:-1]:
+        if size_of(part.blob.size) < MIN_PART_BYTES:
+            raise S3Error(
+                errors.ENTITY_TOO_SMALL,
+                f"Part {part.number} is below the 5 MiB minimum.",
+            )
+
+    md5_of_md5s = hashlib.md5(usedforsecurity=False)
+    for part in parts:
+        md5_of_md5s.update(bytes.fromhex(part.blob.md5))
+    return Assembly(
+        parts,
+        sum(size_of(part.blob.size) for part in parts),
+        f"{md5_of_md5s.hexdigest()}-{len(parts)}",
+        tuple(
+            part.blob.blob_id
+            for number, part in stored.items()
+            if number not in numbers
+        ),
+    )
+
+
+def object_checksum(
+    plan: ChecksumPlan | None, parts: Sequence[Part]
+) -> Checksum | None:
+    """Work out the checksum of an object from its parts', without reading any data."""
+    if plan is None:
+        return None
+    checksums = [_planned_checksum(plan, part) for part in parts]
+    if plan.checksum_type is ChecksumType.COMPOSITE:
+        return payload.composite_checksum(plan.algorithm, checksums)
+    sizes = [part.blob.size for part in parts]
+    return payload.full_object_checksum(
+        plan.algorithm, zip(checksums, sizes, strict=True)
+    )
+
+
+def check_checksum_claims(request: CompletionRequest, actual: Checksum | None) -> None:
+    """Hold a client to what it says about the object's checksum."""
+    if request.checksum is None and request.checksum_type is None:
+        return
+    if actual is None:
+        raise S3Error(
+            errors.INVALID_REQUEST, "This upload did not declare a checksum algorithm."
+        )
+    if request.checksum_type not in {None, actual.checksum_type}:
+        raise S3Error(
+            errors.INVALID_REQUEST,
+            f"This upload has the checksum type {actual.checksum_type}.",
+        )
+    if request.checksum not in {None, actual}:
+        raise S3Error(errors.BAD_DIGEST, "The object's checksum did not match.")
+
+
+def check_size_claim(request: CompletionRequest, size: int) -> None:
+    if request.object_size not in {None, size}:
+        raise S3Error(
+            errors.INVALID_REQUEST,
+            f"The parts hold {size} bytes, not {request.object_size}.",
+        )
+
+
+def _planned_checksum(plan: ChecksumPlan, part: Part) -> Checksum:
+    if part.checksum is None or part.checksum.algorithm != plan.algorithm:
+        raise S3Error(
+            errors.INVALID_PART, f"Part {part.number} has no {plan.algorithm}."
+        )
+    return part.checksum

leblob/s3/payload.py

@@ -0,0 +1,328 @@
+"""What a client declares about an upload's body, and checking that it held."""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import hashlib
+import hmac
+import re
+import zlib
+from dataclasses import dataclass, replace
+from enum import Enum, auto
+from functools import partial
+from typing import TYPE_CHECKING, Protocol
+
+from awscrt import checksums
+
+from leblob.model import Checksum
+from leblob.s3 import errors, sigv4
+from leblob.s3.chunked import ChunkedDecoder, PlainDecoder
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Callable, Iterable, Mapping, Sequence
+
+    from leblob.s3.sigv4 import StreamSeed
+
+_CHECKSUM_PREFIX = "x-amz-checksum-"
+_MD5 = "md5"
+_SHA256 = "sha256"
+_MODE_HEADERS = frozenset({
+    "x-amz-checksum-algorithm",
+    "x-amz-checksum-mode",
+    "x-amz-checksum-type",
+})
+_HEX_SHA256 = re.compile(r"[0-9a-fA-F]{64}")
+
+
+class Hasher(Protocol):
+    def update(self, data: bytes, /) -> None: ...
+
+    def digest(self) -> bytes: ...
+
+
+class _Crc:
+    """Give a CRC function the same shape as a hashlib object."""
+
+    def __init__(self, function: Callable[[bytes, int], int], size: int) -> None:
+        self._function = function
+        self._size = size
+        self._value = 0
+
+    def update(self, data: bytes, /) -> None:
+        self._value = self._function(data, self._value)
+
+    def digest(self) -> bytes:
+        return self._value.to_bytes(self._size, "big")
+
+
+def _sha1() -> Hasher:
+    return hashlib.sha1(usedforsecurity=False)
+
+
+_CHECKSUM_FACTORIES: dict[str, Callable[[], Hasher]] = {
+    "crc32": partial(_Crc, zlib.crc32, 4),
+    "crc32c": partial(_Crc, checksums.crc32c, 4),
+    "crc64nvme": partial(_Crc, checksums.crc64nvme, 8),
+    "sha1": _sha1,
+    _SHA256: hashlib.sha256,
+}
+
+
+_CRC_COMBINERS: dict[str, tuple[Callable[[int, int, int], int], int]] = {
+    "crc32": (checksums.combine_crc32, 4),
+    "crc32c": (checksums.combine_crc32c, 4),
+    "crc64nvme": (checksums.combine_crc64nvme, 8),
+}
+
+
+class Framing(Enum):
+    PLAIN = auto()
+    SIGNED_CHUNKS = auto()
+    SIGNED_CHUNKS_WITH_TRAILER = auto()
+    UNSIGNED_CHUNKS = auto()
+
+
+_FRAMING_BY_CONTENT_SHA256 = {
+    "STREAMING-AWS4-HMAC-SHA256-PAYLOAD": Framing.SIGNED_CHUNKS,
+    "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER": Framing.SIGNED_CHUNKS_WITH_TRAILER,
+    "STREAMING-UNSIGNED-PAYLOAD-TRAILER": Framing.UNSIGNED_CHUNKS,
+}
+
+
+@dataclass(frozen=True, slots=True)
+class PayloadSpec:
+    """Everything the request headers promise about the body."""
+
+    framing: Framing
+    sha256_hex: str | None
+    md5_base64: str | None
+    checksum: Checksum | None
+    trailer_algorithm: str | None
+    declared_length: int | None
+    algorithms: frozenset[str]
+    required_algorithm: str | None = None
+
+
+def parse_spec(headers: Mapping[str, str]) -> PayloadSpec:
+    """Read the body's promised framing, length and checksums from the headers."""
+    content_sha256 = headers.get("x-amz-content-sha256", sigv4.UNSIGNED_PAYLOAD)
+    framing = _FRAMING_BY_CONTENT_SHA256.get(content_sha256, Framing.PLAIN)
+    sha256_hex = _parse_content_sha256(content_sha256, framing)
+    checksum = parse_checksum_header(headers)
+    trailer_algorithm = _parse_trailer_algorithm(headers)
+
+    algorithms = {_MD5}
+    if sha256_hex:
+        algorithms.add(_SHA256)
+    if checksum:
+        algorithms.add(checksum.algorithm)
+    if trailer_algorithm:
+        algorithms.add(trailer_algorithm)
+
+    length_header = (
+        "content-length" if framing is Framing.PLAIN else "x-amz-decoded-content-length"
+    )
+    return PayloadSpec(
+        framing,
+        sha256_hex,
+        headers.get("content-md5"),
+        checksum,
+        trailer_algorithm,
+        parse_length(headers.get(length_header)),
+        frozenset(algorithms),
+    )
+
+
+def requiring(spec: PayloadSpec, algorithm: str) -> PayloadSpec:
+    """Have a body end up with this checksum, as its multipart upload declared."""
+    sent = spec.trailer_algorithm or (spec.checksum and spec.checksum.algorithm)
+    if sent and sent != algorithm:
+        raise S3Error(
+            errors.INVALID_REQUEST,
+            f"This upload declared {algorithm}, but the part has {sent}.",
+        )
+    return replace(
+        spec, algorithms=spec.algorithms | {algorithm}, required_algorithm=algorithm
+    )
+
+
+def check_algorithm_supported(name: str) -> str:
+    """Return a lower-case checksum algorithm name, or refuse it."""
+    algorithm = name.lower()
+    if algorithm not in _CHECKSUM_FACTORIES:
+        raise S3Error(
+            errors.NOT_IMPLEMENTED, f"Checksum algorithm {name} is not implemented."
+        )
+    return algorithm
+
+
+def without_object_checksums(headers: Mapping[str, str]) -> dict[str, str]:
+    """Drop checksum headers that describe an object rather than this request's body.
+
+    CompleteMultipartUpload carries them, next to an XML body they don't apply to.
+    """
+    return {
+        name: value
+        for name, value in headers.items()
+        if not name.startswith(_CHECKSUM_PREFIX)
+    }
+
+
+def new_decoder(
+    spec: PayloadSpec, seed: StreamSeed | None
+) -> PlainDecoder | ChunkedDecoder:
+    """Choose how to read a body. Chunks signed with no signature to chain from fail."""
+    if seed is None and spec.framing in _SIGNED_FRAMINGS:
+        raise S3Error(errors.ACCESS_DENIED, "Signed chunks need a signed request.")
+    match spec.framing:
+        case Framing.PLAIN:
+            return PlainDecoder()
+        case Framing.UNSIGNED_CHUNKS:
+            return ChunkedDecoder(None)
+        case Framing.SIGNED_CHUNKS:
+            return ChunkedDecoder(seed)
+        case Framing.SIGNED_CHUNKS_WITH_TRAILER:
+            return ChunkedDecoder(seed, signed_trailer=True)
+
+
+_SIGNED_FRAMINGS = frozenset({
+    Framing.SIGNED_CHUNKS,
+    Framing.SIGNED_CHUNKS_WITH_TRAILER,
+})
+
+
+def new_hashers(spec: PayloadSpec) -> dict[str, Hasher]:
+    return {name: _new_hasher(name) for name in spec.algorithms}
+
+
+def verify(
+    spec: PayloadSpec,
+    digests: Mapping[str, bytes],
+    size: int,
+    trailers: Mapping[str, str],
+) -> Checksum | None:
+    """Compare the received body with every promise; return its checksum to store."""
+    if spec.declared_length is not None and size != spec.declared_length:
+        raise S3Error(
+            errors.INCOMPLETE_BODY,
+            f"Received {size} bytes, but {spec.declared_length} were declared.",
+        )
+    if spec.sha256_hex is not None and not hmac.compare_digest(
+        digests[_SHA256].hex(), spec.sha256_hex
+    ):
+        raise S3Error(errors.CONTENT_SHA256_MISMATCH)
+    if spec.md5_base64 is not None:
+        _check_content_md5(digests[_MD5], spec.md5_base64)
+
+    checksum = spec.checksum
+    if spec.trailer_algorithm is not None:
+        checksum = _read_trailer_checksum(spec.trailer_algorithm, trailers)
+    if checksum is not None:
+        _check_checksum(digests[checksum.algorithm], checksum)
+        return checksum
+    if spec.required_algorithm is None:
+        return None
+    return Checksum(
+        spec.required_algorithm, _encode_base64(digests[spec.required_algorithm])
+    )
+
+
+def composite_checksum(algorithm: str, parts: Sequence[Checksum]) -> Checksum:
+    """Hash the parts' checksums, in order, and say how many there were."""
+    hasher = _new_hasher(algorithm)
+    for part in parts:
+        hasher.update(base64.b64decode(part.value))
+    return Checksum(algorithm, f"{_encode_base64(hasher.digest())}-{len(parts)}")
+
+
+def full_object_checksum(
+    algorithm: str, parts: Iterable[tuple[Checksum, int]]
+) -> Checksum:
+    """Combine the parts' CRCs into the one the whole object has, reading no data."""
+    combine, size = _CRC_COMBINERS[algorithm]
+    crc = 0
+    for part, length in parts:
+        part_crc = int.from_bytes(base64.b64decode(part.value), "big")
+        crc = combine(crc, part_crc, length)
+    return Checksum(algorithm, _encode_base64(crc.to_bytes(size, "big")))
+
+
+def _new_hasher(name: str) -> Hasher:
+    if name == _MD5:
+        return hashlib.md5(usedforsecurity=False)
+    return _CHECKSUM_FACTORIES[name]()
+
+
+def _parse_content_sha256(value: str, framing: Framing) -> str | None:
+    if framing is not Framing.PLAIN or value == sigv4.UNSIGNED_PAYLOAD:
+        return None
+    if value.startswith("STREAMING-"):
+        raise S3Error(errors.NOT_IMPLEMENTED, f"Payload mode {value} is not supported.")
+    if not _HEX_SHA256.fullmatch(value):
+        raise S3Error(errors.INVALID_ARGUMENT, "x-amz-content-sha256 is not valid.")
+    return value.lower()
+
+
+def parse_checksum_header(headers: Mapping[str, str]) -> Checksum | None:
+    names = [
+        name
+        for name in headers
+        if name.startswith(_CHECKSUM_PREFIX) and name not in _MODE_HEADERS
+    ]
+    if not names:
+        return None
+    if len(names) > 1:
+        raise S3Error(errors.INVALID_REQUEST, "Send at most one checksum header.")
+    algorithm = check_algorithm_supported(names[0].removeprefix(_CHECKSUM_PREFIX))
+    return Checksum(algorithm, headers[names[0]])
+
+
+def _parse_trailer_algorithm(headers: Mapping[str, str]) -> str | None:
+    trailer = headers.get("x-amz-trailer")
+    if trailer is None:
+        return None
+    if not trailer.startswith(_CHECKSUM_PREFIX):
+        raise S3Error(errors.INVALID_REQUEST, f"Unsupported trailer: {trailer}.")
+    return check_algorithm_supported(trailer.removeprefix(_CHECKSUM_PREFIX))
+
+
+def parse_length(value: str | None) -> int | None:
+    if value is None:
+        return None
+    if not value.isdecimal():
+        raise S3Error(errors.INVALID_ARGUMENT, f"Invalid body length: {value!r}.")
+    return int(value)
+
+
+def _read_trailer_checksum(algorithm: str, trailers: Mapping[str, str]) -> Checksum:
+    value = trailers.get(f"{_CHECKSUM_PREFIX}{algorithm}")
+    if value is None:
+        raise S3Error(errors.MALFORMED_TRAILER)
+    return Checksum(algorithm, value)
+
+
+def _check_content_md5(digest: bytes, claimed: str) -> None:
+    decoded = _decode_base64(claimed)
+    if decoded is None:
+        raise S3Error(errors.INVALID_DIGEST, "Content-MD5 is not base64.")
+    if not hmac.compare_digest(digest, decoded):
+        raise S3Error(errors.BAD_DIGEST, "The Content-MD5 did not match.")
+
+
+def _check_checksum(digest: bytes, checksum: Checksum) -> None:
+    decoded = _decode_base64(checksum.value)
+    if decoded is None or not hmac.compare_digest(digest, decoded):
+        raise S3Error(errors.BAD_DIGEST, f"The {checksum.algorithm} did not match.")
+
+
+def _encode_base64(digest: bytes) -> str:
+    return base64.b64encode(digest).decode()
+
+
+def _decode_base64(value: str) -> bytes | None:
+    try:
+        return base64.b64decode(value, validate=True)
+    except binascii.Error:
+        return None