leblob 0.4.0.dev1__py3-none-any.whl

leblob/s3/sigv4.py

@@ -0,0 +1,333 @@
+"""AWS Signature Version 4: parsing and verifying signed requests."""
+
+from __future__ import annotations
+
+import hashlib
+import hmac
+import re
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING
+from urllib.parse import quote, unquote_to_bytes
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from leblob.model import HttpRequest
+
+ALGORITHM = "AWS4-HMAC-SHA256"
+UNSIGNED_PAYLOAD = "UNSIGNED-PAYLOAD"
+EMPTY_SHA256 = hashlib.sha256(b"").hexdigest()
+
+MAX_CLOCK_SKEW_SECONDS = 15 * 60
+MAX_PRESIGN_SECONDS = 7 * 24 * 3600
+
+_AMZ_DATE_FORMAT = "%Y%m%dT%H%M%SZ"
+_AUTHORIZATION = re.compile(
+    rf"{ALGORITHM} +Credential=([^,\s]+), *SignedHeaders=([^,\s]+), *"
+    r"Signature=([0-9a-f]{64})"
+)
+_SIGNATURE = re.compile(r"[0-9a-f]{64}")
+_CREDENTIAL_PARTS = 5
+_PRESIGNED_PARAMETERS = frozenset({
+    "X-Amz-Algorithm",
+    "X-Amz-Credential",
+    "X-Amz-Date",
+    "X-Amz-Expires",
+    "X-Amz-Signature",
+    "X-Amz-SignedHeaders",
+})
+
+
+@dataclass(frozen=True, slots=True)
+class Credential:
+    access_key: str
+    date: str
+    region: str
+    service: str
+
+    @property
+    def scope(self) -> str:
+        return f"{self.date}/{self.region}/{self.service}/aws4_request"
+
+
+@dataclass(frozen=True, slots=True)
+class Claim:
+    """What a request says about its own signature; nothing here is verified yet."""
+
+    credential: Credential
+    signed_headers: tuple[str, ...]
+    signature: str
+    amz_date: str
+    expires: int | None
+
+
+@dataclass(frozen=True, slots=True)
+class StreamSeed:
+    """A verified signature's state, from which chunk signatures are chained."""
+
+    signing_key: bytes
+    scope: str
+    amz_date: str
+    signature: str
+
+
+def is_anonymous(request: HttpRequest) -> bool:
+    """Whether a request claims no signature at all. A wrong one isn't anonymous."""
+    return (
+        "X-Amz-Algorithm" not in request.query
+        and "authorization" not in request.headers
+    )
+
+
+def post_form_access_key(fields: Mapping[str, str]) -> str:
+    """Name the key a browser form claims, if signed in a way this server reads."""
+    if "awsaccesskeyid" in fields:
+        raise S3Error(errors.NOT_IMPLEMENTED, "Forms must be signed with SigV4.")
+    if "policy" not in fields:
+        raise S3Error(errors.ACCESS_DENIED, "Anonymous uploads are not allowed.")
+    if "x-amz-credential" not in fields or "x-amz-signature" not in fields:
+        raise S3Error(errors.INVALID_ARGUMENT, "A form with a policy must be signed.")
+    credential = _parse_credential(fields["x-amz-credential"], errors.INVALID_ARGUMENT)
+    return credential.access_key
+
+
+def verify_post_form(fields: Mapping[str, str], secret: str) -> None:
+    """Check a form's signature, which covers only its policy, as encoded."""
+    credential = _parse_credential(fields["x-amz-credential"], errors.INVALID_ARGUMENT)
+    if fields.get("x-amz-algorithm") != ALGORITHM:
+        raise S3Error(errors.INVALID_ARGUMENT, "Forms must be signed with SigV4.")
+    amz_date = fields.get("x-amz-date", "")
+    try:
+        datetime.strptime(amz_date, _AMZ_DATE_FORMAT).replace(tzinfo=UTC)
+    except ValueError:
+        raise S3Error(errors.INVALID_ARGUMENT, "x-amz-date isn't a date.") from None
+    if not amz_date.startswith(credential.date):
+        raise S3Error(errors.SIGNATURE_MISMATCH, "Credential date is not form date.")
+    signature = fields.get("x-amz-signature", "")
+    if not is_signature(signature):
+        raise S3Error(errors.ACCESS_DENIED, "The form isn't signed.")
+    key = derive_signing_key(secret, credential)
+    expected = _hmac(key, fields["policy"]).hex()
+    if not hmac.compare_digest(expected, signature):
+        raise S3Error(errors.SIGNATURE_MISMATCH)
+
+
+def parse_claim(request: HttpRequest) -> Claim:
+    """Extract the claimed signature from the headers or a presigned URL."""
+    if "X-Amz-Algorithm" in request.query:
+        return _parse_presigned(request.query)
+    authorization = request.headers.get("authorization")
+    if authorization is None:
+        raise S3Error(errors.ACCESS_DENIED, "Anonymous access is not allowed.")
+    return _parse_authorization(authorization, request.headers)
+
+
+def is_signature(text: str) -> bool:
+    """Check the form of a signature, which `compare_digest` requires to be ASCII."""
+    return _SIGNATURE.fullmatch(text) is not None
+
+
+def verify(request: HttpRequest, claim: Claim, secret: str, now: float) -> StreamSeed:
+    """Check a claim against the secret key, or raise the matching S3 error."""
+    _check_time(claim, now)
+    _check_signed_headers(request.headers, claim.signed_headers)
+
+    signing_key = derive_signing_key(secret, claim.credential)
+    string_to_sign = "\n".join((
+        ALGORITHM,
+        claim.amz_date,
+        claim.credential.scope,
+        # Latin-1 restores the header bytes exactly as sent. Everything else in
+        # the canonical request is percent-encoded, so it is plain ASCII.
+        _sha256_hex(canonical_request(request, claim).encode("latin-1")),
+    ))
+    expected = _hmac(signing_key, string_to_sign).hex()
+    if not hmac.compare_digest(expected, claim.signature):
+        raise S3Error(errors.SIGNATURE_MISMATCH)
+    return StreamSeed(signing_key, claim.credential.scope, claim.amz_date, expected)
+
+
+def canonical_request(request: HttpRequest, claim: Claim) -> str:
+    return "\n".join((
+        request.method,
+        canonical_uri(request.raw_path),
+        canonical_query(request.raw_query),
+        "".join(
+            f"{name}:{_normalize_value(request.headers.get(name, ''))}\n"
+            for name in claim.signed_headers
+        ),
+        ";".join(claim.signed_headers),
+        request.headers.get("x-amz-content-sha256", UNSIGNED_PAYLOAD),
+    ))
+
+
+def canonical_uri(raw_path: bytes) -> str:
+    """Encode each path segment the way AWS specifies, keeping slashes literal."""
+    return "/".join(
+        quote(unquote_to_bytes(segment), safe="-_.~")
+        for segment in raw_path.split(b"/")
+    )
+
+
+def canonical_query(raw_query: bytes) -> str:
+    """Sort and normalize the query string, without the signature itself."""
+    pairs = []
+    for item in raw_query.split(b"&"):
+        if not item:
+            continue
+        name, _, value = item.partition(b"=")
+        encoded_name = quote(unquote_to_bytes(name), safe="-_.~")
+        if encoded_name != "X-Amz-Signature":
+            pairs.append((encoded_name, quote(unquote_to_bytes(value), safe="-_.~")))
+    return "&".join(f"{name}={value}" for name, value in sorted(pairs))
+
+
+def derive_signing_key(secret: str, credential: Credential) -> bytes:
+    key = _hmac(f"AWS4{secret}".encode(), credential.date)
+    key = _hmac(key, credential.region)
+    key = _hmac(key, credential.service)
+    return _hmac(key, "aws4_request")
+
+
+def chunk_signature(seed: StreamSeed, previous: str, chunk_sha256: str) -> str:
+    """Signature of one aws-chunked data chunk, chained from the previous one."""
+    return _hmac(
+        seed.signing_key,
+        "\n".join((
+            f"{ALGORITHM}-PAYLOAD",
+            seed.amz_date,
+            seed.scope,
+            previous,
+            EMPTY_SHA256,
+            chunk_sha256,
+        )),
+    ).hex()
+
+
+def trailer_signature(seed: StreamSeed, previous: str, trailer: bytes) -> str:
+    """Signature over the canonical trailing headers of a chunked upload."""
+    return _hmac(
+        seed.signing_key,
+        "\n".join((
+            f"{ALGORITHM}-TRAILER",
+            seed.amz_date,
+            seed.scope,
+            previous,
+            _sha256_hex(trailer),
+        )),
+    ).hex()
+
+
+def _parse_authorization(authorization: str, headers: Mapping[str, str]) -> Claim:
+    match = _AUTHORIZATION.fullmatch(authorization.strip())
+    if match is None:
+        raise S3Error(
+            errors.AUTH_HEADER_MALFORMED,
+            f"Only {ALGORITHM} authorization is supported, in its standard form.",
+        )
+    amz_date = headers.get("x-amz-date")
+    if amz_date is None:
+        raise S3Error(errors.ACCESS_DENIED, "The x-amz-date header is required.")
+    credential_text, signed_headers, signature = match.groups()
+    credential = _parse_credential(credential_text, errors.AUTH_HEADER_MALFORMED)
+    if "x-amz-content-sha256" not in headers:
+        # Without it the signature would say nothing about the body we go on to store.
+        raise S3Error(
+            errors.INVALID_REQUEST, "The x-amz-content-sha256 header is required."
+        )
+    return Claim(
+        credential,
+        tuple(signed_headers.split(";")),
+        signature,
+        amz_date,
+        None,
+    )
+
+
+def _parse_presigned(query: Mapping[str, str]) -> Claim:
+    if not query.keys() >= _PRESIGNED_PARAMETERS:
+        raise S3Error(errors.AUTH_QUERY_ERROR)
+    expires = query["X-Amz-Expires"]
+    if query["X-Amz-Algorithm"] != ALGORITHM or not is_signature(
+        query["X-Amz-Signature"]
+    ):
+        raise S3Error(errors.AUTH_QUERY_ERROR)
+    if not (expires.isdecimal() and 1 <= int(expires) <= MAX_PRESIGN_SECONDS):
+        raise S3Error(errors.ACCESS_DENIED, "X-Amz-Expires is out of range.")
+    return Claim(
+        _parse_credential(query["X-Amz-Credential"], errors.AUTH_QUERY_ERROR),
+        tuple(query["X-Amz-SignedHeaders"].split(";")),
+        query["X-Amz-Signature"],
+        query["X-Amz-Date"],
+        int(expires),
+    )
+
+
+def _parse_credential(text: str, malformed: errors.ErrorCode) -> Credential:
+    parts = text.split("/")
+    if (
+        len(parts) != _CREDENTIAL_PARTS
+        or parts[3] != "s3"
+        or parts[4] != "aws4_request"
+    ):
+        raise S3Error(malformed, f"Malformed credential scope: {text!r}.")
+    return Credential(*parts[:4])
+
+
+def _check_time(claim: Claim, now: float) -> None:
+    try:
+        signed_at = (
+            datetime
+            .strptime(claim.amz_date, _AMZ_DATE_FORMAT)
+            .replace(tzinfo=UTC)
+            .timestamp()
+        )
+    except ValueError:
+        raise S3Error(
+            errors.ACCESS_DENIED, f"Invalid request date: {claim.amz_date!r}."
+        ) from None
+    if not claim.amz_date.startswith(claim.credential.date):
+        raise S3Error(errors.SIGNATURE_MISMATCH, "Credential date is not request date.")
+
+    if signed_at - now > MAX_CLOCK_SKEW_SECONDS:
+        raise S3Error(errors.TIME_TOO_SKEWED)
+    if claim.expires is None:
+        if now - signed_at > MAX_CLOCK_SKEW_SECONDS:
+            raise S3Error(errors.TIME_TOO_SKEWED)
+    elif now - signed_at > claim.expires:
+        raise S3Error(errors.ACCESS_DENIED, "Request has expired.")
+
+
+def _check_signed_headers(
+    headers: Mapping[str, str], signed_headers: tuple[str, ...]
+) -> None:
+    """Refuse requests whose security-relevant headers escape the signature.
+
+    Otherwise anyone holding a presigned PUT URL could add `x-amz-copy-source`
+    and turn the upload into a read of any object.
+    """
+    if "host" not in signed_headers:
+        raise S3Error(errors.ACCESS_DENIED, "The Host header must be signed.")
+    unsigned = sorted(
+        name
+        for name in headers
+        if name.startswith("x-amz-") and name not in signed_headers
+    )
+    if unsigned:
+        raise S3Error(errors.ACCESS_DENIED, f"Header {unsigned[0]} must be signed.")
+
+
+def _normalize_value(value: str) -> str:
+    return " ".join(value.split())
+
+
+def _sha256_hex(data: bytes) -> str:
+    return hashlib.sha256(data).hexdigest()
+
+
+def _hmac(key: bytes, message: str) -> bytes:
+    return hmac.digest(key, message.encode(), "sha256")

leblob/s3/sse.py

@@ -0,0 +1,229 @@
+"""Server-side encryption with a key the client sends each time (SSE-C).
+
+An encrypted blob is a random salt, then chunks of AES-256-GCM. Each blob has its own
+key, derived from the client's and the salt, so a chunk's number can serve as its nonce.
+A chunk's number and whether it is the last are authenticated, so chunks can't be moved
+or dropped. Reading a range needs only the chunks it touches.
+
+The stored bytes are just bytes to everything below this: scrub and repair check them
+against the MD5 of what was stored. The client's key is never stored, only a value that
+shows whether a later request has sent the same one.
+"""
+
+from __future__ import annotations
+
+import base64
+import hashlib
+import hmac
+import secrets
+from dataclasses import dataclass
+from typing import TYPE_CHECKING
+
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+ALGORITHM = "AES256"
+HEADER_PREFIX = "x-amz-server-side-encryption-customer-"
+COPY_SOURCE_PREFIX = "x-amz-copy-source-server-side-encryption-customer-"
+CHUNK_BYTES = 64 * 1024
+SALT_BYTES = 16
+_TAG_BYTES = 16
+_KEY_BYTES = 32
+_NONCE_BYTES = 12
+_CHECK_LABEL = b"leblob sse-c key check"
+_LAST = b"\x01"
+_NOT_LAST = b"\x00"
+
+
+@dataclass(frozen=True, slots=True, repr=False)
+class CustomerKey:
+    """A key a client sent. Not printed, so it can't end up in a log by accident."""
+
+    key: bytes
+    key_md5: str
+
+    def __repr__(self) -> str:
+        return "CustomerKey(...)"
+
+    @property
+    def check(self) -> str:
+        """What is stored, to tell later whether the same key was sent."""
+        return hmac.new(self.key, _CHECK_LABEL, hashlib.sha256).hexdigest()
+
+    def response_headers(self) -> tuple[tuple[str, str], ...]:
+        return (
+            (f"{HEADER_PREFIX}algorithm", ALGORITHM),
+            (f"{HEADER_PREFIX}key-md5", self.key_md5),
+        )
+
+
+def parse_key(
+    headers: Mapping[str, str], prefix: str = HEADER_PREFIX
+) -> CustomerKey | None:
+    """Read a key from the headers, or None if they don't try to give one."""
+    algorithm = headers.get(f"{prefix}algorithm")
+    encoded = headers.get(f"{prefix}key")
+    md5 = headers.get(f"{prefix}key-md5")
+    if algorithm is None and encoded is None and md5 is None:
+        return None
+    if algorithm != ALGORITHM:
+        raise S3Error(
+            errors.INVALID_ARGUMENT, "The encryption algorithm must be AES256."
+        )
+    if encoded is None or md5 is None:
+        raise S3Error(errors.INVALID_ARGUMENT, "Encryption needs a key and its MD5.")
+    try:
+        key = base64.b64decode(encoded, validate=True)
+    except ValueError:  # `binascii.Error` is one, and so is what non-ASCII raises.
+        raise S3Error(
+            errors.INVALID_ARGUMENT, "The encryption key isn't valid."
+        ) from None
+    if len(key) != _KEY_BYTES:
+        raise S3Error(errors.INVALID_ARGUMENT, "The encryption key must have 256 bits.")
+    actual = base64.b64encode(hashlib.md5(key, usedforsecurity=False).digest()).decode()
+    if not hmac.compare_digest(actual.encode(), md5.encode()):
+        raise S3Error(
+            errors.INVALID_ARGUMENT, "The encryption key's MD5 doesn't match."
+        )
+    return CustomerKey(key, md5)
+
+
+def check_key(stored_check: str | None, key: CustomerKey | None) -> None:
+    """Hold a request to what was stored: the same key, or none if none was used."""
+    if stored_check is None:
+        if key is not None:
+            raise S3Error(errors.INVALID_REQUEST, "This object isn't encrypted.")
+        return
+    if key is None:
+        raise S3Error(errors.INVALID_REQUEST, "This object needs its encryption key.")
+    if not hmac.compare_digest(stored_check, key.check):
+        raise S3Error(
+            errors.INVALID_REQUEST, "That isn't this object's encryption key."
+        )
+
+
+def stored_size(plain_size: int) -> int:
+    chunks = max(1, (plain_size + CHUNK_BYTES - 1) // CHUNK_BYTES)
+    return SALT_BYTES + plain_size + chunks * _TAG_BYTES
+
+
+def plain_size(stored: int) -> int:
+    body = stored - SALT_BYTES
+    full = CHUNK_BYTES + _TAG_BYTES
+    chunks, rest = divmod(body, full)
+    return chunks * CHUNK_BYTES + max(0, rest - _TAG_BYTES)
+
+
+def _blob_cipher(key: CustomerKey, salt: bytes) -> AESGCM:
+    return AESGCM(hmac.digest(key.key, salt, "sha256"))
+
+
+def _nonce(number: int) -> bytes:
+    return number.to_bytes(_NONCE_BYTES)
+
+
+def _authenticated(number: int, *, last: bool) -> bytes:
+    return number.to_bytes(8) + (_LAST if last else _NOT_LAST)
+
+
+class Encryptor:
+    """Turn a stream of plain bytes into what an encrypted blob holds."""
+
+    def __init__(self, key: CustomerKey) -> None:
+        salt = secrets.token_bytes(SALT_BYTES)
+        self._cipher = _blob_cipher(key, salt)
+        self._pending = bytearray(salt)
+        self._plain = bytearray()
+        self._number = 0
+
+    def _seal(self, data: bytes, *, last: bool) -> bytes:
+        sealed = self._cipher.encrypt(
+            _nonce(self._number), data, _authenticated(self._number, last=last)
+        )
+        self._number += 1
+        return sealed
+
+    def feed(self, data: bytes) -> bytes:
+        self._plain += data
+        out = self._pending
+        self._pending = bytearray()
+        # Keep at least one byte back: only the last chunk may be marked as last.
+        while len(self._plain) > CHUNK_BYTES:
+            out += self._seal(bytes(self._plain[:CHUNK_BYTES]), last=False)
+            del self._plain[:CHUNK_BYTES]
+        return bytes(out)
+
+    def finish(self) -> bytes:
+        return bytes(self._pending) + self._seal(bytes(self._plain), last=True)
+
+
+@dataclass(frozen=True, slots=True)
+class ReadPlan:
+    """Which stored bytes a plain range of one blob needs, and what to trim."""
+
+    stored_offset: int
+    stored_length: int
+    first_chunk: int
+    skip: int
+    length: int
+
+
+def plan_read(stored: int, start: int, length: int) -> ReadPlan:
+    full = CHUNK_BYTES + _TAG_BYTES
+    first = start // CHUNK_BYTES
+    last = (start + length - 1) // CHUNK_BYTES
+    offset = SALT_BYTES + first * full
+    end = min(SALT_BYTES + (last + 1) * full, stored)
+    return ReadPlan(offset, end - offset, first, start - first * CHUNK_BYTES, length)
+
+
+class Decryptor:
+    """Turn the stored chunks of a range back into plain bytes, checking each."""
+
+    def __init__(
+        self, key: CustomerKey, salt: bytes, stored: int, plan: ReadPlan
+    ) -> None:
+        self._cipher = _blob_cipher(key, salt)
+        self._buffer = bytearray()
+        self._number = plan.first_chunk
+        self._skip = plan.skip
+        self._left = plan.length
+        full = CHUNK_BYTES + _TAG_BYTES
+        self._last_number = max(0, (stored - SALT_BYTES - 1) // full)
+
+    def _open(self, chunk: bytes) -> bytes:
+        try:
+            plain = self._cipher.decrypt(
+                _nonce(self._number),
+                chunk,
+                _authenticated(self._number, last=self._number == self._last_number),
+            )
+        except InvalidTag:
+            message = "An encrypted chunk failed its check."
+            raise OSError(message) from None
+        self._number += 1
+        return plain
+
+    def _trimmed(self, plain: bytes) -> bytes:
+        kept = plain[self._skip : self._skip + self._left]
+        self._skip = max(0, self._skip - len(plain))
+        self._left -= len(kept)
+        return kept
+
+    def feed(self, data: bytes) -> bytes:
+        self._buffer += data
+        full = CHUNK_BYTES + _TAG_BYTES
+        out = bytearray()
+        while len(self._buffer) >= full:
+            out += self._trimmed(self._open(bytes(self._buffer[:full])))
+            del self._buffer[:full]
+        return bytes(out)
+
+    def finish(self) -> bytes:
+        return self._trimmed(self._open(bytes(self._buffer))) if self._buffer else b""

leblob/s3/tagging.py

@@ -0,0 +1,48 @@
+"""Object tags: what S3 allows in them, and their form in a request header."""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+from urllib.parse import parse_qsl
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+MAX_TAGS = 10
+MAX_KEY_CHARACTERS = 128
+MAX_VALUE_CHARACTERS = 256
+# As AWS allows. This also keeps out what XML can't carry back to the client.
+_ALLOWED = re.compile(r"[\w +\-=.:/@]*")
+
+
+def build_tags(pairs: Iterable[tuple[str, str]]) -> dict[str, str]:
+    """Return valid tags, ordered by key, or raise InvalidTag."""
+    listed = list(pairs)
+    tags = dict(sorted(listed))
+    if len(tags) < len(listed):
+        raise S3Error(errors.INVALID_TAG, "Tag keys must be unique.")
+    if len(tags) > MAX_TAGS:
+        raise S3Error(
+            errors.INVALID_TAG, f"An object can have {MAX_TAGS} tags at most."
+        )
+    for key, value in tags.items():
+        _check_text(key, value)
+    return tags
+
+
+def parse_header(value: str) -> dict[str, str]:
+    """Read the `x-amz-tagging` header, which is written like a query string."""
+    return build_tags(parse_qsl(value, keep_blank_values=True))
+
+
+def _check_text(key: str, value: str) -> None:
+    if not 1 <= len(key) <= MAX_KEY_CHARACTERS:
+        raise S3Error(errors.INVALID_TAG, "A tag key has 1 to 128 characters.")
+    if len(value) > MAX_VALUE_CHARACTERS:
+        raise S3Error(errors.INVALID_TAG, "A tag value has 256 characters at most.")
+    if not (_ALLOWED.fullmatch(key) and _ALLOWED.fullmatch(value)):
+        raise S3Error(errors.INVALID_TAG, "A tag holds a character S3 doesn't allow.")