leblob 0.4.0.dev1__py3-none-any.whl

leblob/s3/post_form.py

@@ -0,0 +1,344 @@
+"""Browser form uploads: reading `multipart/form-data`, and judging its policy.
+
+The form can hold gigabytes, so `FormParser` takes it a chunk at a time, and is pure.
+A body that ends before its closing delimiter never reports the end of its file, which
+is what stops a cut-off upload from becoming an object.
+"""
+
+from __future__ import annotations
+
+import base64
+import binascii
+import json
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from enum import Enum, auto
+from typing import TYPE_CHECKING
+
+from leblob.s3 import errors
+from leblob.s3.errors import S3Error
+
+if TYPE_CHECKING:
+    from collections.abc import Iterator, Mapping
+
+MAX_FIELDS_BYTES = 20 * 1024  # As AWS limits what comes before the file.
+_MAX_BOUNDARY_BYTES = 200
+_HEADERS_END = b"\r\n\r\n"
+_CLOSING = b"--"
+_LINE_END = b"\r\n"
+_BOUNDARY = re.compile(r'boundary=(?:"([^"]+)"|([^\s;]+))', re.IGNORECASE)
+_DISPOSITION_NAME = re.compile(r'[;\s]name="((?:[^"\\]|\\.)*)"', re.IGNORECASE)
+_DISPOSITION_FILENAME = re.compile(r'[;\s]filename="((?:[^"\\]|\\.)*)"', re.IGNORECASE)
+_ESCAPED = re.compile(r"\\(.)")
+_EXPIRATION = re.compile(
+    r"[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(\.[0-9]+)?Z"
+)
+FILE_FIELD = "file"
+_NOT_FOR_THE_POLICY = frozenset({"file", "policy", "x-amz-signature"})
+# A checksum can only make a form stricter, so it needs no one's leave.
+_EXEMPT_PREFIXES = ("x-ignore-", "x-amz-checksum-")
+_LENGTH_CONDITION = "content-length-range"
+_EQUALS_ARGUMENTS = 3
+_REFUSAL = "Invalid according to Policy: "
+
+
+@dataclass(frozen=True, slots=True)
+class Field:
+    name: str
+    value: str
+
+
+@dataclass(frozen=True, slots=True)
+class FileStart:
+    filename: str
+
+
+@dataclass(frozen=True, slots=True)
+class FileData:
+    data: bytes
+
+
+@dataclass(frozen=True, slots=True)
+class FileEnd:
+    pass
+
+
+type Event = Field | FileStart | FileData | FileEnd
+
+
+@dataclass(frozen=True, slots=True)
+class SizeLimits:
+    smallest: int
+    largest: int | None
+
+
+class _State(Enum):
+    BEFORE_FIRST = auto()
+    AFTER_DELIMITER = auto()
+    HEADERS = auto()
+    VALUE = auto()
+    FILE = auto()
+    SKIPPING = auto()
+    DONE = auto()
+
+
+def parse_boundary(content_type: str) -> bytes:
+    found = _BOUNDARY.search(content_type)
+    boundary = (found[1] or found[2]) if found else ""
+    if not boundary or len(boundary) > _MAX_BOUNDARY_BYTES:
+        raise S3Error(errors.MALFORMED_POST, "The form has no usable boundary.")
+    return boundary.encode("latin-1")
+
+
+class FormParser:
+    """Turn the chunks of a form into events. Only the first file is reported.
+
+    S3 ignores anything after the file, so such parts are skipped without keeping them.
+    """
+
+    def __init__(self, boundary: bytes) -> None:
+        self._delimiter = b"\r\n--" + boundary
+        # So that the first delimiter, which has no line end of its own, looks the same.
+        self._buffer = bytearray(_LINE_END)
+        self._state = _State.BEFORE_FIRST
+        self._seen_file = False
+        self._field_name = ""
+        self._field_bytes = 0
+
+    def feed(self, chunk: bytes) -> Iterator[Event]:
+        self._buffer += chunk
+        progress = True
+        while progress:
+            progress = False
+            for event in self._step():
+                progress = True
+                if event is not None:
+                    yield event
+        self._check_size()
+
+    def finish(self) -> None:
+        if self._state is not _State.DONE:
+            raise S3Error(errors.MALFORMED_POST, "The form ends before it should.")
+
+    def _check_size(self) -> None:
+        if not self._seen_file and self._field_bytes + len(self._buffer) > (
+            MAX_FIELDS_BYTES
+        ):
+            raise S3Error(errors.MAX_POST_PRE_DATA)
+
+    def _step(self) -> Iterator[Event | None]:
+        """Do what the buffer allows of one state. Yields once for each advance."""
+        if self._state is _State.BEFORE_FIRST:
+            yield from self._skip_to_delimiter()
+        elif self._state is _State.AFTER_DELIMITER:
+            yield from self._read_after_delimiter()
+        elif self._state is _State.HEADERS:
+            yield from self._read_headers()
+        elif self._state is _State.VALUE:
+            yield from self._read_value()
+        elif self._state is _State.FILE:
+            yield from self._read_file()
+        elif self._state is _State.SKIPPING:
+            yield from self._skip_to_delimiter()
+        else:
+            self._buffer.clear()
+
+    def _take(self, length: int, then_skip: int = 0) -> bytes:
+        taken = bytes(self._buffer[:length])
+        del self._buffer[: length + then_skip]
+        return taken
+
+    def _skip_to_delimiter(self) -> Iterator[None]:
+        at = self._buffer.find(self._delimiter)
+        if at >= 0:
+            self._take(at, len(self._delimiter))
+            self._state = _State.AFTER_DELIMITER
+            yield None
+        elif self._seen_file:
+            # Keep only what could be the start of a delimiter.
+            keep = len(self._delimiter) - 1
+            if len(self._buffer) > keep:
+                self._take(len(self._buffer) - keep)
+
+    def _read_after_delimiter(self) -> Iterator[None]:
+        if len(self._buffer) < len(_CLOSING):
+            return
+        ending = self._take(len(_CLOSING))
+        if ending == _CLOSING:
+            self._state = _State.DONE
+        elif ending == _LINE_END:
+            self._state = _State.HEADERS
+        else:
+            raise S3Error(errors.MALFORMED_POST, "Bad text after a form boundary.")
+        yield None
+
+    def _read_headers(self) -> Iterator[Event | None]:
+        at = self._buffer.find(_HEADERS_END)
+        if at < 0:
+            if self._seen_file and len(self._buffer) > MAX_FIELDS_BYTES:
+                raise S3Error(
+                    errors.MALFORMED_POST, "A form part's headers are too long."
+                )
+            return
+        headers = self._decode(self._take(at, len(_HEADERS_END)))
+        self._field_bytes += at + len(_HEADERS_END)
+        disposition = next(
+            (
+                line
+                for line in headers.split("\r\n")
+                if line.lower().startswith("content-disposition:")
+            ),
+            "",
+        )
+        name = _DISPOSITION_NAME.search(disposition)
+        if name is None:
+            raise S3Error(errors.MALFORMED_POST, "A form part has no name.")
+        self._field_name = _unescape(name[1]).lower()
+        if self._seen_file:
+            self._state = _State.SKIPPING
+            yield None
+        elif self._field_name == FILE_FIELD:
+            filename = _DISPOSITION_FILENAME.search(disposition)
+            self._seen_file = True
+            self._state = _State.FILE
+            yield FileStart(_unescape(filename[1]) if filename else "")
+        else:
+            self._state = _State.VALUE
+            yield None
+
+    def _read_value(self) -> Iterator[Event]:
+        at = self._buffer.find(self._delimiter)
+        if at < 0:
+            return
+        self._field_bytes += at + len(self._delimiter)
+        value = self._decode(self._take(at, len(self._delimiter)))
+        self._state = _State.AFTER_DELIMITER
+        yield Field(self._field_name, value)
+
+    def _read_file(self) -> Iterator[Event]:
+        at = self._buffer.find(self._delimiter)
+        if at >= 0:
+            data = self._take(at, len(self._delimiter))
+            if data:
+                yield FileData(data)
+            self._state = _State.AFTER_DELIMITER
+            yield FileEnd()
+            return
+        safe = len(self._buffer) - (len(self._delimiter) - 1)
+        if safe > 0:
+            yield FileData(self._take(safe))
+
+    @staticmethod
+    def _decode(data: bytes) -> str:
+        try:
+            return data.decode()
+        except UnicodeDecodeError:
+            raise S3Error(errors.MALFORMED_POST, "A form field isn't UTF-8.") from None
+
+
+def _unescape(text: str) -> str:
+    return _ESCAPED.sub(r"\1", text)
+
+
+def check_policy(fields: Mapping[str, str], now: float) -> SizeLimits:
+    """Hold a form to its policy: nothing it doesn't allow, nor less than it asks."""
+    document = _load_policy(fields["policy"])
+    conditions = document.get("conditions")
+    if not isinstance(conditions, list):
+        raise S3Error(errors.INVALID_POLICY, "The policy has no conditions.")
+    _check_expiration(document.get("expiration"), now)
+
+    covered: set[str] = set()
+    limits = SizeLimits(0, None)
+    for condition in conditions:
+        if _is_length_condition(condition):
+            limits = _narrowed(limits, condition)
+        else:
+            covered.add(_check_condition(condition, fields))
+    for name in fields:
+        exempt = name in _NOT_FOR_THE_POLICY or name.startswith(_EXEMPT_PREFIXES)
+        if not exempt and name not in covered:
+            raise S3Error(errors.ACCESS_DENIED, f"{_REFUSAL}Extra field: {name}.")
+    return limits
+
+
+def _load_policy(encoded: str) -> dict[str, object]:
+    try:
+        document = json.loads(base64.b64decode(encoded, validate=True))
+    except (binascii.Error, ValueError, RecursionError):
+        raise S3Error(errors.INVALID_POLICY, "The policy can't be read.") from None
+    if not isinstance(document, dict):
+        raise S3Error(errors.INVALID_POLICY, "The policy can't be read.")
+    return document
+
+
+def _check_expiration(expiration: object, now: float) -> None:
+    if not isinstance(expiration, str):
+        raise S3Error(errors.INVALID_POLICY, "The policy has no expiration.")
+    if _EXPIRATION.fullmatch(expiration) is None:
+        raise S3Error(errors.INVALID_POLICY, "Bad expiration in the policy.")
+    try:
+        moment = datetime.fromisoformat(expiration)
+    except ValueError:
+        raise S3Error(errors.INVALID_POLICY, "Bad expiration in the policy.") from None
+    if moment.timestamp() < now:
+        raise S3Error(errors.ACCESS_DENIED, f"{_REFUSAL}Policy expired.")
+
+
+def _is_length_condition(condition: object) -> bool:
+    return (
+        isinstance(condition, list)
+        and bool(condition)
+        and isinstance(condition[0], str)
+        and condition[0].lower() == _LENGTH_CONDITION
+    )
+
+
+def _narrowed(limits: SizeLimits, condition: list[object]) -> SizeLimits:
+    try:
+        _, smallest, largest = condition
+        low, high = int(str(smallest)), int(str(largest))
+    except ValueError:
+        raise S3Error(errors.INVALID_POLICY, "Bad content-length-range.") from None
+    if low < 0 or high < low:
+        raise S3Error(errors.INVALID_POLICY, "Bad content-length-range.")
+    return SizeLimits(
+        max(limits.smallest, low),
+        high if limits.largest is None else min(limits.largest, high),
+    )
+
+
+def _check_condition(condition: object, fields: Mapping[str, str]) -> str:
+    """Check a condition holds, and return the field it covers."""
+    if isinstance(condition, dict) and len(condition) == 1:
+        [(name, wanted)] = condition.items()
+        return _check_field("eq", f"${name}", wanted, fields)
+    if isinstance(condition, list) and len(condition) == _EQUALS_ARGUMENTS:
+        operator, name, wanted = condition
+        return _check_field(operator, name, wanted, fields)
+    raise S3Error(errors.INVALID_POLICY, "A policy condition can't be read.")
+
+
+def _check_field(
+    operator: object, name: object, wanted: object, fields: Mapping[str, str]
+) -> str:
+    if not (
+        isinstance(operator, str)
+        and isinstance(name, str)
+        and name.startswith("$")
+        and isinstance(wanted, str)
+    ):
+        raise S3Error(errors.INVALID_POLICY, "A policy condition can't be read.")
+    field = name[1:].lower()
+    value = fields.get(field)
+    verb = operator.lower()
+    if verb == "eq":
+        holds = value == wanted
+    elif verb == "starts-with":
+        holds = value is not None and value.startswith(wanted)
+    else:
+        raise S3Error(errors.INVALID_POLICY, f"Unknown condition: {operator!r}.")
+    if not holds:
+        raise S3Error(errors.ACCESS_DENIED, f"{_REFUSAL}Condition failed: {field}.")
+    return field