pierre-sdk 0.1.0.dev4__py3-none-any.whl

pierre/_content.py

@@ -0,0 +1,168 @@
+"""Upload content in each form that the SDK accepts, as byte chunks."""
+
+import inspect
+from typing import (
+    IO,
+    Any,
+    AsyncIterable,
+    AsyncIterator,
+    Iterable,
+    Iterator,
+    List,
+    Tuple,
+    Union,
+)
+
+from pierre._errors import PierreError
+
+__all__ = [
+    "MAX_CHUNK_BYTES",
+    "Content",
+    "async_chunks",
+    "async_close",
+    "chunks",
+    "close",
+    "is_bytes",
+]
+
+# One piece of content: text, which the SDK sends as UTF-8, or bytes.
+ContentPiece = Union[str, bytes, bytearray, memoryview]
+
+# Content of a file or a diff: one piece, a binary file, or an iterable of pieces. `AsyncPierre`
+# also takes an async iterable of pieces and a file whose `read` is async.
+Content = Union[
+    ContentPiece,
+    IO[bytes],
+    Iterable[ContentPiece],
+    AsyncIterable[ContentPiece],
+]
+
+# The gateway limits a decoded chunk to 4 MiB.
+MAX_CHUNK_BYTES = 4 * 1024 * 1024
+
+_CONTENT_FORMS = "a string, bytes, a binary file, or an iterable of bytes"
+
+
+def is_bytes(value: object) -> bool:
+    return isinstance(value, (bytes, bytearray, memoryview))
+
+
+def _piece(value: object) -> bytes:
+    if isinstance(value, str):
+        return value.encode("utf-8")
+    if isinstance(value, (bytes, bytearray, memoryview)):
+        return bytes(value)
+    raise PierreError(
+        f"Each piece of `content` must be bytes or a string, not {type(value).__name__}."
+    )
+
+
+def _pieces(content: object) -> Iterator[bytes]:
+    if isinstance(content, str) or is_bytes(content):
+        yield _piece(content)
+        return
+    read = getattr(content, "read", None)
+    if callable(read):
+        while True:
+            data = read(MAX_CHUNK_BYTES)
+            if inspect.isawaitable(data):
+                if inspect.iscoroutine(data):
+                    data.close()
+                raise PierreError(
+                    "`Pierre` cannot read a file whose `read` is async. Use `AsyncPierre`."
+                )
+            if not data:
+                return
+            yield _piece(data)
+    if hasattr(content, "__aiter__"):
+        raise PierreError("`Pierre` cannot read an async iterable. Use `AsyncPierre`.")
+    if isinstance(content, Iterable):
+        for item in content:
+            yield _piece(item)
+        return
+    raise PierreError(f"`content` must be {_CONTENT_FORMS}.")
+
+
+async def _async_pieces(content: object) -> AsyncIterator[bytes]:
+    if isinstance(content, str) or is_bytes(content):
+        yield _piece(content)
+        return
+    read = getattr(content, "read", None)
+    if callable(read):
+        while True:
+            data = read(MAX_CHUNK_BYTES)
+            if inspect.isawaitable(data):
+                data = await data
+            if not data:
+                return
+            yield _piece(data)
+    if isinstance(content, AsyncIterable):
+        async for item in content:
+            yield _piece(item)
+        return
+    if isinstance(content, Iterable):
+        for item in content:
+            yield _piece(item)
+        return
+    raise PierreError(f"`content` must be {_CONTENT_FORMS}, or an async iterable of bytes.")
+
+
+class _Chunker:
+    """Splits pieces into chunks of at most `MAX_CHUNK_BYTES`, and joins small pieces up to that
+    size."""
+
+    def __init__(self) -> None:
+        self._parts: List[bytes] = []
+        self._size = 0
+
+    def add(self, piece: bytes) -> Iterator[bytes]:
+        """Gives each full chunk. A full chunk goes out only when more bytes follow, so that the
+        last chunk can have `eof`."""
+        offset = 0
+        while offset < len(piece):
+            if self._size == MAX_CHUNK_BYTES:
+                yield self.take()
+            length = min(len(piece) - offset, MAX_CHUNK_BYTES - self._size)
+            self._parts.append(piece[offset : offset + length])
+            self._size += length
+            offset += length
+
+    def take(self) -> bytes:
+        data = self._parts[0] if len(self._parts) == 1 else b"".join(self._parts)
+        self._parts = []
+        self._size = 0
+        return data
+
+
+def chunks(content: object) -> Iterator[Tuple[bytes, bool]]:
+    """The chunks of content, each with `eof`. The last chunk has `eof`, and empty content gives
+    one empty chunk."""
+    chunker = _Chunker()
+    for piece in _pieces(content):
+        for full in chunker.add(piece):
+            yield full, False
+    yield chunker.take(), True
+
+
+async def async_chunks(content: object) -> AsyncIterator[Tuple[bytes, bool]]:
+    """`chunks` for `AsyncPierre`, which also reads async iterables and async files."""
+    chunker = _Chunker()
+    async for piece in _async_pieces(content):
+        for full in chunker.add(piece):
+            yield full, False
+    yield chunker.take(), True
+
+
+def close(content: Any) -> None:
+    """Closes a file that a content function returned. The SDK opened it, so the SDK closes it."""
+    method = getattr(content, "close", None)
+    if callable(method) and callable(getattr(content, "read", None)):
+        method()
+
+
+async def async_close(content: Any) -> None:
+    method = getattr(content, "close", None)
+    if callable(method) and callable(getattr(content, "read", None)):
+        result = method()
+        if inspect.isawaitable(result):
+            await result

pierre/_credentials.py

@@ -0,0 +1,321 @@
+"""Credentials and token signing."""
+
+import asyncio
+import base64
+import hashlib
+import inspect
+import json
+import re
+import time
+from dataclasses import dataclass
+from typing import (
+    Any,
+    Awaitable,
+    Callable,
+    Dict,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    Sequence,
+    Tuple,
+    Union,
+)
+
+from cryptography.hazmat.primitives import hashes, serialization
+from cryptography.hazmat.primitives.asymmetric import ec, padding, rsa
+from cryptography.hazmat.primitives.asymmetric.utils import decode_dss_signature
+
+from pierre._errors import PierreError
+from pierre._types import RefPolicy, TokenRequest
+
+# A token that the SDK signs for one request lives 5 minutes.
+REQUEST_TOKEN_SECONDS = 5 * 60
+# The longest token lifetime that the SDK signs.
+MAX_TOKEN_SECONDS = 60 * 60
+# The `sub` claim of the tokens that the SDK signs.
+SUBJECT = "pierre-sdk-python"
+
+# Ref policies after the check: `(pattern, ops)` pairs, with `None` for a rule with no `ops`.
+Rules = Tuple[Tuple[str, Optional[Tuple[str, ...]]], ...]
+
+GetToken = Callable[[TokenRequest], Union[str, Awaitable[str]]]
+
+_PEM = re.compile(r"-----BEGIN PRIVATE KEY-----([\s\S]+?)-----END PRIVATE KEY-----")
+
+# The signing algorithm and hash of each EC curve.
+_CURVES: Dict[str, Tuple[str, hashes.HashAlgorithm]] = {
+    "secp256r1": ("ES256", hashes.SHA256()),
+    "secp384r1": ("ES384", hashes.SHA384()),
+    "secp521r1": ("ES512", hashes.SHA512()),
+}
+
+
+def check_ref_policies(policies: object, name: str) -> Optional[Rules]:
+    """Checks the shape of ref policies. The gateway checks the pattern forms and the ops."""
+    if policies is None:
+        return None
+    if isinstance(policies, (str, bytes)) or not isinstance(policies, Sequence):
+        raise PierreError(f"`{name}` must be a list of rules.")
+    rules: List[Tuple[str, Optional[Tuple[str, ...]]]] = []
+    for index, policy in enumerate(policies):
+        at = f"`{name}[{index}]`"
+        if not isinstance(policy, Mapping):
+            raise PierreError(f"{at} must be a dict with `pattern` and `ops`.")
+        pattern = policy.get("pattern")
+        if not isinstance(pattern, str) or pattern == "":
+            raise PierreError(f"{at}.pattern must be a non-empty string.")
+        ops = policy.get("ops")
+        if ops is None:
+            rules.append((pattern, None))
+            continue
+        if (
+            isinstance(ops, (str, bytes))
+            or not isinstance(ops, Sequence)
+            or any(not isinstance(op, str) or op == "" for op in ops)
+        ):
+            raise PierreError(f"{at}.ops must be a list of op names.")
+        rules.append((pattern, tuple(ops)))
+    return tuple(rules)
+
+
+def copy_ref_policies(policies: object) -> object:
+    """A copy of ref policies and their ops, so that a later change by the caller does not change
+    a request. The check runs later, so a value of another type stays as it is."""
+    if not isinstance(policies, (list, tuple)):
+        return policies
+    copied: List[object] = []
+    for policy in policies:
+        if isinstance(policy, Mapping):
+            rule = dict(policy)
+            if isinstance(rule.get("ops"), (list, tuple)):
+                rule["ops"] = list(rule["ops"])
+            copied.append(rule)
+        else:
+            copied.append(policy)
+    return copied
+
+
+def policies_of(rules: Rules) -> List[RefPolicy]:
+    """The rules as new `RefPolicy` dicts."""
+    policies: List[RefPolicy] = []
+    for pattern, ops in rules:
+        policies.append(
+            {"pattern": pattern} if ops is None else {"pattern": pattern, "ops": list(ops)}
+        )
+    return policies
+
+
+def refs_claim(rules: Rules) -> Optional[List[List[Any]]]:
+    """The `refs` claim of the rules, or `None` for no rule."""
+    if len(rules) == 0:
+        return None
+    return [[pattern, list(ops or ())] for pattern, ops in rules]
+
+
+def check_ttl(ttl: object, name: str) -> int:
+    """Checks a token lifetime: a whole number of seconds from 1 to 3600."""
+    if isinstance(ttl, bool) or not isinstance(ttl, int) or ttl < 1 or ttl > MAX_TOKEN_SECONDS:
+        raise PierreError(
+            f"`{name}` must be a whole number of seconds from 1 to {MAX_TOKEN_SECONDS}."
+        )
+    return ttl
+
+
+@dataclass(frozen=True)
+class Claims:
+    """The claims of a token that the SDK signs."""
+
+    iss: str
+    sub: str
+    repo: Optional[str]
+    scopes: Tuple[str, ...]
+    # The lifetime in seconds: `exp` is `iat` plus this value.
+    ttl: int
+    rules: Rules
+
+
+@dataclass(frozen=True)
+class TokenNeeds:
+    """The token that a request needs."""
+
+    scopes: Tuple[str, ...]
+    repo: Optional[str]
+    ttl: int
+    rules: Rules
+
+    def request(self) -> TokenRequest:
+        """The request that `get_token` receives."""
+        request: TokenRequest = {"scopes": list(self.scopes), "ttl": self.ttl}
+        if self.repo is not None:
+            request["repo"] = self.repo
+        if len(self.rules) > 0:
+            request["ref_policies"] = policies_of(self.rules)
+        return request
+
+
+def _base64url(data: bytes) -> str:
+    return base64.urlsafe_b64encode(data).rstrip(b"=").decode("ascii")
+
+
+def _json(value: object) -> bytes:
+    return json.dumps(value, separators=(",", ":")).encode("utf-8")
+
+
+@dataclass(frozen=True)
+class _SigningKey:
+    key: Union[ec.EllipticCurvePrivateKey, rsa.RSAPrivateKey]
+    alg: str
+    # The lowercase hex SHA-256 of the SPKI DER of the public key.
+    kid: str
+
+
+def _load_signing_key(pem: str) -> _SigningKey:
+    """Loads a PKCS #8 PEM private key, and finds its algorithm and key ID."""
+    match = _PEM.search(pem)
+    if match is None:
+        raise PierreError('The private key must be a PEM block of type "PRIVATE KEY" (PKCS #8).')
+    try:
+        key = serialization.load_pem_private_key(match.group(0).encode("ascii"), password=None)
+    except (TypeError, ValueError, UnicodeEncodeError):
+        key = None
+    if isinstance(key, ec.EllipticCurvePrivateKey) and key.curve.name in _CURVES:
+        alg = _CURVES[key.curve.name][0]
+    elif isinstance(key, rsa.RSAPrivateKey):
+        alg = "RS256"
+    else:
+        raise PierreError(
+            "The private key is not a PKCS #8 EC (P-256, P-384, or P-521) or RSA key in PEM form."
+        )
+    spki = key.public_key().public_bytes(
+        serialization.Encoding.DER, serialization.PublicFormat.SubjectPublicKeyInfo
+    )
+    return _SigningKey(key=key, alg=alg, kid=hashlib.sha256(spki).hexdigest())
+
+
+class Signer:
+    """A private key that signs tokens. It loads the key on the first signature, and loads it
+    again after a failure, for example when the key file appears later."""
+
+    def __init__(self, load: Callable[[], str]) -> None:
+        self._load = load
+        self._key: Optional[_SigningKey] = None
+
+    def sign(self, claims: Claims) -> str:
+        if self._key is None:
+            self._key = _load_signing_key(self._load())
+        signing = self._key
+        now = int(time.time())
+        header = {"alg": signing.alg, "typ": "JWT", "kid": signing.kid}
+        payload: Dict[str, Any] = {"iss": claims.iss, "sub": claims.sub}
+        if claims.repo is not None:
+            payload["repo"] = claims.repo
+        payload["scopes"] = list(claims.scopes)
+        payload["iat"] = now
+        payload["exp"] = now + claims.ttl
+        refs = refs_claim(claims.rules)
+        if refs is not None:
+            payload["refs"] = refs
+        signing_input = f"{_base64url(_json(header))}.{_base64url(_json(payload))}".encode("ascii")
+        if isinstance(signing.key, ec.EllipticCurvePrivateKey):
+            hash_algorithm = _CURVES[signing.key.curve.name][1]
+            der = signing.key.sign(signing_input, ec.ECDSA(hash_algorithm))
+            r, s = decode_dss_signature(der)
+            size = (signing.key.curve.key_size + 7) // 8
+            signature = r.to_bytes(size, "big") + s.to_bytes(size, "big")
+        else:
+            signature = signing.key.sign(signing_input, padding.PKCS1v15(), hashes.SHA256())
+        return f"{signing_input.decode('ascii')}.{_base64url(signature)}"
+
+
+class Credentials:
+    """The source of the token of each request: a private key, a fixed token, or `get_token`."""
+
+    def __init__(
+        self,
+        kind: Literal["private_key", "token", "get_token"],
+        *,
+        org: Optional[str] = None,
+        signer: Optional[Signer] = None,
+        token: Optional[str] = None,
+        get_token: Optional[GetToken] = None,
+    ) -> None:
+        self.kind = kind
+        self.org = org
+        self.signer = signer
+        self._token = token
+        self._get_token = get_token
+
+    @property
+    def fixed(self) -> bool:
+        """Whether the token is fixed, so the SDK cannot set its claims."""
+        return self.kind == "token"
+
+    def sign(self, needs: TokenNeeds, subject: str = SUBJECT) -> str:
+        if self.signer is None or self.org is None:
+            raise PierreError("The client has no private key.")
+        return self.signer.sign(
+            Claims(
+                iss=self.org,
+                sub=subject,
+                repo=needs.repo,
+                scopes=needs.scopes,
+                ttl=needs.ttl,
+                rules=needs.rules,
+            )
+        )
+
+    def token(self, needs: TokenNeeds) -> str:
+        """The token of a request in `Pierre`."""
+        if self.kind == "private_key":
+            return self.sign(needs)
+        if self.kind == "token":
+            return self._token or ""
+        result = self._call_get_token(needs)
+        if inspect.isawaitable(result):
+            if inspect.iscoroutine(result):
+                result.close()
+            raise PierreError(
+                "`Pierre` needs a `get_token` that returns a token string. "
+                "Give an async `get_token` to `AsyncPierre`."
+            )
+        return _checked_token(result)
+
+    async def async_token(self, needs: TokenNeeds) -> str:
+        """The token of a request in `AsyncPierre`. `get_token` can be sync or async. A sync
+        `get_token` runs in a thread, so that it does not block the event loop and the timeout
+        can stop the wait."""
+        if self.kind != "get_token":
+            return self.token(needs)
+        if inspect.iscoroutinefunction(self._get_token):
+            result = self._call_get_token(needs)
+        else:
+            result = await asyncio.to_thread(self._call_get_token, needs)
+        if inspect.isawaitable(result):
+            result = await result
+        return _checked_token(result)
+
+    def _call_get_token(self, needs: TokenNeeds) -> object:
+        if self._get_token is None:
+            raise PierreError("The client has no `get_token`.")
+        return self._get_token(needs.request())
+
+
+def _checked_token(token: object) -> str:
+    if not isinstance(token, str) or token == "":
+        raise PierreError("`get_token` must return a token string.")
+    return token
+
+
+def token_issuer(token: str) -> Optional[str]:
+    """The `iss` claim of a token, or `None`. The SDK does not verify the token."""
+    parts = token.split(".")
+    if len(parts) < 2:
+        return None
+    payload = parts[1]
+    try:
+        claims = json.loads(base64.urlsafe_b64decode(payload + "=" * (-len(payload) % 4)))
+    except (ValueError, TypeError):
+        return None
+    issuer = claims.get("iss") if isinstance(claims, dict) else None
+    return issuer if isinstance(issuer, str) and issuer != "" else None

pierre/_errors.py

@@ -0,0 +1,205 @@
+"""The errors of the SDK."""
+
+from typing import Any, Dict, Literal, Mapping, Optional, Type, Union, cast
+
+import httpx
+
+from pierre._generated.schemas import (
+    NotesWriteResult,
+    RefUpdateResult,
+    RestoredCommit,
+    WrittenCommit,
+)
+
+# The docs page with the table of problem codes, in the Markdown form for agents.
+ERROR_CODES_DOC_URL = "https://code.storage/docs/api/overview.md#error-codes"
+
+
+class PierreError(Exception):
+    """The base class of every error that the SDK raises."""
+
+
+class APIConnectionError(PierreError):
+    """The request got no response."""
+
+
+class APITimeoutError(APIConnectionError):
+    """The request got no response before its timeout."""
+
+
+def _text(value: object) -> Optional[str]:
+    return value if isinstance(value, str) and value != "" else None
+
+
+def _problem(body: object) -> Mapping[str, Any]:
+    return body if isinstance(body, dict) else {}
+
+
+class APIError(PierreError):
+    """An HTTP error response. Branch on `code`, not on the message."""
+
+    status: int
+    # The problem `code`, for example `repository_not_found`.
+    code: Optional[str]
+    title: Optional[str]
+    detail: Optional[str]
+    # The `X-Request-Id` response header, or `None` when the server does not send it.
+    request_id: Optional[str]
+    # The headers of the error response, for example `retry-after`.
+    headers: httpx.Headers
+    # The parsed response body, with any extension members, or its text when it is not JSON.
+    body: object
+    # The docs page that explains the codes.
+    doc_url: str
+
+    def __init__(self, status: int, body: object, headers: Optional[httpx.Headers] = None) -> None:
+        problem = _problem(body)
+        code = _text(problem.get("code"))
+        title = _text(problem.get("title"))
+        detail = (
+            _text(problem.get("detail"))
+            or _text(problem.get("error"))
+            or _text(problem.get("message"))
+        )
+        prefix = f"{status}" if code is None else f"{status} {code}"
+        super().__init__(f"{prefix}: {detail or title or 'request failed'}")
+        self.status = status
+        self.code = code
+        self.title = title
+        self.detail = detail
+        self.headers = headers if headers is not None else httpx.Headers()
+        self.request_id = self.headers.get("x-request-id")
+        self.body = body
+        self.doc_url = ERROR_CODES_DOC_URL
+
+
+class BadRequestError(APIError):
+    """A 400 response."""
+
+
+class AuthenticationError(APIError):
+    """A 401 response."""
+
+
+class PermissionDeniedError(APIError):
+    """A 403 response."""
+
+
+class NotFoundError(APIError):
+    """A 404 response."""
+
+
+class ConflictError(APIError):
+    """A 409 response."""
+
+
+class RateLimitError(APIError):
+    """A 429 response."""
+
+
+class InternalServerError(APIError):
+    """A 5xx response."""
+
+
+class MergeConflictError(ConflictError):
+    """A merge conflict. It carries the conflict fields of the response."""
+
+    conflict_type: Optional[str]
+    conflict_paths: object
+    merge_base_sha: Optional[str]
+
+    def __init__(self, status: int, body: object, headers: Optional[httpx.Headers] = None) -> None:
+        super().__init__(status, body, headers)
+        problem = _problem(body)
+        self.conflict_type = _text(problem.get("conflict_type"))
+        self.conflict_paths = problem.get("conflict_paths")
+        self.merge_base_sha = _text(problem.get("merge_base_sha"))
+
+
+class RefGuardError(ConflictError):
+    """A Git write whose ref guard failed: a response with the code `ref_guard_failed`. `guard`
+    names the request field of the guard, for example `expected_target_sha`. `current_sha` is the
+    commit of the guarded ref, or 40 zeros when the ref does not exist; send it as the guard of the
+    next attempt. The body of a commit write, a restore, or a notes write also keeps `result`, which
+    `body` gives."""
+
+    guard: Optional[str]
+    current_sha: Optional[str]
+
+    def __init__(self, status: int, body: object, headers: Optional[httpx.Headers] = None) -> None:
+        super().__init__(status, body, headers)
+        problem = _problem(body)
+        self.guard = _text(problem.get("guard"))
+        self.current_sha = _text(problem.get("current_sha"))
+
+
+class RefUpdateError(APIError):
+    """A Git write whose ref update failed: an error response whose body has `result`. It carries
+    `result` and `commit`. A Git write method returns only when the ref update succeeded."""
+
+    # The result of the ref update. `status` gives the reason, for example `conflict` or
+    # `precondition_failed`. A notes write sends a `NotesWriteResult`, which has only `success`,
+    # `status`, and `message`.
+    result: Union[RefUpdateResult, NotesWriteResult]
+    # The commit of the write, or `None` when the response has none. A restore has no `blob_count`.
+    commit: Optional[Union[WrittenCommit, RestoredCommit]]
+
+    def __init__(self, status: int, body: object, headers: Optional[httpx.Headers] = None) -> None:
+        super().__init__(status, body, headers)
+        problem = _problem(body)
+        self.result = cast(Union[RefUpdateResult, NotesWriteResult], problem.get("result"))
+        self.commit = problem.get("commit")
+
+
+WebhookVerificationFailure = Literal[
+    "missing_signature",
+    "malformed_signature",
+    "signature_mismatch",
+    "timestamp_out_of_range",
+    "invalid_payload",
+]
+
+
+class WebhookVerificationError(PierreError):
+    """`verify_webhook` rejected the delivery. `reason` says why."""
+
+    def __init__(self, reason: WebhookVerificationFailure, message: str) -> None:
+        super().__init__(message)
+        self.reason: WebhookVerificationFailure = reason
+
+
+_BY_STATUS: Dict[int, Type[APIError]] = {
+    400: BadRequestError,
+    401: AuthenticationError,
+    403: PermissionDeniedError,
+    404: NotFoundError,
+    409: ConflictError,
+    429: RateLimitError,
+}
+
+# The codes of a merge conflict. Today the conflict body can also have `conflict_type` only.
+_MERGE_CONFLICT_CODES = frozenset({"merge_conflict"})
+
+# The code of a failed ref guard. Its body can also have `result`, so it is read first.
+_REF_GUARD_FAILED = "ref_guard_failed"
+
+
+def api_error(status: int, body: object, headers: httpx.Headers) -> APIError:
+    """Picks the error class of an error response from the status and the body."""
+    problem = _problem(body)
+    code = _text(problem.get("code"))
+    if code == _REF_GUARD_FAILED:
+        return RefGuardError(status, body, headers)
+    if isinstance(problem.get("result"), dict):
+        return RefUpdateError(status, body, headers)
+    if (code is not None and code in _MERGE_CONFLICT_CODES) or isinstance(
+        problem.get("conflict_type"), str
+    ):
+        return MergeConflictError(status, body, headers)
+    error_class = _BY_STATUS.get(status, InternalServerError if status >= 500 else APIError)
+    return error_class(status, body, headers)
+
+
+def problem_code(body: object) -> Optional[str]:
+    """The problem `code` of a parsed body, or `None`."""
+    return _text(_problem(body).get("code"))

pierre/_generated/__init__.py

@@ -0,0 +1,2 @@
+# Generated by sdkgen. Do not edit this file.
+"""The generated layer of the SDK."""