pipelex-api 0.86.1__tar.gz → 0.87.0__tar.gz

{pipelex_api-0.86.1 → pipelex_api-0.87.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: pipelex-api
-Version: 0.86.1
+Version: 0.87.0
 Summary: Pipelex API
 Project-URL: Homepage, https://pipelex.com
 Project-URL: Repository, https://github.com/Pipelex/pipelex
@@ -16,7 +16,7 @@ Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
 Requires-Python: <3.15,>=3.11
 Requires-Dist: fastapi<0.142,>=0.118.0
-Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.86.1
+Requires-Dist: pipelex[anthropic,bedrock,fal,google,google-genai,mistralai]==0.87.0
 Requires-Dist: pyjwt>=2.10.1
 Requires-Dist: uvicorn>=0.37.0
 Provides-Extra: dev

pipelex_api-0.87.0/pipelex_api/blob_store.py

@@ -0,0 +1,190 @@
+"""The runner's blob store: bytes keyed by their SHA-256, and the listings of the trees it holds in full.
+
+A sender asks which of its hashes the runner lacks (`POST /v1/blobs/missing`), uploads only those
+(`PUT /v1/blobs/{sha256}`), then sends its method as a map of file entries naming them by hash. This runner keeps those
+bytes in its configured storage provider, local or cloud, so a self-hosted deployment gets the shape the hosted plane
+has (`wip/extra-files/design.md`, D5 and D6), under one prefix per caller:
+
+    <scope>/blobs/<sha256>   a file's bytes, written once and never changed
+    <scope>/trees/<hex>      a tree's listing, under the hexadecimal part of its digest
+
+The scope is the run's trusted read scope, the one a multi-tenant host sends for the organization it serves, or else the
+caller's own id (`single-tenant` on a deployment that identifies no caller): a hash entry resolves only in the caller's
+own scope, never across tenants, so asking whether a hash is held tells a caller nothing of another's files.
+
+A listing is written only after every file it names is held, by this runner once it has read a request's tree in
+full, and never by an upload: `PUT /v1/blobs/{sha256}` writes under `blobs/` alone, so an uploaded file that happens to
+be a well-formed listing proves nothing. A listing present means the whole tree is present, which lets
+`POST /v1/blobs/missing` answer a tree's digest with one lookup.
+
+Every object under `blobs/<sha256>` holds, whole, the bytes whose SHA-256 is that hash: the key is the hash this runner
+computes from the bytes it writes, the runner's own `PUT /v1/blobs/{sha256}` writes only bytes it checked against the
+path's hash, and every built-in storage provider publishes a write whole or not at all (the local one through a
+temporary file renamed onto the key). So an object's stored size, read from its metadata without reading its bytes, is
+the size of that hash's bytes, and nothing deletes or rewrites a blob. A listing, written once each of its files was
+held at its size, therefore stays true.
+
+From this the blob store answers without reading what it does not need:
+
+- `POST /v1/blobs/missing` lists a hash this runner does not hold at the size the sender states, an object held at
+  another size included, which only a storage that does not publish writes whole could leave short, and which the
+  upload then writes over.
+- A run looks up each hash's stored size first. A hash held at another size than an entry states is a 422
+  `BlobMismatch`, never `MissingBlobs`, since uploading those bytes again would change nothing: the entry is wrong.
+  Only a blob held at the size its entries state is read, so the bytes read are bounded by the sizes the request
+  states, which its census already held to the tree's limits, and each is checked against its hash once read. Bytes
+  that do not hash to their key mean the storage itself is damaged, which no caller can repair: a 500
+  `BlobStoreCorrupted`.
+
+Lookups and reads run a bounded number at a time, so one request naming many blobs never opens them all at once.
+"""
+
+from collections.abc import Iterable, Mapping
+from functools import partial
+
+from mthds.method_tree.digest import DIGEST_PREFIX, sha256_hex, tree_listing
+from mthds.method_tree.file_entry import BlobStore, InMemoryBlobStore
+from mthds.method_tree.models import MethodTree
+from pipelex.core.method_trees.exceptions import MethodFileUnavailableError
+from pipelex.core.method_trees.method_files import TreeListing
+from pipelex.runtime_hub import get_storage_provider
+from pipelex.tools.misc.async_utils import gather_bounded
+from pipelex.tools.storage.exceptions import StorageFileNotFoundError
+from pipelex.tools.storage.storage_provider_abstract import PIPELEX_STORAGE_SCHEME, StorageProviderAbstract
+
+from pipelex_api.error_types import ErrorType
+from pipelex_api.errors import raise_internal_server_error
+
+BLOBS_PREFIX = "blobs"
+TREES_PREFIX = "trees"
+_BLOB_CONTENT_TYPE = "application/octet-stream"
+_LISTING_CONTENT_TYPE = "text/plain; charset=utf-8"
+# How many lookups of a blob's size, which read metadata alone, and how many reads or writes of a blob's bytes, one
+# request runs at a time.
+MAX_CONCURRENT_BLOB_LOOKUPS = 16
+MAX_CONCURRENT_BLOB_TRANSFERS = 4
+
+
+class RunnerBlobStore:
+    """The blobs and tree listings of one caller's scope, over the deployment's storage provider."""
+
+    def __init__(self, *, scope: str, storage: StorageProviderAbstract) -> None:
+        self._scope = scope
+        self._storage = storage
+
+    @classmethod
+    def make_for_scope(cls, *, scope: str) -> "RunnerBlobStore":
+        """The blob store of `scope`, over the storage provider the runtime booted with."""
+        return cls(scope=scope, storage=get_storage_provider())
+
+    @property
+    def scope(self) -> str:
+        return self._scope
+
+    def _blob_key(self, *, sha256: str) -> str:
+        return f"{self._scope}/{BLOBS_PREFIX}/{sha256}"
+
+    def _tree_key(self, *, tree_digest: str) -> str:
+        return f"{self._scope}/{TREES_PREFIX}/{tree_digest.removeprefix(DIGEST_PREFIX)}"
+
+    async def _holds_key(self, *, key: str) -> bool:
+        try:
+            await self._storage.load_size(f"{PIPELEX_STORAGE_SCHEME}{key}")
+        except StorageFileNotFoundError:
+            return False
+        return True
+
+    async def stored_size(self, *, sha256: str) -> int | None:
+        """The size of the object stored under the hash `sha256`, read from its metadata, or None when there is none."""
+        try:
+            return await self._storage.load_size(f"{PIPELEX_STORAGE_SCHEME}{self._blob_key(sha256=sha256)}")
+        except StorageFileNotFoundError:
+            return None
+
+    async def holds_blob(self, *, sha256: str, size: int) -> bool:
+        """Whether the bytes whose SHA-256 is `sha256` are held at `size`, an object of another size being no blob."""
+        return await self.stored_size(sha256=sha256) == size
+
+    async def stored_sizes(self, *, sha256s: Iterable[str]) -> dict[str, int | None]:
+        """Each hash's stored size, None for a hash not held, looked up a bounded number at a time."""
+        hashes = list(sha256s)
+        sizes = await gather_bounded([partial(self.stored_size, sha256=sha256) for sha256 in hashes], max_concurrency=MAX_CONCURRENT_BLOB_LOOKUPS)
+        return dict(zip(hashes, sizes, strict=True))
+
+    async def _read_verified(self, *, sha256: str, size: int) -> bytes:
+        """The stored bytes of `sha256`, held at `size`, checked against their hash.
+
+        Raises (as a problem document): a 500 `BlobStoreCorrupted` for bytes that are not `size` bytes hashing to `sha256`.
+        """
+        data = await self._storage.load(f"{PIPELEX_STORAGE_SCHEME}{self._blob_key(sha256=sha256)}")
+        if len(data) != size or sha256_hex(data) != sha256:
+            msg = (
+                f"The bytes this runner's blob store holds under {sha256} do not hash to it: its storage is damaged, "
+                "and its operator must remove that object for the blob to be uploaded again."
+            )
+            raise_internal_server_error(message=msg, error_type=ErrorType.BLOB_STORE_CORRUPTED)
+        return data
+
+    async def read_held(self, *, sizes: Mapping[str, int]) -> BlobStore:
+        """Read these held blobs, each at its stored size, as the synchronous blob store a map of file entries is read with.
+
+        Args:
+            sizes: Each hash to read, with the size it is held at, which the caller bounded before asking.
+        """
+        blobs = await gather_bounded(
+            [partial(self._read_verified, sha256=sha256, size=size) for sha256, size in sizes.items()],
+            max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS,
+        )
+        return InMemoryBlobStore(blobs)
+
+    async def store_blob(self, *, data: bytes) -> str:
+        """Hold `data` under its SHA-256, writing it unless it is held at its size already, and return the hash.
+
+        An object under the hash at another size, which only a storage that does not publish writes whole could leave,
+        is written over.
+        """
+        sha256 = sha256_hex(data)
+        if not await self.holds_blob(sha256=sha256, size=len(data)):
+            await self._storage.store(data, key=self._blob_key(sha256=sha256), content_type=_BLOB_CONTENT_TYPE)
+        return sha256
+
+    async def holds_tree(self, *, tree_digest: str) -> bool:
+        """Whether the listing of the tree `tree_digest` names is held, which means every file of the tree is."""
+        return await self._holds_key(key=self._tree_key(tree_digest=tree_digest))
+
+    async def tree_hashes(self, *, tree_digest: str) -> set[str] | None:
+        """The hashes of the files of the tree `tree_digest` names, when its listing is held, else None.
+
+        The listing is read and checked against the digest, so a sender naming a tree it does not send learns nothing
+        held that its own blobs are not part of.
+        """
+        try:
+            listing = await self._storage.load(f"{PIPELEX_STORAGE_SCHEME}{self._tree_key(tree_digest=tree_digest)}")
+        except StorageFileNotFoundError:
+            return None
+        try:
+            tree_listing_read = TreeListing.parse(tree_digest=tree_digest, listing=listing)
+        except MethodFileUnavailableError:
+            # Not the listing the digest names: the tree is not held.
+            return None
+        return set(tree_listing_read.files.values())
+
+    async def record_tree(self, *, tree: MethodTree, contents: Mapping[str, bytes]) -> None:
+        """Hold every file of a tree read in full, then its listing, so a later request can send it by hash.
+
+        The listing is written last, once every file it names is held, so a listing present always means its whole
+        tree is, and a tree whose listing is held already is left as it is, without a lookup per file.
+
+        Args:
+            tree: The tree, as a map of file entries made it.
+            contents: Each file's bytes by tree path.
+        """
+        if await self.holds_tree(tree_digest=tree.digest):
+            return
+        await gather_bounded([partial(self.store_blob, data=data) for data in contents.values()], max_concurrency=MAX_CONCURRENT_BLOB_TRANSFERS)
+        await self._storage.store(tree_listing(tree.files), key=self._tree_key(tree_digest=tree.digest), content_type=_LISTING_CONTENT_TYPE)
+
+    async def missing_blobs(self, *, sizes: Mapping[str, int]) -> list[str]:
+        """The hashes of `sizes` not held at their size, in the mapping's order."""
+        stored = await self.stored_sizes(sha256s=sizes)
+        return [sha256 for sha256, size in sizes.items() if stored[sha256] != size]

pipelex_api-0.87.0/pipelex_api/blob_upload.py

@@ -0,0 +1,102 @@
+"""The signed upload targets `POST /v1/blobs/missing` names, and their check on `PUT /v1/blobs/{sha256}`.
+
+A sender uploads a blob to the target the runner named, carrying exactly the headers it named, so the target itself
+must say that the upload is allowed, as a presigned URL does on an object store. This runner signs each target it
+names: its URL is its own `PUT /v1/blobs/{sha256}` with a query naming the blob store's scope, an expiry and an
+HMAC-SHA256 signature over the scope, the hash and the expiry. The upload route checks the signature and the expiry,
+writes under the scope the signature binds and nowhere else, and answers a missing, forged or expired signature with a
+403 `BlobUploadNotAuthorized`. The upload route therefore needs no bearer token: only a caller who passed the
+deployment's authentication on `POST /v1/blobs/missing` holds a target, for one hash, in its own scope, for a while.
+
+The key is `BLOB_UPLOAD_SIGNING_KEY` when the deployment sets it, which it must when several instances serve one
+deployment, since a target named by one instance is checked by whichever instance receives the upload; otherwise each
+process draws a random key of its own at its first signature.
+
+A target's URL starts with `PUBLIC_BASE_URL` when the deployment sets it, the address its clients reach the API at,
+such as `https://runner.example.com`: behind a proxy that terminates TLS or serves the API under a path prefix, the URL
+the runner received a request at is not the one a client can upload to. Unset, the target starts with the URL of the
+request that asked for it.
+"""
+
+import hashlib
+import hmac
+import secrets
+import time
+from functools import cache
+from urllib.parse import urlencode, urlsplit
+
+from fastapi import Request
+from pipelex.system.environment import get_optional_env
+
+from pipelex_api.error_types import ErrorType
+from pipelex_api.errors import raise_internal_server_error
+from pipelex_api.limits import BLOB_UPLOAD_URL_TTL_SECONDS
+
+PUBLIC_BASE_URL_ENV = "PUBLIC_BASE_URL"
+_UPLOAD_ROUTE_NAME = "put_blob"
+
+SCOPE_PARAM = "scope"
+EXPIRES_PARAM = "expires"
+SIGNATURE_PARAM = "signature"
+
+
+@cache
+def _signing_key() -> bytes:
+    configured = get_optional_env("BLOB_UPLOAD_SIGNING_KEY")
+    if configured:
+        return configured.encode("utf-8")
+    return secrets.token_bytes(32)
+
+
+def _signature(*, scope: str, sha256: str, expires: int) -> str:
+    message = f"{scope}\n{sha256}\n{expires}".encode()
+    return hmac.new(_signing_key(), message, hashlib.sha256).hexdigest()
+
+
+def signed_upload_query(*, scope: str, sha256: str) -> str:
+    """The query string of an upload target for one blob in one scope, valid for `BLOB_UPLOAD_URL_TTL_SECONDS`."""
+    expires = int(time.time()) + BLOB_UPLOAD_URL_TTL_SECONDS
+    return urlencode({SCOPE_PARAM: scope, EXPIRES_PARAM: expires, SIGNATURE_PARAM: _signature(scope=scope, sha256=sha256, expires=expires)})
+
+
+def _public_base_url() -> str | None:
+    """`PUBLIC_BASE_URL` without its trailing slash, or None when unset.
+
+    Raises (as a problem document): a 500 `ServerMisconfigured` for a value that is not an absolute http or https URL
+    without a query or a fragment.
+    """
+    configured = get_optional_env(PUBLIC_BASE_URL_ENV)
+    if not configured:
+        return None
+    parts = urlsplit(configured)
+    if parts.scheme not in {"http", "https"} or not parts.netloc or parts.query or parts.fragment:
+        msg = f"Server configuration error: {PUBLIC_BASE_URL_ENV} must be an absolute http or https URL, such as https://runner.example.com."
+        raise_internal_server_error(message=msg, error_type=ErrorType.SERVER_MISCONFIGURED)
+    return configured.rstrip("/")
+
+
+def upload_target_url(*, request: Request, scope: str, sha256: str) -> str:
+    """The signed URL a sender uploads one blob to: this runner's `PUT /v1/blobs/{sha256}`, at its public address."""
+    query = signed_upload_query(scope=scope, sha256=sha256)
+    base_url = _public_base_url()
+    if base_url is None:
+        return f"{request.url_for(_UPLOAD_ROUTE_NAME, sha256=sha256)}?{query}"
+    return f"{base_url}{request.app.url_path_for(_UPLOAD_ROUTE_NAME, sha256=sha256)}?{query}"
+
+
+def is_upload_authorized(*, scope: str | None, sha256: str, expires: str | None, signature: str | None) -> bool:
+    """Whether an upload carries a signature this runner made for this hash and scope, still within its expiry.
+
+    Every part of the query is the caller's text, read as such: an expiry that is not a decimal integer, or a signature
+    holding characters no signature this runner makes holds, is no valid target, never a failure of the check.
+    """
+    if scope is None or expires is None or signature is None:
+        return False
+    if not expires.isascii() or not expires.isdecimal():
+        return False
+    expires_at = int(expires)
+    if expires_at < int(time.time()):
+        return False
+    # Compared as bytes: `compare_digest` refuses text holding a character outside ASCII, which a forged signature may.
+    expected = _signature(scope=scope, sha256=sha256, expires=expires_at).encode("ascii")
+    return hmac.compare_digest(expected, signature.encode("utf-8", "surrogatepass"))

pipelex_api-0.87.0/pipelex_api/bundle.py

@@ -0,0 +1,87 @@
+"""Decode a `bundle_b64` archive into its entries, under the guards a zip archive needs.
+
+A run request may carry its method as `bundle_b64`, a base64-encoded zip archive of the method's folder, beside the
+map of file entries `files` that replaces it. This module only opens the archive: it bounds the encoded string before
+decoding it, refuses an archive with too many members, and reads each member through a bounded stream so a zip bomb
+whose header under-reports its size cannot decompress past the ceiling. The entries it returns keep the names the
+archive gives them; `pipelex_api.method_tree_request` reads them as a map of file entries, so the archive's names are
+judged by the one rule of the method tree, as the keys of `files` are: a name that is not a tree path, or that names a
+file the membership rule keeps out of every tree, is refused, and so is one name given twice.
+
+Nothing here writes to disk, imports or executes the archive's Python.
+"""
+
+import base64
+import binascii
+import zipfile
+from io import BytesIO
+
+from pipelex import log
+from pipelex.tools.log.error_fields import error_fields
+
+from pipelex_api.error_types import ErrorType
+from pipelex_api.errors import raise_bad_request, raise_payload_too_large, raise_validation_error
+from pipelex_api.limits import MAX_BUNDLE_FILES, MAX_BUNDLE_TOTAL_BYTES
+
+# Base64 inflates by 4/3; a zip is compressed, so a bundle whose DECOMPRESSED content is within
+# the ceiling encodes to well under this. Bounding the base64 string BEFORE decoding stops a
+# ~100 MiB request-body (the only other bound) from being expanded into ~75 MiB of heap just to
+# be rejected later by the decompressed-size guard. Slack (+4) covers padding.
+_MAX_BUNDLE_B64_CHARS = MAX_BUNDLE_TOTAL_BYTES * 4 // 3 + 4
+
+
+def _guard_count(count: int) -> None:
+    if count == 0:
+        raise_validation_error(message="Bundle is empty (no files)", error_type=ErrorType.INVALID_BUNDLE)
+    if count > MAX_BUNDLE_FILES:
+        raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_FILES}-file limit (got {count})")
+
+
+def _guard_running_total(total_bytes: int) -> None:
+    if total_bytes > MAX_BUNDLE_TOTAL_BYTES:
+        raise_payload_too_large(message=f"Bundle exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB decompressed-size limit")
+
+
+def decode_bundle_archive(*, bundle_b64: str) -> list[tuple[str, bytes]]:
+    """Decode a base64 zip and return its file members as `(name, bytes)`, in archive order, directories skipped.
+
+    Zip-bomb guard: each member is read through a bounded stream so a lying uncompressed-size header cannot force
+    unbounded decompression — the running total is checked against `MAX_BUNDLE_TOTAL_BYTES` as bytes are pulled. A
+    cheap length check on the still-encoded string runs FIRST, so an oversized payload is refused before it is buffered
+    into memory as decoded bytes. A name the archive gives twice is returned twice, for the tree's reader to refuse.
+
+    Raises (as problem documents): an oversized archive (413 `PayloadTooLarge`), a value that is not base64 (400
+    `InvalidBase64`), and one that is not a zip archive or holds no file (422 `InvalidBundle`).
+    """
+    if len(bundle_b64) > _MAX_BUNDLE_B64_CHARS:
+        raise_payload_too_large(message=f"bundle_b64 exceeds the {MAX_BUNDLE_TOTAL_BYTES // 1024} KiB compressed-size limit")
+    try:
+        raw = base64.b64decode(bundle_b64, validate=True)
+    except (binascii.Error, ValueError) as decode_error:
+        # DEBUG: the refusal below is a caller's mistake, which the error handler already logs; this keeps the cause.
+        log.debug("A bundle's base64 could not be decoded", fields=error_fields(exc=decode_error))
+        raise_bad_request(message="bundle_b64 is not valid base64", error_type=ErrorType.INVALID_BASE64)
+
+    try:
+        archive = zipfile.ZipFile(BytesIO(raw))
+    except zipfile.BadZipFile as zip_error:
+        # DEBUG: the refusal below is a caller's mistake, which the error handler already logs; this keeps the cause.
+        log.debug("A bundle is not a valid zip archive", fields=error_fields(exc=zip_error))
+        raise_validation_error(message="bundle_b64 is not a valid zip archive", error_type=ErrorType.INVALID_BUNDLE)
+
+    entries: list[tuple[str, bytes]] = []
+    total_bytes = 0
+    with archive:
+        members = [info for info in archive.infolist() if not info.is_dir()]
+        _guard_count(len(members))
+        budget = MAX_BUNDLE_TOTAL_BYTES
+        for info in members:
+            # Bounded read: pull at most (remaining budget + 1) bytes so a zip bomb whose header
+            # under-reports its size still cannot decompress past the ceiling.
+            with archive.open(info) as member:
+                data = member.read(budget + 1)
+            total_bytes += len(data)
+            _guard_running_total(total_bytes)
+            budget = MAX_BUNDLE_TOTAL_BYTES - total_bytes
+            entries.append((info.filename, data))
+    return entries

pipelex_api-0.87.0/pipelex_api/bundle_resources.py

@@ -0,0 +1,84 @@
+"""Decode the files a request sends in `resources`, the files its bundle's steps read, such as a template file.
+
+`resources` is a JSON object mapping each file's bundle-relative path, the exact path a step names (a `PipeDocGen`
+step's `template_file`), to the file's bytes in standard base64. Beside `mthds_contents`, every bundle of the request is
+read as sitting in one directory, so one path holds one file, and the decoded files go to the runtime as its
+`submitted_files`, keyed by canonical path, which refuses a file none of the bundle's steps names once the bundle is
+parsed, and holds the rest as the files of the tree the bundle sits at the root of. Beside a method tree (`files` or
+`bundle_b64`), the decoded files join the tree at their paths from its root (`pipelex_api.method_tree_request`), where
+the tree's rule judges them as it judges the tree's own files.
+
+Refusals, in the order they are checked:
+
+- more `resources` than `MAX_RESOURCES_PER_REQUEST` → 413 `PayloadTooLarge`;
+- a path that cannot name a file inside the bundle (absolute, a `..` segment, a backslash, a colon, empty, or a
+  second spelling of a path already submitted) → 422 `InvalidResourcePath`, before any value is decoded;
+- a file whose encoded length already exceeds the per-file ceiling → 413 `PayloadTooLarge`, before decoding;
+- a value that is not valid base64 → 400 `InvalidBase64`, as `bundle_b64` is;
+- a decoded file over `MAX_RESOURCE_FILE_BYTES`, or a running total over `MAX_RESOURCES_TOTAL_BYTES` → 413.
+"""
+
+import base64
+import binascii
+from typing import NoReturn
+
+from pipelex import log
+from pipelex.core.bundle_resources.bundle_resource import canonical_resource_path
+from pipelex.core.bundle_resources.exceptions import BundleResourcePathError
+from pipelex.tools.log.error_fields import error_fields
+
+from pipelex_api.error_types import ErrorType
+from pipelex_api.errors import raise_bad_request, raise_payload_too_large, raise_validation_error
+from pipelex_api.limits import MAX_RESOURCE_FILE_BYTES, MAX_RESOURCES_PER_REQUEST, MAX_RESOURCES_TOTAL_BYTES
+
+# Base64 inflates by 4/3, so a file within the ceiling encodes to at most this many characters (+4 covers padding).
+# Checking the encoded length first refuses an oversized file before it is decoded into memory.
+_MAX_RESOURCE_B64_CHARS = MAX_RESOURCE_FILE_BYTES * 4 // 3 + 4
+
+
+def refuse_two_files_under_one_path(*, path: str, where: str) -> NoReturn:
+    """Refuse two different files sent under one path, naming the path, with a 422 `InvalidResourcePath`."""
+    msg = (
+        f"Two different files are named '{path}', {where}. A path names one file of a method: send one of them, or rename "
+        "one and the step that reads it."
+    )
+    raise_validation_error(message=msg, error_type=ErrorType.INVALID_RESOURCE_PATH)
+
+
+def decode_bundle_resources(*, resources: dict[str, str] | None) -> dict[str, bytes] | None:
+    """The files `resources` sends, decoded and keyed by canonical path, or None when there are none."""
+    if not resources:
+        return None
+    if len(resources) > MAX_RESOURCES_PER_REQUEST:
+        raise_payload_too_large(message=f"resources exceeds the {MAX_RESOURCES_PER_REQUEST}-file limit (got {len(resources)})")
+    # Every path first, so a second spelling of one file is refused as such before any value is decoded.
+    sent_by_path: dict[str, tuple[str, str]] = {}
+    for path, encoded in resources.items():
+        try:
+            canonical_path = canonical_resource_path(path=path)
+        except BundleResourcePathError as exc:
+            raise_validation_error(message=exc.message, error_type=ErrorType.INVALID_RESOURCE_PATH)
+        if canonical_path in sent_by_path:
+            msg = f"Two files submitted with the bundle name the same file '{canonical_path}'."
+            raise_validation_error(message=msg, error_type=ErrorType.INVALID_RESOURCE_PATH)
+        sent_by_path[canonical_path] = (path, encoded)
+    files: dict[str, bytes] = {}
+    total_bytes = 0
+    for canonical_path, (path, encoded) in sent_by_path.items():
+        if len(encoded) > _MAX_RESOURCE_B64_CHARS:
+            raise_payload_too_large(message=f"The file '{path}' in resources exceeds the {MAX_RESOURCE_FILE_BYTES // 1024} KiB per-file limit")
+        try:
+            data = base64.b64decode(encoded, validate=True)
+        except (binascii.Error, ValueError) as decode_error:
+            # DEBUG: the refusal below is a caller's mistake, which the error handler already logs; this keeps the cause.
+            log.debug("A submitted file's base64 could not be decoded", fields=error_fields(exc=decode_error))
+            raise_bad_request(message=f"The file '{path}' in resources is not valid base64", error_type=ErrorType.INVALID_BASE64)
+        if len(data) > MAX_RESOURCE_FILE_BYTES:
+            raise_payload_too_large(
+                message=f"The file '{canonical_path}' a step reads exceeds the {MAX_RESOURCE_FILE_BYTES // 1024} KiB per-file limit"
+            )
+        total_bytes += len(data)
+        if total_bytes > MAX_RESOURCES_TOTAL_BYTES:
+            raise_payload_too_large(message=f"The files the bundle's steps read exceed the {MAX_RESOURCES_TOTAL_BYTES // 1024} KiB total-size limit")
+        files[canonical_path] = data
+    return files

{pipelex_api-0.86.1 → pipelex_api-0.87.0}/pipelex_api/error_types.py

@@ -58,9 +58,34 @@ class ErrorType(StrEnum):
     INVALID_MODEL_REFERENCE = "InvalidModelReference"
     INVALID_BASE64 = "InvalidBase64"
     PAYLOAD_TOO_LARGE = "PayloadTooLarge"
-    # A run request carried a malformed method bundle (`bundle_b64` / `files`): a corrupt zip, an
-    # unsafe entry name (absolute path or `..` traversal), or both transport forms supplied at once.
+    # A run request carried a malformed method bundle (`bundle_b64` / `files`): a corrupt zip, a
+    # bundle with no `.mthds` file, or both transport forms supplied at once.
     INVALID_BUNDLE = "InvalidBundle"
+    # A request's map of file entries (`files`, a tree of `dependencies`, a `bundle_b64` archive read as
+    # one) is not a method tree the MTHDS standard admits: a key given twice, a key that is not a tree
+    # path or names a file the membership rule keeps out of every tree, keys that collide, an empty map,
+    # or a malformed entry. A 422 whose `tree_refusal` member carries the standard's refusal code and
+    # the paths it names.
+    INVALID_METHOD_FILES = "InvalidMethodFiles"
+    # A request names, by hash, bytes this runner's blob store does not hold at the stated size. A 422
+    # whose `missing_blobs` member lists every such hash at once, so a sender learns in one answer what to
+    # upload (`POST /v1/blobs/missing`, then `PUT /v1/blobs/{sha256}`).
+    MISSING_BLOBS = "MissingBlobs"
+    # A method tree, or a blob, passes one of this runner's limits on a tree's file count, its total
+    # size or one file's size (`MAX_METHOD_TREE_FILES`, `MAX_METHOD_TREE_TOTAL_KIB`,
+    # `MAX_METHOD_TREE_FILE_KIB`). A 422 naming the limit, refused before any blob is read or uploaded.
+    METHOD_TREE_TOO_LARGE = "MethodTreeTooLarge"
+    # `PUT /v1/blobs/{sha256}` was sent bytes whose SHA-256 is not the one its path names,
+    # `POST /v1/blobs/missing` named one hash twice with different sizes, or a request's hash entry states a size other
+    # than the one this runner holds that hash's bytes at, which uploading them again would not change. A 422 whose
+    # `mismatched_blobs` member, on a request's entries, lists each such hash with the size held.
+    BLOB_MISMATCH = "BlobMismatch"
+    # The bytes this runner's blob store holds under a hash do not hash to it: the storage is damaged. A 500 no caller
+    # can repair; the operator removes the object, and the blob is then uploaded again.
+    BLOB_STORE_CORRUPTED = "BlobStoreCorrupted"
+    # `PUT /v1/blobs/{sha256}` was called without the signed query `POST /v1/blobs/missing` names in its upload
+    # target, with one whose signature does not hold, or past its expiry. A 403: the upload is not stored.
+    BLOB_UPLOAD_NOT_AUTHORIZED = "BlobUploadNotAuthorized"
     # A request's `resources` map keys a submitted file by a path that cannot name a file inside the
     # bundle: absolute, with a `..` segment, a backslash or a colon, empty, or a second spelling of a
     # path already submitted. A 422: the key is the path a step names, relative to the bundle.

{pipelex_api-0.86.1 → pipelex_api-0.87.0}/pipelex_api/errors.py

@@ -15,6 +15,7 @@ are the single translation point for everything the API does not author
 itself.
 """
 
+from collections.abc import Mapping
 from typing import Any, NoReturn
 
 from pipelex.base_exceptions import ErrorDomain
@@ -49,6 +50,7 @@ def _raise_api_error(
     status: int,
     error_domain: ErrorDomain,
     headers: dict[str, str] | None = None,
+    members: Mapping[str, Any] | None = None,
 ) -> NoReturn:
     """Build the RFC 7807 document and raise `ApiError`.
 
@@ -67,6 +69,9 @@ def _raise_api_error(
         request_id=None,
         error_domain=error_domain,
     )
+    if members:
+        # Extension members of RFC 7807 (section 3.2), such as `missing_blobs`: data a client acts on, beside the prose.
+        document.update(members)
     raise ApiError(status_code=status, document=document, headers=headers)
 
 
@@ -75,6 +80,15 @@ def raise_validation_error(message: str, error_type: ErrorType = ErrorType.VALID
     _raise_api_error(error_type=error_type, message=message, status=422, error_domain=ErrorDomain.INPUT)
 
 
+def raise_validation_problem(*, message: str, error_type: ErrorType, members: Mapping[str, Any]) -> NoReturn:
+    """Raise a 422 RFC 7807 problem response for invalid caller input, carrying extension members a client acts on.
+
+    The members join the document beside `detail`, such as the `missing_blobs` a sender uploads before it sends the
+    request again, or the `tree_refusal` naming the standard's refusal code and the paths it names.
+    """
+    _raise_api_error(error_type=error_type, message=message, status=422, error_domain=ErrorDomain.INPUT, members=members)
+
+
 def raise_bad_request(message: str, error_type: ErrorType = ErrorType.BAD_REQUEST) -> NoReturn:
     """Raise a 400 RFC 7807 problem response for a malformed request."""
     _raise_api_error(error_type=error_type, message=message, status=400, error_domain=ErrorDomain.INPUT)

{pipelex_api-0.86.1 → pipelex_api-0.87.0}/pipelex_api/exception_handlers.py

@@ -341,11 +341,6 @@ _ERROR_TYPE_STATUS_OVERRIDES: dict[str, int] = {
     #     sandboxed PipeFuncs, not in-process Python) → 403, the same status as the bundle
     #     transport's custom-code gate.
     "MethodStructuresRefusedError": 403,
-    # The files submitted in `resources` that the bundle's steps read when the method loads, its HTML templates,
-    # total more than they may, since they travel inline with the method wherever it runs. Only the parsed bundle
-    # says which files those are, so the library refuses them rather than this server's decoding: the request
-    # entity itself is too large → 413, as the decoder's own size refusals are.
-    "BundleResourcesTooLargeError": 413,
 }
 
 

{pipelex_api-0.86.1 → pipelex_api-0.87.0}/pipelex_api/limits.py

@@ -23,11 +23,20 @@ MAX_METHOD_REF_LEN = 512  # `method_ref` selector strings; a fixed schema bound,
 MAX_JSON_NESTING_DEPTH = 128
 DEFAULT_MAX_CALLBACK_URLS = 5
 DEFAULT_MAX_CALLBACK_URL_LEN = 2048
-DEFAULT_MAX_BUNDLE_FILES = 128  # entries in a materialized method bundle (.mthds + .py + requirements.txt)
+DEFAULT_MAX_BUNDLE_FILES = 128  # members of a `bundle_b64` archive, its shipped packages' files included
 DEFAULT_MAX_BUNDLE_TOTAL_KIB = 8 * 1024  # 8 MiB decompressed across the whole bundle (zip-bomb guard)
 DEFAULT_MAX_RESOURCES_PER_REQUEST = 32  # files submitted with a bundle that its steps read (`resources`)
 DEFAULT_MAX_RESOURCE_FILE_KIB = 5 * 1024  # 5 MiB decoded per submitted file, such as an office template
 DEFAULT_MAX_RESOURCES_TOTAL_KIB = 10 * 1024  # 10 MiB decoded across a request's submitted files
+# A method tree a request sends (`files`, each tree of `dependencies`, a `bundle_b64` archive) or fetches by `method_ref`,
+# whatever form its files travel in: inline, Base64 or by hash. The request body bounds only the inline entries.
+DEFAULT_MAX_METHOD_TREE_FILES = 1000
+DEFAULT_MAX_METHOD_TREE_TOTAL_KIB = 64 * 1024  # 64 MiB across the files of one tree
+DEFAULT_MAX_METHOD_TREE_FILE_KIB = 32 * 1024  # 32 MiB for one file of a tree, and for one blob uploaded
+# Every tree one request carries together, the method's own and its dependency trees, beside each tree's own limits.
+DEFAULT_MAX_METHOD_DEPENDENCIES = 32  # dependency trees beside the method's own
+DEFAULT_MAX_METHOD_REQUEST_TOTAL_KIB = 128 * 1024  # 128 MiB across the method's tree and its dependency trees
+DEFAULT_BLOB_UPLOAD_URL_TTL_SECONDS = 3600  # how long an upload target `POST /v1/blobs/missing` names stays valid
 DEFAULT_MAX_METHOD_CACHE_CLONES = 64  # cached method-package clones (one per resolved commit SHA)
 DEFAULT_MAX_METHOD_CACHE_TOTAL_KIB = 512 * 1024  # 512 MiB across all cached clones
 DEFAULT_MAX_METHOD_CACHE_AGE_HOURS = 24  # a cached clone unused for this long is evicted
@@ -50,6 +59,15 @@ MAX_RESOURCES_PER_REQUEST = get_positive_int_env(env_var="MAX_RESOURCES_PER_REQU
 MAX_RESOURCE_FILE_BYTES = get_positive_int_env(env_var="MAX_RESOURCE_FILE_KIB", default=DEFAULT_MAX_RESOURCE_FILE_KIB) * 1024
 MAX_RESOURCES_TOTAL_BYTES = get_positive_int_env(env_var="MAX_RESOURCES_TOTAL_KIB", default=DEFAULT_MAX_RESOURCES_TOTAL_KIB) * 1024
 
+MAX_METHOD_TREE_FILES = get_positive_int_env(env_var="MAX_METHOD_TREE_FILES", default=DEFAULT_MAX_METHOD_TREE_FILES)
+MAX_METHOD_TREE_TOTAL_BYTES = get_positive_int_env(env_var="MAX_METHOD_TREE_TOTAL_KIB", default=DEFAULT_MAX_METHOD_TREE_TOTAL_KIB) * 1024
+MAX_METHOD_TREE_FILE_BYTES = get_positive_int_env(env_var="MAX_METHOD_TREE_FILE_KIB", default=DEFAULT_MAX_METHOD_TREE_FILE_KIB) * 1024
+
+MAX_METHOD_DEPENDENCIES = get_positive_int_env(env_var="MAX_METHOD_DEPENDENCIES", default=DEFAULT_MAX_METHOD_DEPENDENCIES)
+MAX_METHOD_REQUEST_TOTAL_BYTES = get_positive_int_env(env_var="MAX_METHOD_REQUEST_TOTAL_KIB", default=DEFAULT_MAX_METHOD_REQUEST_TOTAL_KIB) * 1024
+
+BLOB_UPLOAD_URL_TTL_SECONDS = get_positive_int_env(env_var="BLOB_UPLOAD_URL_TTL_SECONDS", default=DEFAULT_BLOB_UPLOAD_URL_TTL_SECONDS)
+
 MAX_METHOD_CACHE_CLONES = get_positive_int_env(env_var="MAX_METHOD_CACHE_CLONES", default=DEFAULT_MAX_METHOD_CACHE_CLONES)
 MAX_METHOD_CACHE_TOTAL_BYTES = get_positive_int_env(env_var="MAX_METHOD_CACHE_TOTAL_KIB", default=DEFAULT_MAX_METHOD_CACHE_TOTAL_KIB) * 1024
 MAX_METHOD_CACHE_AGE_SECONDS = get_positive_int_env(env_var="MAX_METHOD_CACHE_AGE_HOURS", default=DEFAULT_MAX_METHOD_CACHE_AGE_HOURS) * 3600

{pipelex_api-0.86.1 → pipelex_api-0.87.0}/pipelex_api/main.py

@@ -31,6 +31,7 @@ from pipelex_api.exception_handlers import register_exception_handlers
 from pipelex_api.middleware import RequestIdMiddleware, request_body_size_middleware
 from pipelex_api.openapi_schema import PipelexFastAPI
 from pipelex_api.routes import router as api_router
+from pipelex_api.routes.blob_upload import router as blob_upload_router
 from pipelex_api.routes.health import router as health_router
 from pipelex_api.routes.version import router as version_router
 from pipelex_api.security import get_auth_dependency
@@ -187,6 +188,11 @@ fastapi_app.include_router(health_router)
 # detection before they have credentials).
 fastapi_app.include_router(version_router, prefix="/v1")
 
+# `PUT /v1/blobs/{sha256}` is authorized by the signed query of the upload target `POST /v1/blobs/missing` names
+# (`pipelex_api.blob_upload`), so a sender uploads with exactly the headers that target names: it mounts without the
+# auth dependency, and only a caller who passed it on `POST /v1/blobs/missing` holds a target.
+fastapi_app.include_router(blob_upload_router, prefix="/v1")
+
 # Register all other routes WITH authentication, the dependency selected by the AUTH_MODE env var. An
 # unrecognized AUTH_MODE raises `InvalidAuthModeError` here and the app fails to boot, the same fail-fast
 # posture as ERROR_DISCLOSURE above: a mistyped mode must never serve these routes unauthenticated.